MetacouplingLLM 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. metacouplingllm/__init__.py +208 -0
  2. metacouplingllm/core.py +4542 -0
  3. metacouplingllm/data/Papers.zip +0 -0
  4. metacouplingllm/data/PeriTelecoupling_clean.csv +49953 -0
  5. metacouplingllm/data/PeriTelecoupling_subset.csv +11 -0
  6. metacouplingllm/data/chunk_embeddings.manifest.json +11 -0
  7. metacouplingllm/data/chunk_embeddings.npy +0 -0
  8. metacouplingllm/data/pericoupled_adm1_edge_list.csv +8291 -0
  9. metacouplingllm/data/telecoupling_literature.bib +3498 -0
  10. metacouplingllm/knowledge/__init__.py +1 -0
  11. metacouplingllm/knowledge/adm1_pericoupling.py +591 -0
  12. metacouplingllm/knowledge/citations.py +109 -0
  13. metacouplingllm/knowledge/countries.py +557 -0
  14. metacouplingllm/knowledge/examples.py +713 -0
  15. metacouplingllm/knowledge/framework.py +717 -0
  16. metacouplingllm/knowledge/literature.py +518 -0
  17. metacouplingllm/knowledge/pericoupling.py +241 -0
  18. metacouplingllm/knowledge/rag.py +1791 -0
  19. metacouplingllm/knowledge/references.py +112 -0
  20. metacouplingllm/knowledge/websearch.py +1842 -0
  21. metacouplingllm/llm/__init__.py +1 -0
  22. metacouplingllm/llm/client.py +636 -0
  23. metacouplingllm/llm/parser.py +868 -0
  24. metacouplingllm/output/__init__.py +1 -0
  25. metacouplingllm/output/formatter.py +276 -0
  26. metacouplingllm/prompts/__init__.py +1 -0
  27. metacouplingllm/prompts/builder.py +477 -0
  28. metacouplingllm/prompts/templates.py +556 -0
  29. metacouplingllm/visualization/__init__.py +114 -0
  30. metacouplingllm/visualization/adm1_map.py +884 -0
  31. metacouplingllm/visualization/worldmap.py +1372 -0
  32. metacouplingllm-0.1.0.dist-info/METADATA +440 -0
  33. metacouplingllm-0.1.0.dist-info/RECORD +35 -0
  34. metacouplingllm-0.1.0.dist-info/WHEEL +4 -0
  35. metacouplingllm-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,1791 @@
1
+ """
2
+ Retrieval-Augmented Generation (RAG) for metacoupling paper evidence.
3
+
4
+ Loads full-text markdown papers, matches them against the BibTeX literature
5
+ database, chunks them into passages, and retrieves relevant passages given
6
+ a query (typically derived from an LLM analysis).
7
+
8
+ Two retrieval backends are available:
9
+
10
+ - :class:`EmbeddingRetriever` (default): semantic retrieval using
11
+ ``fastembed`` + pre-computed corpus embeddings bundled with the
12
+ package. Catches synonyms, paraphrases, and related concepts.
13
+ - :class:`TfIdfIndex` (fallback): lexical retrieval using TF-IDF +
14
+ cosine similarity. Fast, no external dependencies, but misses
15
+ semantic matches. Used when ``fastembed`` is not installed or the
16
+ pre-computed embedding file is missing.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import hashlib
22
+ import json
23
+ import logging
24
+ import math
25
+ import os
26
+ import re
27
+ from collections import Counter
28
+ from dataclasses import dataclass, field
29
+ from pathlib import Path
30
+ from typing import TYPE_CHECKING
31
+
32
+ if TYPE_CHECKING:
33
+ from metacouplingllm.knowledge.literature import Paper
34
+
35
+
36
+ logger = logging.getLogger(__name__)
37
+
38
+ # ---------------------------------------------------------------------------
39
+ # Bundled papers extraction
40
+ # ---------------------------------------------------------------------------
41
+
42
+ _BUNDLED_ZIP = Path(__file__).parent / ".." / "data" / "Papers.zip"
43
+
44
+
45
+ def _extract_bundled_papers(verbose: bool = False) -> Path:
46
+ """Extract the bundled ``data/Papers.zip`` to a cache directory.
47
+
48
+ Returns the path to the extracted ``Papers/`` directory. If the
49
+ files have already been extracted (same zip mtime), the cached
50
+ version is reused.
51
+ """
52
+ import shutil
53
+ import tempfile
54
+ import zipfile
55
+
56
+ zip_path = _BUNDLED_ZIP.resolve()
57
+ if not zip_path.exists():
58
+ return Path("__nonexistent__")
59
+
60
+ # Use a stable cache directory based on the package location
61
+ cache_root = Path(tempfile.gettempdir()) / "metacoupling_rag_cache"
62
+ papers_dir = cache_root / "Papers"
63
+ stamp_file = cache_root / ".extracted_stamp"
64
+
65
+ # Check if already extracted and up-to-date
66
+ zip_mtime = zip_path.stat().st_mtime
67
+ if papers_dir.is_dir() and stamp_file.exists():
68
+ try:
69
+ cached_mtime = float(stamp_file.read_text().strip())
70
+ if cached_mtime == zip_mtime:
71
+ with zipfile.ZipFile(zip_path, "r") as zf:
72
+ expected_files = {
73
+ Path(name).name
74
+ for name in zf.namelist()
75
+ if name.startswith("Papers/") and name.endswith(".md")
76
+ }
77
+ extracted_files = {
78
+ path.name for path in papers_dir.glob("*.md")
79
+ }
80
+ if expected_files == extracted_files:
81
+ if verbose:
82
+ print(
83
+ f"[RAG] Using cached bundled papers: {papers_dir}"
84
+ )
85
+ return papers_dir
86
+ if verbose:
87
+ print("[RAG] Refreshing bundled papers cache: file list changed.")
88
+ except (ValueError, OSError):
89
+ pass
90
+
91
+ # Extract
92
+ if verbose:
93
+ print(f"[RAG] Extracting bundled Papers.zip → {cache_root}")
94
+
95
+ cache_root.mkdir(parents=True, exist_ok=True)
96
+ if papers_dir.exists():
97
+ shutil.rmtree(papers_dir, ignore_errors=True)
98
+ with zipfile.ZipFile(zip_path, "r") as zf:
99
+ zf.extractall(cache_root)
100
+
101
+ stamp_file.write_text(str(zip_mtime))
102
+
103
+ if verbose:
104
+ md_count = sum(1 for f in papers_dir.iterdir() if f.suffix == ".md")
105
+ print(f"[RAG] Extracted {md_count} bundled paper files.")
106
+
107
+ return papers_dir
108
+
109
+
110
+ # ---------------------------------------------------------------------------
111
+ # Data structures
112
+ # ---------------------------------------------------------------------------
113
+
114
+
115
+ @dataclass
116
+ class TextChunk:
117
+ """A passage extracted from a full-text paper.
118
+
119
+ Attributes
120
+ ----------
121
+ paper_key:
122
+ BibTeX citation key of the matched paper.
123
+ paper_title:
124
+ Title of the paper.
125
+ authors:
126
+ Author string.
127
+ year:
128
+ Publication year.
129
+ section:
130
+ Section heading the passage comes from (e.g., "INTRODUCTION").
131
+ text:
132
+ The actual passage text.
133
+ chunk_index:
134
+ Position of this chunk within the paper (0-indexed).
135
+ """
136
+
137
+ paper_key: str = ""
138
+ paper_title: str = ""
139
+ authors: str = ""
140
+ year: int = 0
141
+ section: str = ""
142
+ text: str = ""
143
+ chunk_index: int = 0
144
+
145
+
146
+ @dataclass
147
+ class RetrievalResult:
148
+ """A retrieved passage with its relevance score.
149
+
150
+ Attributes
151
+ ----------
152
+ chunk:
153
+ The text chunk that was retrieved.
154
+ score:
155
+ Cosine similarity score (0–1).
156
+ """
157
+
158
+ chunk: TextChunk
159
+ score: float = 0.0
160
+
161
+
162
+ # ---------------------------------------------------------------------------
163
+ # Paper matching: filename → BibTeX entry
164
+ # ---------------------------------------------------------------------------
165
+
166
+ # Pattern for Zotero-style markdown filenames:
167
+ # "Author et al. - YYYY - Title.md"
168
+ # "Author and Author2 - YYYY - Title.md"
169
+ _FILENAME_RE = re.compile(
170
+ r"^(?P<authors>.+?)\s*-\s*(?P<year>\d{4})\s*-\s*(?P<title>.+)\.md$",
171
+ re.IGNORECASE,
172
+ )
173
+
174
+
175
+ def _normalise_for_match(text: str) -> str:
176
+ """Normalise text for fuzzy matching: lowercase, collapse whitespace, strip punctuation."""
177
+ text = text.lower()
178
+ text = re.sub(r"[^\w\s]", "", text)
179
+ text = re.sub(r"\s+", " ", text).strip()
180
+ return text
181
+
182
+
183
+ def _match_paper_to_db(
184
+ filename: str, db_papers: list[Paper],
185
+ ) -> Paper | None:
186
+ """Match a markdown filename to a BibTeX Paper entry.
187
+
188
+ Uses a two-step approach:
189
+ 1. Extract year from filename and filter candidates.
190
+ 2. Fuzzy-match the title from the filename against candidate titles.
191
+
192
+ Returns the best matching Paper, or None if no match found.
193
+ """
194
+ m = _FILENAME_RE.match(filename)
195
+ if not m:
196
+ return None
197
+
198
+ file_year = int(m.group("year"))
199
+ file_title = _normalise_for_match(m.group("title"))
200
+ file_authors = _normalise_for_match(m.group("authors"))
201
+
202
+ # Filter by year (allow ±1 year tolerance for pre-prints)
203
+ candidates = [p for p in db_papers if abs(p.year - file_year) <= 1]
204
+ if not candidates:
205
+ return None
206
+
207
+ # Score each candidate by title word overlap
208
+ best_score = 0.0
209
+ best_paper: Paper | None = None
210
+
211
+ file_title_words = set(file_title.split())
212
+ file_author_words = set(file_authors.split())
213
+
214
+ for paper in candidates:
215
+ paper_title = _normalise_for_match(paper.title)
216
+ paper_title_words = set(paper_title.split())
217
+
218
+ if not paper_title_words:
219
+ continue
220
+
221
+ # Jaccard similarity on title words
222
+ intersection = file_title_words & paper_title_words
223
+ union = file_title_words | paper_title_words
224
+ title_score = len(intersection) / len(union) if union else 0.0
225
+
226
+ # Bonus for author match
227
+ paper_author_norm = _normalise_for_match(paper.authors)
228
+ paper_author_words = set(paper_author_norm.split())
229
+ author_overlap = len(file_author_words & paper_author_words)
230
+ author_bonus = min(0.2, author_overlap * 0.05)
231
+
232
+ score = title_score + author_bonus
233
+
234
+ if score > best_score:
235
+ best_score = score
236
+ best_paper = paper
237
+
238
+ # Require at least 30% title overlap
239
+ if best_score < 0.3:
240
+ return None
241
+
242
+ return best_paper
243
+
244
+
245
+ def _paper_from_filename(filename: str) -> Paper | None:
246
+ """Create fallback paper metadata from a Zotero-style markdown filename."""
247
+ from metacouplingllm.knowledge.literature import Paper
248
+
249
+ m = _FILENAME_RE.match(filename)
250
+ if m:
251
+ title = re.sub(r"\s+", " ", m.group("title")).strip()
252
+ authors = re.sub(r"\s+", " ", m.group("authors")).strip()
253
+ try:
254
+ year = int(m.group("year"))
255
+ except ValueError:
256
+ year = 0
257
+ else:
258
+ stem = Path(filename).stem
259
+ year_match = re.search(r"\b((?:18|19|20)\d{2})\b", stem)
260
+ year = int(year_match.group(1)) if year_match else 0
261
+ if " - " in stem:
262
+ authors, title = stem.split(" - ", 1)
263
+ elif "_" in stem:
264
+ authors, title = stem.split("_", 1)
265
+ else:
266
+ authors, title = "Unknown", stem
267
+ authors = re.sub(r"\s+", " ", authors).strip() or "Unknown"
268
+ title = re.sub(r"[_\s]+", " ", title).strip() or stem
269
+ if not title:
270
+ return None
271
+ if not title:
272
+ return None
273
+ if not authors:
274
+ authors = "Unknown"
275
+ if not isinstance(year, int):
276
+ year = 0
277
+ key_source = f"{authors}_{year}_{title}".lower()
278
+ key = re.sub(r"[^a-z0-9]+", "_", key_source).strip("_")[:80]
279
+ return Paper(
280
+ key=key or _normalise_for_match(filename).replace(" ", "_")[:80],
281
+ title=title,
282
+ authors=authors,
283
+ year=year,
284
+ keywords={
285
+ "telecoupling",
286
+ "metacoupling",
287
+ "rag-fallback-metadata",
288
+ },
289
+ )
290
+
291
+
292
+ # ---------------------------------------------------------------------------
293
+ # Text chunking
294
+ # ---------------------------------------------------------------------------
295
+
296
+ # Section heading patterns in markdown
297
+ _HEADING_RE = re.compile(r"^#{1,3}\s+(.+)$", re.MULTILINE)
298
+ _REFERENCE_SECTION_RE = re.compile(
299
+ r"(?im)^(?:#{1,6}\s*)?"
300
+ r"(references|bibliography|literature cited)\s*$"
301
+ )
302
+ _BIBLIOGRAPHY_HEADING_RE = re.compile(
303
+ r"^(?:(?:18|19|20)\d{2}|\d{3,})\."
304
+ )
305
+ _REFERENCE_LEAD_RE = re.compile(
306
+ r"^(?:(?:[A-Z]\.\s*)?(?:18|19|20)\d{2}\.|"
307
+ r"\d{3,}\.\s+[A-Z])"
308
+ )
309
+ _REFERENCE_AUTHOR_RE = re.compile(
310
+ r"\b[A-Z][A-Za-z'`.-]+,\s*(?:[A-Z]\.\s*){1,3}"
311
+ )
312
+ _REFERENCE_YEAR_RE = re.compile(r"(?:\(|\b)(?:18|19|20)\d{2}(?:\)|\.)")
313
+ _REFERENCE_TERM_RE = re.compile(
314
+ r"\b(?:doi|proceedings|thesis|dissertation|editors?|"
315
+ r"university|press|journal|rome|fao|pages?)\b",
316
+ re.IGNORECASE,
317
+ )
318
+ _REFERENCE_URL_RE = re.compile(r"https?://|doi[:/]", re.IGNORECASE)
319
+
320
+
321
+ def _truncate_at_references(text: str) -> str:
322
+ """Drop everything from the first references-style section onward."""
323
+ match = _REFERENCE_SECTION_RE.search(text)
324
+ if not match:
325
+ return text
326
+ return text[:match.start()].rstrip()
327
+
328
+
329
+ def _is_reference_heading(heading: str) -> bool:
330
+ """Return True when *heading* looks like bibliography content."""
331
+ heading = heading.strip()
332
+ heading_lower = heading.lower()
333
+ if heading_lower in (
334
+ "references", "bibliography", "literature cited",
335
+ "acknowledgments", "acknowledgements", "appendix",
336
+ "supplementary material", "supporting information",
337
+ ):
338
+ return True
339
+ return bool(_BIBLIOGRAPHY_HEADING_RE.match(heading))
340
+
341
+
342
+ def _looks_like_reference_chunk(text: str) -> bool:
343
+ """Heuristically reject chunks that read like bibliography entries."""
344
+ compact = re.sub(r"\s+", " ", text).strip()
345
+ if not compact:
346
+ return False
347
+
348
+ year_count = len(_REFERENCE_YEAR_RE.findall(compact))
349
+ author_count = len(_REFERENCE_AUTHOR_RE.findall(compact))
350
+ term_count = len(_REFERENCE_TERM_RE.findall(compact))
351
+ has_url = bool(_REFERENCE_URL_RE.search(compact))
352
+ has_page_ref = bool(re.search(r"\bpages?\s+\d", compact, re.IGNORECASE))
353
+ has_reference_lead = bool(_REFERENCE_LEAD_RE.match(compact))
354
+
355
+ if term_count >= 3:
356
+ return True
357
+ if has_reference_lead and (term_count >= 1 or has_page_ref or year_count >= 1):
358
+ return True
359
+ if has_page_ref and term_count >= 2:
360
+ return True
361
+ if author_count >= 2 and (year_count >= 1 or term_count >= 1 or has_page_ref):
362
+ return True
363
+ if year_count >= 2 and (term_count >= 1 or has_url or has_page_ref):
364
+ return True
365
+ return False
366
+
367
+
368
+ def _chunk_markdown(
369
+ text: str,
370
+ paper: Paper,
371
+ max_chunk_words: int = 250,
372
+ overlap_words: int = 50,
373
+ ) -> list[TextChunk]:
374
+ """Split a markdown document into overlapping text chunks.
375
+
376
+ Strategy:
377
+ 1. Split by section headings (## or ###).
378
+ 2. Within each section, split into ~max_chunk_words chunks with overlap.
379
+ 3. Filter out very short chunks (< 30 words) and non-text chunks.
380
+ """
381
+ # Remove image placeholders and citation artifacts
382
+ text = re.sub(r"<!--\s*image\s*-->", "", text)
383
+ text = re.sub(r"\|\s*#\s*\|.*?\n", "", text) # table headers
384
+ text = _truncate_at_references(text)
385
+
386
+ # Split into sections by headings
387
+ sections: list[tuple[str, str]] = []
388
+ parts = _HEADING_RE.split(text)
389
+
390
+ # parts alternates: [pre-heading text, heading1, text1, heading2, text2, ...]
391
+ current_heading = "HEADER"
392
+ i = 0
393
+ if len(parts) > 0 and not _HEADING_RE.match(parts[0] if parts[0] else ""):
394
+ sections.append((current_heading, parts[0]))
395
+ i = 1
396
+
397
+ while i < len(parts) - 1:
398
+ heading = parts[i].strip()
399
+ body = parts[i + 1] if i + 1 < len(parts) else ""
400
+ sections.append((heading, body))
401
+ i += 2
402
+
403
+ chunks: list[TextChunk] = []
404
+ chunk_idx = 0
405
+
406
+ for heading, body in sections:
407
+ if _is_reference_heading(heading):
408
+ continue
409
+
410
+ # Clean the body text
411
+ body = re.sub(r"\s+", " ", body).strip()
412
+ if not body:
413
+ continue
414
+
415
+ words = body.split()
416
+ if len(words) < 30:
417
+ continue
418
+
419
+ # Split into chunks with overlap
420
+ start = 0
421
+ while start < len(words):
422
+ end = min(start + max_chunk_words, len(words))
423
+ chunk_words = words[start:end]
424
+ chunk_text = " ".join(chunk_words)
425
+
426
+ if len(chunk_words) >= 30 and not _looks_like_reference_chunk(
427
+ chunk_text
428
+ ):
429
+ chunks.append(TextChunk(
430
+ paper_key=paper.key,
431
+ paper_title=paper.title,
432
+ authors=paper.authors,
433
+ year=paper.year,
434
+ section=heading,
435
+ text=chunk_text,
436
+ chunk_index=chunk_idx,
437
+ ))
438
+ chunk_idx += 1
439
+
440
+ # Advance with overlap
441
+ if end >= len(words):
442
+ break
443
+ start = end - overlap_words
444
+
445
+ return chunks
446
+
447
+
448
+ # ---------------------------------------------------------------------------
449
+ # TF-IDF retrieval engine
450
+ # ---------------------------------------------------------------------------
451
+
452
+ # Stopwords for TF-IDF
453
+ _STOPWORDS = frozenset({
454
+ "a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for",
455
+ "of", "with", "by", "from", "is", "it", "as", "be", "was", "were",
456
+ "are", "been", "being", "has", "had", "have", "do", "does", "did",
457
+ "will", "would", "could", "should", "may", "might", "can", "shall",
458
+ "not", "no", "nor", "so", "if", "then", "than", "that", "this",
459
+ "these", "those", "their", "its", "our", "his", "her", "my", "your",
460
+ "we", "they", "he", "she", "you", "i", "me", "us", "them", "him",
461
+ "also", "more", "most", "much", "many", "some", "any", "all", "each",
462
+ "every", "both", "few", "several", "other", "another", "such", "only",
463
+ "just", "very", "too", "even", "still", "already", "about", "over",
464
+ "after", "before", "between", "through", "during", "into", "out",
465
+ "up", "down", "which", "who", "whom", "what", "where", "when", "how",
466
+ "while", "there", "here", "et", "al", "fig", "figure", "table",
467
+ "however", "thus", "therefore", "although", "because", "since",
468
+ "whether", "either", "neither", "per", "via",
469
+ })
470
+
471
+ _WORD_RE = re.compile(r"[a-z][a-z0-9-]+")
472
+
473
+
474
+ def _tokenise(text: str) -> list[str]:
475
+ """Tokenise text into lowercase words, removing stopwords."""
476
+ words = _WORD_RE.findall(text.lower())
477
+ return [w for w in words if w not in _STOPWORDS and len(w) > 2]
478
+
479
+
480
+ _GENERIC_CITATION_TOKENS = frozenset({
481
+ "agent", "agents", "cause", "causes", "coupled", "coupling",
482
+ "effect", "effects", "flow", "flows", "framework", "human",
483
+ "humannature", "impact", "impacts", "information", "interaction",
484
+ "interactions", "intracoupling", "metacoupling", "nature",
485
+ "pericoupling", "receiving", "sending", "spillover", "system",
486
+ "systems", "telecoupling",
487
+ })
488
+
489
+
490
+ def _specific_tokens(tokens: set[str] | list[str]) -> set[str]:
491
+ """Return the non-generic subset of *tokens* for citation matching."""
492
+ return {t for t in tokens if t not in _GENERIC_CITATION_TOKENS}
493
+
494
+
495
+ def _citation_policy(section: str) -> tuple[int, float, int, int]:
496
+ """Return citation thresholds for the current output section.
497
+
498
+ Returns
499
+ -------
500
+ ``(min_overlap, min_ratio, min_specific_overlap, max_citations)``
501
+ """
502
+ if section in {"CAUSES", "EFFECTS", "RESEARCH GAPS"}:
503
+ return 4, 0.35, 2, 2
504
+ return 3, 0.20, 1, 3
505
+
506
+
507
+ def _best_excerpt(
508
+ text: str,
509
+ anchor_text: str = "",
510
+ max_chars: int = 300,
511
+ ) -> str:
512
+ """Extract a short excerpt centered on the most relevant passage."""
513
+ compact = re.sub(r"\s+", " ", text).strip()
514
+ if len(compact) <= max_chars:
515
+ return compact
516
+
517
+ anchor_tokens = _specific_tokens(set(_tokenise(anchor_text)))
518
+ if not anchor_tokens:
519
+ return compact[:max_chars].rstrip() + "..."
520
+
521
+ words = compact.split()
522
+ window_words = max(30, min(60, max_chars // 6))
523
+ step = max(10, window_words // 3)
524
+ best_range = (0, min(len(words), window_words))
525
+ best_score = (-1.0, -1.0, -1.0)
526
+
527
+ for start in range(0, len(words), step):
528
+ end = min(len(words), start + window_words)
529
+ window_text = " ".join(words[start:end])
530
+ window_tokens = _specific_tokens(set(_tokenise(window_text)))
531
+ overlap = anchor_tokens & window_tokens
532
+ if not overlap:
533
+ continue
534
+
535
+ overlap_ratio = len(overlap) / len(anchor_tokens)
536
+ density = len(overlap) / max(1, len(window_tokens))
537
+ score = (float(len(overlap)), overlap_ratio, density)
538
+ if score > best_score:
539
+ best_score = score
540
+ best_range = (start, end)
541
+
542
+ start, end = best_range
543
+ excerpt = " ".join(words[start:end]).strip()
544
+ if len(excerpt) > max_chars:
545
+ excerpt = excerpt[:max_chars].rsplit(" ", 1)[0].rstrip()
546
+ if start > 0:
547
+ excerpt = "..." + excerpt
548
+ if end < len(words):
549
+ excerpt = excerpt.rstrip() + "..."
550
+ return excerpt
551
+
552
+
553
+ # ---------------------------------------------------------------------------
554
+ # Embedding retrieval (default backend)
555
+ # ---------------------------------------------------------------------------
556
+
557
+ # Default embedding model. BGE-base improves semantic recall over BGE-small
558
+ # while keeping the model download at runtime rather than bundling it.
559
+ DEFAULT_EMBEDDING_MODEL = "BAAI/bge-base-en-v1.5"
560
+ DEFAULT_EMBEDDING_DIMS = 768
561
+
562
+ # Path to the pre-computed corpus embeddings bundled with the package.
563
+ _BUNDLED_EMBEDDINGS_PATH = (
564
+ Path(__file__).parent / ".." / "data" / "chunk_embeddings.npy"
565
+ )
566
+
567
+ # Path to the sidecar manifest that records the chunk-order fingerprint the
568
+ # .npy was built against. Used to detect silent chunk/embedding mismatches
569
+ # that can occur when the .npy is built on one platform and loaded on
570
+ # another — see ``compute_chunk_fingerprint`` below for rationale.
571
+ _BUNDLED_MANIFEST_PATH = (
572
+ Path(__file__).parent / ".." / "data" / "chunk_embeddings.manifest.json"
573
+ )
574
+
575
+
576
+ def compute_chunk_fingerprint(chunks: "list[TextChunk]") -> str:
577
+ """Return a deterministic SHA-256 fingerprint of a chunk list's order.
578
+
579
+ The fingerprint is computed over ``paper_key:chunk_index`` pairs
580
+ (joined with ``|``) and is independent of chunk text content. Two
581
+ chunk lists produce the same fingerprint iff they reference the
582
+ same papers in the same order with the same per-paper chunking.
583
+
584
+ This is used by :class:`EmbeddingRetriever` to verify that the
585
+ runtime chunks are in the same order as the chunks that were used
586
+ to build ``chunk_embeddings.npy``. A mismatch means the pre-
587
+ computed embeddings at index ``k`` correspond to a different chunk
588
+ than the runtime chunk at the same index — every retrieval result
589
+ would silently return the wrong passage.
590
+
591
+ Parameters
592
+ ----------
593
+ chunks:
594
+ Ordered list of :class:`TextChunk`. Normally obtained from
595
+ :meth:`RAGEngine.load`.
596
+
597
+ Returns
598
+ -------
599
+ Hex-digest SHA-256 string.
600
+ """
601
+ joined = b"|".join(
602
+ f"{c.paper_key}:{c.chunk_index}".encode("utf-8") for c in chunks
603
+ )
604
+ return hashlib.sha256(joined).hexdigest()
605
+
606
+
607
+ def _load_precomputed_embeddings() -> "np.ndarray | None":
608
+ """Load the pre-computed corpus embeddings shipped with the package.
609
+
610
+ Returns ``None`` when the file is missing (e.g., in development
611
+ before the build step has been run) so callers can fall back to
612
+ regenerating or using a different backend.
613
+ """
614
+ try:
615
+ import numpy as np # type: ignore[import-not-found]
616
+ except ImportError:
617
+ return None
618
+
619
+ if not _BUNDLED_EMBEDDINGS_PATH.exists():
620
+ return None
621
+ try:
622
+ return np.load(_BUNDLED_EMBEDDINGS_PATH)
623
+ except Exception as exc:
624
+ print(f"[RAG] Failed to load pre-computed embeddings: {exc}")
625
+ return None
626
+
627
+
628
+ def _load_chunk_manifest() -> "dict | None":
629
+ """Load the sidecar chunk-order manifest, if it exists.
630
+
631
+ Returns ``None`` when the file is missing (legacy installs that
632
+ shipped before the manifest was introduced — the loader warns but
633
+ proceeds) or when the file is malformed.
634
+ """
635
+ if not _BUNDLED_MANIFEST_PATH.exists():
636
+ return None
637
+ try:
638
+ with _BUNDLED_MANIFEST_PATH.open("r", encoding="utf-8") as fh:
639
+ return json.load(fh)
640
+ except (OSError, ValueError) as exc:
641
+ logger.warning(
642
+ "Failed to read chunk-embeddings manifest at %s: %s",
643
+ _BUNDLED_MANIFEST_PATH.resolve(), exc,
644
+ )
645
+ return None
646
+
647
+
648
+ class EmbeddingRetriever:
649
+ """Semantic retrieval over pre-computed corpus embeddings.
650
+
651
+ Unlike :class:`TfIdfIndex`, this retriever uses dense vector
652
+ embeddings from a pre-trained transformer model (via ``fastembed``
653
+ + ONNX Runtime). Queries and chunks are compared via cosine
654
+ similarity on 768-dimensional vectors, which captures semantic
655
+ similarity — synonyms, paraphrases, and related concepts that
656
+ TF-IDF misses.
657
+
658
+ The corpus embeddings are **pre-computed and shipped with the
659
+ package** as a ``.npy`` file (see ``scripts/build_embeddings.py``).
660
+ At runtime the retriever only needs to embed the query, which
661
+ takes ~40 ms on CPU after a one-time ~2 s model load.
662
+
663
+ Parameters
664
+ ----------
665
+ chunks:
666
+ List of :class:`TextChunk` objects (must be in the SAME order
667
+ used when the pre-computed embeddings were generated).
668
+ model_name:
669
+ Name of the embedding model. Must match the model used to
670
+ produce the pre-computed corpus embeddings. Defaults to
671
+ ``"BAAI/bge-base-en-v1.5"``.
672
+ precomputed_embeddings:
673
+ Optional pre-computed ``(n_chunks, dims)`` numpy array. When
674
+ ``None``, the retriever tries to load the bundled file. When
675
+ the shape doesn't match ``len(chunks)``, the embeddings are
676
+ discarded and a warning is printed (falling back to re-encoding
677
+ is caller's responsibility).
678
+
679
+ Raises
680
+ ------
681
+ ImportError
682
+ If ``fastembed`` is not installed.
683
+ RuntimeError
684
+ If no pre-computed embeddings are available AND
685
+ ``allow_rebuild`` is False (the default).
686
+ """
687
+
688
+ def __init__(
689
+ self,
690
+ chunks: "list[TextChunk]",
691
+ *,
692
+ model_name: str = DEFAULT_EMBEDDING_MODEL,
693
+ precomputed_embeddings: "np.ndarray | None" = None,
694
+ allow_rebuild: bool = False,
695
+ verbose: bool = False,
696
+ ) -> None:
697
+ try:
698
+ import numpy as np # type: ignore[import-not-found]
699
+ except ImportError as exc: # pragma: no cover
700
+ raise ImportError(
701
+ "numpy is required for EmbeddingRetriever. "
702
+ "Install with `pip install numpy`."
703
+ ) from exc
704
+
705
+ try:
706
+ from fastembed import TextEmbedding # type: ignore[import-not-found]
707
+ except ImportError as exc:
708
+ raise ImportError(
709
+ "fastembed is required for EmbeddingRetriever. "
710
+ "Install with `pip install fastembed`, or use the "
711
+ "TF-IDF backend by passing rag_backend='tfidf' to "
712
+ "MetacouplingAssistant."
713
+ ) from exc
714
+
715
+ self._chunks = chunks
716
+ self._n_docs = len(chunks)
717
+ self._model_name = model_name
718
+ self._verbose = verbose
719
+ self._np = np
720
+ self._TextEmbedding = TextEmbedding
721
+
722
+ # Lazily initialised embedder (first query triggers download +
723
+ # model load). Keeping this lazy avoids a ~2 s startup cost
724
+ # when the retriever is constructed but never used.
725
+ self._embedder: object | None = None
726
+
727
+ # Load or validate the corpus embeddings.
728
+ embeddings = precomputed_embeddings
729
+ if embeddings is None:
730
+ embeddings = _load_precomputed_embeddings()
731
+
732
+ if embeddings is None:
733
+ if not allow_rebuild:
734
+ raise RuntimeError(
735
+ "No pre-computed embeddings found at "
736
+ f"{_BUNDLED_EMBEDDINGS_PATH.resolve()}. Run "
737
+ "`python scripts/build_embeddings.py` to generate "
738
+ "the file, or pass allow_rebuild=True to build it "
739
+ "at runtime (~60 s on CPU)."
740
+ )
741
+ embeddings = self._build_corpus_embeddings()
742
+
743
+ if embeddings.shape[0] != self._n_docs:
744
+ raise RuntimeError(
745
+ f"Pre-computed embedding count ({embeddings.shape[0]}) "
746
+ f"does not match chunk count ({self._n_docs}). The "
747
+ "embeddings file is stale — re-run "
748
+ "`python scripts/build_embeddings.py`."
749
+ )
750
+
751
+ # Verify chunk-order fingerprint against the bundled manifest.
752
+ # The shape check above only guarantees counts match. We ALSO
753
+ # need to verify the chunks are in the SAME ORDER they were in
754
+ # when the .npy was built — otherwise embedding [k] points to
755
+ # the wrong chunk. This matters because Python's ``Path.__lt__``
756
+ # sorts case-INsensitively on Windows and case-sensitively on
757
+ # Linux/macOS, so building the .npy on one OS and loading it on
758
+ # another can silently corrupt the mapping whenever the corpus
759
+ # has mixed-case filenames.
760
+ #
761
+ # Only run the manifest check when the BUNDLED embeddings were
762
+ # loaded (precomputed_embeddings was None on entry). When a
763
+ # caller passes their own embeddings, the bundled manifest
764
+ # doesn't apply — it's the caller's responsibility to ensure
765
+ # the order matches their custom embeddings.
766
+ using_bundled = precomputed_embeddings is None
767
+ manifest = _load_chunk_manifest() if using_bundled else None
768
+ if manifest is not None:
769
+ stored_hash = manifest.get("paper_key_order_sha256", "")
770
+ runtime_hash = compute_chunk_fingerprint(chunks)
771
+ if stored_hash and stored_hash != runtime_hash:
772
+ raise RuntimeError(
773
+ "Chunk-order fingerprint mismatch between the "
774
+ "pre-computed embeddings and the chunks loaded at "
775
+ "runtime. The chunks in chunk_embeddings.npy were "
776
+ "built in a different order than the one produced "
777
+ "by RAGEngine.load() on this machine — every "
778
+ "retrieval result would silently return the wrong "
779
+ "passage. This usually happens when the package "
780
+ "was built on one OS and installed on another, or "
781
+ "when Papers.zip was updated without rebuilding "
782
+ "the embeddings.\n\n"
783
+ "Fix: run `python scripts/build_embeddings.py "
784
+ "--force` on this machine to regenerate the .npy "
785
+ "and manifest, or install a newer package version.\n\n"
786
+ f" expected (manifest): {stored_hash[:16]}...\n"
787
+ f" actual (runtime): {runtime_hash[:16]}..."
788
+ )
789
+ elif self._verbose:
790
+ # Legacy install without a manifest — warn but proceed.
791
+ logger.warning(
792
+ "No chunk-embeddings manifest found at %s. Cannot "
793
+ "verify chunk-order integrity; retrieval may silently "
794
+ "return wrong passages if the .npy was built on a "
795
+ "different platform. Consider re-running "
796
+ "`python scripts/build_embeddings.py --force`.",
797
+ _BUNDLED_MANIFEST_PATH.resolve(),
798
+ )
799
+
800
+ # L2-normalize for cosine similarity via dot product. Bundled
801
+ # embeddings may be stored as float16 to keep the wheel small;
802
+ # scoring is always done in float32.
803
+ embeddings = np.asarray(embeddings, dtype=np.float32)
804
+ norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
805
+ norms = np.where(norms == 0, 1.0, norms)
806
+ self._chunk_vecs = (embeddings / norms).astype(np.float32)
807
+
808
+ # ------------------------------------------------------------------
809
+ # Internal helpers
810
+ # ------------------------------------------------------------------
811
+
812
+ def _ensure_embedder(self) -> object:
813
+ """Lazily initialise the fastembed encoder."""
814
+ if self._embedder is None:
815
+ if self._verbose:
816
+ print(
817
+ f"[RAG] Loading embedding model "
818
+ f"{self._model_name} (first use may take ~10 s "
819
+ "while the ONNX model is downloaded/cached)..."
820
+ )
821
+ self._embedder = self._TextEmbedding(
822
+ model_name=self._model_name,
823
+ )
824
+ return self._embedder
825
+
826
+ def _build_corpus_embeddings(self) -> "np.ndarray":
827
+ """Encode the chunks with the active model (runtime fallback).
828
+
829
+ Only called when the pre-computed file is missing AND the user
830
+ opts in via ``allow_rebuild=True``. Prefer pre-computing once
831
+ and shipping the .npy file with the package.
832
+ """
833
+ embedder = self._ensure_embedder()
834
+ texts = [chunk.text for chunk in self._chunks]
835
+ if self._verbose:
836
+ print(
837
+ f"[RAG] Encoding {len(texts)} chunks with "
838
+ f"{self._model_name} (this takes ~60 s on CPU)..."
839
+ )
840
+ vectors = self._np.array(
841
+ list(embedder.embed(texts)),
842
+ dtype=self._np.float32,
843
+ )
844
+ return vectors
845
+
846
+ # ------------------------------------------------------------------
847
+ # Public API
848
+ # ------------------------------------------------------------------
849
+
850
+ def query(
851
+ self,
852
+ query_text: str,
853
+ top_k: int = 5,
854
+ min_score: float = 0.3,
855
+ max_chunks_per_paper: int = 3,
856
+ ) -> list[RetrievalResult]:
857
+ """Retrieve the top-k most relevant chunks for a query.
858
+
859
+ Parameters
860
+ ----------
861
+ query_text:
862
+ The search query text.
863
+ top_k:
864
+ Maximum number of results (chunks) to return. Note: this
865
+ counts individual chunks, not unique papers — multiple
866
+ chunks from the same paper may appear when that paper's
867
+ most relevant content is spread across different sections.
868
+ min_score:
869
+ Minimum cosine similarity threshold. BGE-small cosine
870
+ similarity typically ranges 0.3–0.9 for relevant results,
871
+ so this default is stricter than the TF-IDF default.
872
+ max_chunks_per_paper:
873
+ Maximum number of chunks from the same paper that can
874
+ appear in the result set. Default ``3`` lets strongly-
875
+ matching papers contribute several high-scoring chunks
876
+ rather than a single chunk that may not cover all of the
877
+ paper's relevant content. Set to ``1`` for the legacy
878
+ one-chunk-per-paper behavior, or raise to ``5+`` when a
879
+ single paper is expected to be highly relevant and its
880
+ evidence is spread across many sections.
881
+
882
+ Returns
883
+ -------
884
+ List of :class:`RetrievalResult` sorted by score descending,
885
+ with at most ``max_chunks_per_paper`` chunks from any single
886
+ paper.
887
+ """
888
+ if not self._chunks:
889
+ return []
890
+ if not query_text or not query_text.strip():
891
+ return []
892
+ if max_chunks_per_paper < 1:
893
+ max_chunks_per_paper = 1
894
+
895
+ embedder = self._ensure_embedder()
896
+ query_vec = next(iter(embedder.embed([query_text])))
897
+ query_vec = self._np.asarray(query_vec, dtype=self._np.float32)
898
+ q_norm = float(self._np.linalg.norm(query_vec))
899
+ if q_norm == 0.0:
900
+ return []
901
+ query_vec = query_vec / q_norm
902
+
903
+ # Cosine similarity = dot product of L2-normalized vectors.
904
+ scores = self._chunk_vecs @ query_vec # shape (n_chunks,)
905
+
906
+ # Two-pass selection:
907
+ # Pass 1 prefers chunks from sections we have NOT yet seen
908
+ # for a given paper — this buys section diversity first
909
+ # (Introduction + Results + Discussion when all score highly).
910
+ # Chunks from sections we've already seen get deferred.
911
+ # Pass 2 fills any remaining slots (per-paper cap and
912
+ # top_k) from the deferred list — so long sections that
913
+ # the chunker split into several sub-topic chunks (e.g.,
914
+ # "4. Results" with inbound vs outbound sub-parts) can
915
+ # still contribute multiple chunks when the paper has
916
+ # budget to spare.
917
+ order = self._np.argsort(-scores)
918
+ deduped: list[RetrievalResult] = []
919
+ paper_counts: dict[str, int] = {}
920
+ paper_sections_seen: dict[str, set[str]] = {}
921
+ deferred: list[tuple[int, float]] = []
922
+ for idx in order:
923
+ score = float(scores[idx])
924
+ if score < min_score:
925
+ break
926
+ chunk = self._chunks[int(idx)]
927
+ paper_key = chunk.paper_key
928
+ if paper_counts.get(paper_key, 0) >= max_chunks_per_paper:
929
+ continue
930
+ sections = paper_sections_seen.setdefault(paper_key, set())
931
+ if chunk.section in sections:
932
+ deferred.append((int(idx), score))
933
+ continue
934
+ sections.add(chunk.section)
935
+ paper_counts[paper_key] = paper_counts.get(paper_key, 0) + 1
936
+ deduped.append(RetrievalResult(chunk=chunk, score=score))
937
+ if len(deduped) >= top_k:
938
+ return deduped
939
+ # Pass 2 — use remaining per-paper budget on deferred chunks,
940
+ # in score-desc order (``deferred`` is already score-desc
941
+ # because ``order`` was).
942
+ for idx, score in deferred:
943
+ chunk = self._chunks[int(idx)]
944
+ paper_key = chunk.paper_key
945
+ if paper_counts.get(paper_key, 0) >= max_chunks_per_paper:
946
+ continue
947
+ paper_counts[paper_key] = paper_counts.get(paper_key, 0) + 1
948
+ deduped.append(RetrievalResult(chunk=chunk, score=score))
949
+ if len(deduped) >= top_k:
950
+ break
951
+ return deduped
952
+
953
+
954
+ class TfIdfIndex:
955
+ """In-memory TF-IDF index for text chunks.
956
+
957
+ Parameters
958
+ ----------
959
+ chunks:
960
+ List of :class:`TextChunk` objects to index.
961
+ """
962
+
963
+ def __init__(self, chunks: list[TextChunk]) -> None:
964
+ self._chunks = chunks
965
+ self._n_docs = len(chunks)
966
+ self._vocab: dict[str, int] = {} # word → doc-frequency
967
+ self._tf_vectors: list[dict[str, float]] = []
968
+
969
+ self._build_index()
970
+
971
+ def _build_index(self) -> None:
972
+ """Compute TF and DF for all chunks."""
973
+ doc_freq: Counter[str] = Counter()
974
+ tf_raw: list[Counter[str]] = []
975
+
976
+ for chunk in self._chunks:
977
+ tokens = _tokenise(chunk.text)
978
+ tf = Counter(tokens)
979
+ tf_raw.append(tf)
980
+ # Document frequency: count each term once per document
981
+ doc_freq.update(tf.keys())
982
+
983
+ self._vocab = dict(doc_freq)
984
+
985
+ # Normalise TF to TF log-normalised: 1 + log(tf)
986
+ for tf in tf_raw:
987
+ total = sum(tf.values()) or 1
988
+ normed: dict[str, float] = {}
989
+ for word, count in tf.items():
990
+ normed[word] = (1 + math.log(count)) / total
991
+ self._tf_vectors.append(normed)
992
+
993
+ def query(
994
+ self,
995
+ query_text: str,
996
+ top_k: int = 5,
997
+ min_score: float = 0.01,
998
+ max_chunks_per_paper: int = 3,
999
+ ) -> list[RetrievalResult]:
1000
+ """Retrieve the top-k most relevant chunks for a query.
1001
+
1002
+ Parameters
1003
+ ----------
1004
+ query_text:
1005
+ The search query text.
1006
+ top_k:
1007
+ Maximum number of results (chunks) to return. Note: this
1008
+ counts individual chunks, not unique papers.
1009
+ min_score:
1010
+ Minimum cosine similarity threshold.
1011
+ max_chunks_per_paper:
1012
+ Maximum number of chunks from the same paper that can
1013
+ appear in the result set. Default ``3``. Set to ``1``
1014
+ for the legacy one-chunk-per-paper behavior.
1015
+
1016
+ Returns
1017
+ -------
1018
+ List of :class:`RetrievalResult` sorted by score descending,
1019
+ with at most ``max_chunks_per_paper`` chunks per paper.
1020
+ """
1021
+ if not self._chunks:
1022
+ return []
1023
+ if max_chunks_per_paper < 1:
1024
+ max_chunks_per_paper = 1
1025
+
1026
+ query_tokens = _tokenise(query_text)
1027
+ if not query_tokens:
1028
+ return []
1029
+
1030
+ # Build query TF-IDF vector
1031
+ query_tf = Counter(query_tokens)
1032
+ query_vec: dict[str, float] = {}
1033
+ for word, count in query_tf.items():
1034
+ df = self._vocab.get(word, 0)
1035
+ if df == 0:
1036
+ continue
1037
+ idf = math.log((self._n_docs + 1) / (df + 1)) + 1
1038
+ query_vec[word] = (1 + math.log(count)) * idf
1039
+
1040
+ if not query_vec:
1041
+ return []
1042
+
1043
+ # Compute query vector magnitude
1044
+ q_mag = math.sqrt(sum(v * v for v in query_vec.values()))
1045
+ if q_mag == 0:
1046
+ return []
1047
+
1048
+ # Score each document
1049
+ results: list[tuple[float, int]] = []
1050
+ for idx, doc_tf in enumerate(self._tf_vectors):
1051
+ dot = 0.0
1052
+ for word, q_val in query_vec.items():
1053
+ d_val = doc_tf.get(word, 0.0)
1054
+ if d_val > 0:
1055
+ df = self._vocab.get(word, 0)
1056
+ idf = math.log((self._n_docs + 1) / (df + 1)) + 1
1057
+ dot += q_val * d_val * idf
1058
+
1059
+ # Document magnitude (approximation — use cached if needed)
1060
+ d_mag = math.sqrt(
1061
+ sum(
1062
+ (v * (math.log((self._n_docs + 1) / (self._vocab.get(w, 1) + 1)) + 1)) ** 2
1063
+ for w, v in doc_tf.items()
1064
+ )
1065
+ )
1066
+ if d_mag == 0:
1067
+ continue
1068
+
1069
+ cosine = dot / (q_mag * d_mag)
1070
+ if cosine >= min_score:
1071
+ results.append((cosine, idx))
1072
+
1073
+ # Sort by score descending
1074
+ results.sort(key=lambda x: x[0], reverse=True)
1075
+
1076
+ # Two-pass selection (see EmbeddingRetriever.query() for the
1077
+ # rationale): Pass 1 prefers chunks from distinct sections per
1078
+ # paper; Pass 2 fills remaining per-paper budget with deferred
1079
+ # chunks from sections we already took from, so long sections
1080
+ # split into several sub-topic chunks can still contribute
1081
+ # multiple chunks when budget allows.
1082
+ paper_counts: dict[str, int] = {}
1083
+ paper_sections_seen: dict[str, set[str]] = {}
1084
+ deferred: list[tuple[float, int]] = []
1085
+ deduped: list[RetrievalResult] = []
1086
+ for score, idx in results:
1087
+ chunk = self._chunks[idx]
1088
+ paper_key = chunk.paper_key
1089
+ if paper_counts.get(paper_key, 0) >= max_chunks_per_paper:
1090
+ continue
1091
+ sections = paper_sections_seen.setdefault(paper_key, set())
1092
+ if chunk.section in sections:
1093
+ deferred.append((score, idx))
1094
+ continue
1095
+ sections.add(chunk.section)
1096
+ paper_counts[paper_key] = paper_counts.get(paper_key, 0) + 1
1097
+ deduped.append(RetrievalResult(chunk=chunk, score=score))
1098
+ if len(deduped) >= top_k:
1099
+ return deduped
1100
+ for score, idx in deferred:
1101
+ chunk = self._chunks[idx]
1102
+ paper_key = chunk.paper_key
1103
+ if paper_counts.get(paper_key, 0) >= max_chunks_per_paper:
1104
+ continue
1105
+ paper_counts[paper_key] = paper_counts.get(paper_key, 0) + 1
1106
+ deduped.append(RetrievalResult(chunk=chunk, score=score))
1107
+ if len(deduped) >= top_k:
1108
+ break
1109
+
1110
+ return deduped
1111
+
1112
+
1113
+ # ---------------------------------------------------------------------------
1114
+ # RAG Engine — high-level API
1115
+ # ---------------------------------------------------------------------------
1116
+
1117
+
1118
+ class RAGEngine:
1119
+ """Retrieval-Augmented Generation engine for metacoupling papers.
1120
+
1121
+ Loads full-text papers from a directory, matches them against the
1122
+ BibTeX literature database, chunks them, and builds a retrieval
1123
+ index.
1124
+
1125
+ Two retrieval backends are available:
1126
+
1127
+ - ``"embeddings"`` (default): semantic retrieval using fastembed +
1128
+ pre-computed corpus embeddings bundled with the package. Catches
1129
+ synonyms, paraphrases, and related concepts.
1130
+ - ``"tfidf"``: lexical retrieval using TF-IDF + cosine similarity.
1131
+ Fallback when fastembed is unavailable.
1132
+ - ``"auto"``: try embeddings first, fall back to TF-IDF on failure.
1133
+
1134
+ Parameters
1135
+ ----------
1136
+ papers_dir:
1137
+ Path to directory containing full-text markdown papers.
1138
+ max_chunk_words:
1139
+ Maximum words per text chunk.
1140
+ overlap_words:
1141
+ Number of overlapping words between consecutive chunks.
1142
+ verbose:
1143
+ Print progress information during loading.
1144
+ backend:
1145
+ Retrieval backend: ``"embeddings"`` (default), ``"tfidf"``,
1146
+ or ``"auto"``. ``"auto"`` tries embeddings first and
1147
+ transparently falls back to TF-IDF if fastembed or the
1148
+ pre-computed embedding file are missing.
1149
+ """
1150
+
1151
+ def __init__(
1152
+ self,
1153
+ papers_dir: str | Path,
1154
+ max_chunk_words: int = 250,
1155
+ overlap_words: int = 50,
1156
+ verbose: bool = False,
1157
+ backend: str = "auto",
1158
+ ) -> None:
1159
+ self._papers_dir = Path(papers_dir)
1160
+ self._max_chunk_words = max_chunk_words
1161
+ self._overlap_words = overlap_words
1162
+ self._verbose = verbose
1163
+ if backend not in ("embeddings", "tfidf", "auto"):
1164
+ raise ValueError(
1165
+ f"Unknown backend: {backend!r}. "
1166
+ "Expected 'embeddings', 'tfidf', or 'auto'."
1167
+ )
1168
+ self._backend = backend
1169
+ self._active_backend: str | None = None
1170
+ self._index: TfIdfIndex | EmbeddingRetriever | None = None
1171
+ self._matched_count: int = 0
1172
+ self._total_chunks: int = 0
1173
+ self._total_files: int = 0
1174
+
1175
+ @property
1176
+ def is_loaded(self) -> bool:
1177
+ """Return True if the index has been built."""
1178
+ return self._index is not None
1179
+
1180
+ @property
1181
+ def backend(self) -> str | None:
1182
+ """Return the name of the active retrieval backend.
1183
+
1184
+ One of ``"embeddings"``, ``"tfidf"``, or ``None`` if
1185
+ :meth:`load` has not been called yet.
1186
+ """
1187
+ return self._active_backend
1188
+
1189
+ @property
1190
+ def matched_papers(self) -> int:
1191
+ """Number of papers matched to the literature database."""
1192
+ return self._matched_count
1193
+
1194
+ @property
1195
+ def total_chunks(self) -> int:
1196
+ """Total number of text chunks in the index."""
1197
+ return self._total_chunks
1198
+
1199
+ @property
1200
+ def total_files(self) -> int:
1201
+ """Total number of markdown files found."""
1202
+ return self._total_files
1203
+
1204
+ def load(self) -> None:
1205
+ """Load papers, match to database, chunk, and build index.
1206
+
1207
+ This should be called once before querying. It may take a few
1208
+ seconds depending on the number of papers.
1209
+
1210
+ If the configured ``papers_dir`` does not exist, the engine
1211
+ automatically falls back to the bundled ``data/Papers.zip``
1212
+ shipped with the package.
1213
+ """
1214
+ from metacouplingllm.knowledge.literature import _get_database
1215
+
1216
+ # Validate papers directory exists; fall back to bundled zip
1217
+ if not self._papers_dir.exists() or not self._papers_dir.is_dir():
1218
+ self._papers_dir = _extract_bundled_papers(self._verbose)
1219
+
1220
+ if not self._papers_dir.exists() or not self._papers_dir.is_dir():
1221
+ msg = (
1222
+ f"[RAG] Papers directory does not exist: "
1223
+ f"{self._papers_dir.resolve()}"
1224
+ )
1225
+ print(msg)
1226
+ self._index = TfIdfIndex([])
1227
+ return
1228
+
1229
+ db_papers = _get_database()
1230
+ if not db_papers:
1231
+ print("[RAG] WARNING: No literature database loaded. "
1232
+ "RAG will not produce evidence.")
1233
+ self._index = TfIdfIndex([])
1234
+ return
1235
+
1236
+ # Collect markdown files.
1237
+ #
1238
+ # IMPORTANT: sort by the filename STRING, not by the Path object.
1239
+ # Python's ``Path.__lt__`` is platform-dependent: on Windows it
1240
+ # uses case-INsensitive comparison (via ``_str_normcase``), but
1241
+ # on Linux/macOS it uses case-sensitive Unicode codepoint order.
1242
+ # Sorting ``Path`` objects directly would produce a different
1243
+ # chunk order on Windows vs Linux for any corpus that contains
1244
+ # filenames with mixed leading case (e.g., "da Silva..." vs
1245
+ # "Dandan..."). That in turn corrupts the chunks-to-embeddings
1246
+ # mapping for the pre-computed ``chunk_embeddings.npy`` file
1247
+ # whenever it's built on one OS and loaded on another. Using
1248
+ # ``key=lambda p: p.name`` forces pure-string (case-sensitive)
1249
+ # comparison on every platform, so the chunk order is
1250
+ # deterministic across OSes. The manifest integrity check in
1251
+ # ``EmbeddingRetriever.__init__`` catches mismatches if this
1252
+ # ever drifts again.
1253
+ md_files = sorted(
1254
+ self._papers_dir.glob("*.md"), key=lambda p: p.name,
1255
+ )
1256
+ self._total_files = len(md_files)
1257
+
1258
+ if self._total_files == 0:
1259
+ print(
1260
+ f"[RAG] WARNING: No .md files found in "
1261
+ f"{self._papers_dir.resolve()}"
1262
+ )
1263
+ self._index = TfIdfIndex([])
1264
+ return
1265
+
1266
+ if self._verbose:
1267
+ print(f"[RAG] Found {self._total_files} markdown files.")
1268
+
1269
+ all_chunks: list[TextChunk] = []
1270
+ matched = 0
1271
+ fallback_metadata = 0
1272
+
1273
+ for md_path in md_files:
1274
+ # Match to database
1275
+ paper = _match_paper_to_db(md_path.name, db_papers)
1276
+ if paper is None:
1277
+ paper = _paper_from_filename(md_path.name)
1278
+ if paper is None:
1279
+ continue
1280
+ fallback_metadata += 1
1281
+ else:
1282
+ matched += 1
1283
+
1284
+ # Read and chunk the file
1285
+ try:
1286
+ text = md_path.read_text(encoding="utf-8", errors="replace")
1287
+ except Exception:
1288
+ continue
1289
+
1290
+ chunks = _chunk_markdown(
1291
+ text, paper,
1292
+ max_chunk_words=self._max_chunk_words,
1293
+ overlap_words=self._overlap_words,
1294
+ )
1295
+ all_chunks.extend(chunks)
1296
+
1297
+ self._matched_count = matched
1298
+ self._total_chunks = len(all_chunks)
1299
+
1300
+ indexed_papers = matched + fallback_metadata
1301
+
1302
+ # Always print summary so user knows RAG loaded. Report indexed
1303
+ # papers first; BibTeX matching is metadata quality, not corpus
1304
+ # coverage.
1305
+ print(
1306
+ f"[RAG] Indexed {indexed_papers}/{self._total_files} papers "
1307
+ f"-> {len(all_chunks)} chunks."
1308
+ )
1309
+ if matched == 0 and fallback_metadata == 0:
1310
+ print(
1311
+ "[RAG] WARNING: No papers matched. Ensure filenames follow "
1312
+ "Zotero format: 'Author - YYYY - Title.md'"
1313
+ )
1314
+
1315
+ self._index = self._build_retriever(all_chunks)
1316
+
1317
+ def _build_retriever(
1318
+ self, chunks: list[TextChunk],
1319
+ ) -> "TfIdfIndex | EmbeddingRetriever":
1320
+ """Build the configured retriever for the given chunks.
1321
+
1322
+ Handles the embeddings → TF-IDF fallback logic when
1323
+ ``backend="auto"``. Prints the selected backend so users
1324
+ know which retriever is active.
1325
+ """
1326
+ if self._backend == "tfidf":
1327
+ self._active_backend = "tfidf"
1328
+ if self._verbose:
1329
+ print("[RAG] Using TF-IDF retrieval backend.")
1330
+ return TfIdfIndex(chunks)
1331
+
1332
+ # "embeddings" or "auto"
1333
+ try:
1334
+ retriever = EmbeddingRetriever(
1335
+ chunks,
1336
+ verbose=self._verbose,
1337
+ )
1338
+ self._active_backend = "embeddings"
1339
+ print(
1340
+ f"[RAG] Using embedding retrieval backend "
1341
+ f"({DEFAULT_EMBEDDING_MODEL})."
1342
+ )
1343
+ return retriever
1344
+ except (ImportError, RuntimeError) as exc:
1345
+ if self._backend == "embeddings":
1346
+ # User explicitly asked for embeddings — surface the
1347
+ # error loudly but still fall back to keep the
1348
+ # package usable.
1349
+ print(
1350
+ f"[RAG] WARNING: embedding backend unavailable: "
1351
+ f"{exc}. Falling back to TF-IDF."
1352
+ )
1353
+ elif self._verbose:
1354
+ print(
1355
+ f"[RAG] Embedding backend unavailable ({exc}). "
1356
+ "Using TF-IDF fallback."
1357
+ )
1358
+ self._active_backend = "tfidf"
1359
+ return TfIdfIndex(chunks)
1360
+
1361
+ def retrieve(
1362
+ self,
1363
+ query: str,
1364
+ top_k: int = 5,
1365
+ min_score: float | None = None,
1366
+ max_chunks_per_paper: int = 3,
1367
+ ) -> list[RetrievalResult]:
1368
+ """Retrieve relevant passages for a query string.
1369
+
1370
+ Parameters
1371
+ ----------
1372
+ query:
1373
+ Free-text query (e.g., research description or analysis text).
1374
+ top_k:
1375
+ Maximum number of passages (chunks) to return. Multiple
1376
+ chunks may come from the same paper when that paper's
1377
+ most relevant content is spread across different sections.
1378
+ min_score:
1379
+ Minimum similarity threshold. When ``None`` (default), a
1380
+ backend-appropriate value is used: 0.01 for TF-IDF,
1381
+ 0.3 for embedding cosine similarity.
1382
+ max_chunks_per_paper:
1383
+ Cap on how many chunks from the same paper may appear in
1384
+ the result set (default ``3``). Set to ``1`` for the
1385
+ legacy one-chunk-per-paper behavior. Useful when a single
1386
+ paper is highly relevant but its key evidence is scattered
1387
+ across Introduction / Results / Discussion sections that
1388
+ a single chunk can't cover on its own.
1389
+
1390
+ Returns
1391
+ -------
1392
+ List of :class:`RetrievalResult` sorted by relevance.
1393
+
1394
+ Raises
1395
+ ------
1396
+ RuntimeError
1397
+ If :meth:`load` has not been called.
1398
+ """
1399
+ if self._index is None:
1400
+ raise RuntimeError("RAG engine not loaded. Call load() first.")
1401
+ if min_score is None:
1402
+ min_score = (
1403
+ 0.3 if self._active_backend == "embeddings" else 0.01
1404
+ )
1405
+ return self._index.query(
1406
+ query,
1407
+ top_k=top_k,
1408
+ min_score=min_score,
1409
+ max_chunks_per_paper=max_chunks_per_paper,
1410
+ )
1411
+
1412
+ def retrieve_for_analysis(
1413
+ self,
1414
+ parsed: object, # ParsedAnalysis, but avoid circular import
1415
+ top_k: int = 5,
1416
+ min_score: float | None = None,
1417
+ max_chunks_per_paper: int = 3,
1418
+ ) -> list[RetrievalResult]:
1419
+ """Retrieve passages relevant to a parsed LLM analysis.
1420
+
1421
+ Extracts query terms from the parsed analysis and retrieves
1422
+ matching passages from the full-text paper index.
1423
+
1424
+ Parameters
1425
+ ----------
1426
+ parsed:
1427
+ A :class:`~metacouplingllm.llm.parser.ParsedAnalysis` object.
1428
+ top_k:
1429
+ Maximum number of results.
1430
+ min_score:
1431
+ Minimum similarity score.
1432
+ max_chunks_per_paper:
1433
+ See :meth:`retrieve`. Default ``3``.
1434
+
1435
+ Returns
1436
+ -------
1437
+ List of :class:`RetrievalResult`.
1438
+ """
1439
+ query = _build_query_from_analysis(parsed)
1440
+ if not query:
1441
+ return []
1442
+ return self.retrieve(
1443
+ query,
1444
+ top_k=top_k,
1445
+ min_score=min_score,
1446
+ max_chunks_per_paper=max_chunks_per_paper,
1447
+ )
1448
+
1449
+
1450
+ # ---------------------------------------------------------------------------
1451
+ # Query builder from ParsedAnalysis
1452
+ # ---------------------------------------------------------------------------
1453
+
1454
+
1455
+ def _build_query_from_analysis(parsed: object) -> str:
1456
+ """Build a search query from a ParsedAnalysis object.
1457
+
1458
+ Extracts key terms from classification, systems, flows, and causes.
1459
+ """
1460
+ parts: list[str] = []
1461
+
1462
+ # Coupling classification
1463
+ cc = getattr(parsed, "coupling_classification", "")
1464
+ if cc:
1465
+ parts.append(cc)
1466
+
1467
+ # Systems
1468
+ systems = getattr(parsed, "systems", {})
1469
+ for role in ("sending", "receiving", "spillover"):
1470
+ val = systems.get(role, "")
1471
+ if isinstance(val, dict):
1472
+ for key in ("name", "geographic_scope", "description"):
1473
+ v = val.get(key, "")
1474
+ if v:
1475
+ parts.append(v)
1476
+ elif isinstance(val, str) and val:
1477
+ parts.append(val)
1478
+
1479
+ # Flows
1480
+ flows = getattr(parsed, "flows", [])
1481
+ for flow in flows:
1482
+ if isinstance(flow, dict):
1483
+ desc = flow.get("description", "")
1484
+ if desc:
1485
+ parts.append(desc)
1486
+ cat = flow.get("category", "")
1487
+ if cat:
1488
+ parts.append(cat)
1489
+ elif isinstance(flow, str) and flow:
1490
+ parts.append(flow)
1491
+
1492
+ # Causes — can be a list of strings, a dict of lists, or a dict of strings
1493
+ causes = getattr(parsed, "causes", [])
1494
+ if isinstance(causes, dict):
1495
+ for key, val in causes.items():
1496
+ if isinstance(val, list):
1497
+ for item in val:
1498
+ if isinstance(item, str) and item:
1499
+ parts.append(item)
1500
+ elif isinstance(val, str) and val:
1501
+ parts.append(val)
1502
+ elif isinstance(causes, list):
1503
+ for cat in causes:
1504
+ if isinstance(cat, str) and cat:
1505
+ parts.append(cat)
1506
+ elif isinstance(cat, dict):
1507
+ desc = cat.get("description", "")
1508
+ if desc:
1509
+ parts.append(desc)
1510
+
1511
+ # Effects — can be a list of strings, a dict of lists, or a dict of strings
1512
+ effects = getattr(parsed, "effects", [])
1513
+ if isinstance(effects, dict):
1514
+ for key, val in effects.items():
1515
+ if isinstance(val, list):
1516
+ for item in val:
1517
+ if isinstance(item, str) and item:
1518
+ parts.append(item)
1519
+ elif isinstance(val, str) and val:
1520
+ parts.append(val)
1521
+ elif isinstance(effects, list):
1522
+ for eff in effects:
1523
+ if isinstance(eff, dict):
1524
+ desc = eff.get("description", "")
1525
+ if desc:
1526
+ parts.append(desc)
1527
+ elif isinstance(eff, str) and eff:
1528
+ parts.append(eff)
1529
+
1530
+ return " ".join(parts)
1531
+
1532
+
1533
+ # ---------------------------------------------------------------------------
1534
+ # Formatting
1535
+ # ---------------------------------------------------------------------------
1536
+
1537
+
1538
+ def format_evidence(
1539
+ results: list[RetrievalResult],
1540
+ anchor_text: str = "",
1541
+ backend: str = "tfidf",
1542
+ ) -> str:
1543
+ """Format retrieved passages as a human-readable evidence block.
1544
+
1545
+ Parameters
1546
+ ----------
1547
+ results:
1548
+ List of :class:`RetrievalResult` from retrieval.
1549
+ anchor_text:
1550
+ Optional query or analysis text used to choose a more relevant
1551
+ excerpt from each evidence chunk.
1552
+ backend:
1553
+ Retrieval backend that produced the scores (``"tfidf"`` or
1554
+ ``"embeddings"``). Used to pick appropriate High/Medium/Low
1555
+ thresholds since the two backends have very different score
1556
+ ranges.
1557
+
1558
+ Returns
1559
+ -------
1560
+ A formatted string ready for display.
1561
+ """
1562
+ if not results:
1563
+ return "No supporting evidence found in the full-text paper database."
1564
+
1565
+ lines: list[str] = []
1566
+ lines.append("=" * 70)
1567
+ lines.append(" SUPPORTING EVIDENCE FROM LITERATURE")
1568
+ lines.append("=" * 70)
1569
+ lines.append("")
1570
+
1571
+ for i, r in enumerate(results, 1):
1572
+ chunk = r.chunk
1573
+ # Shorten author list
1574
+ authors = chunk.authors
1575
+ if len(authors) > 60:
1576
+ parts = authors.split(" and ")
1577
+ if len(parts) > 2:
1578
+ authors = parts[0] + " et al."
1579
+
1580
+ confidence = _score_to_confidence(r.score, backend=backend)
1581
+
1582
+ lines.append(f" [{i}] {chunk.paper_title}")
1583
+ lines.append(f" {authors} ({chunk.year})")
1584
+ lines.append(f" Section: {chunk.section}")
1585
+ lines.append(f" Confidence: {confidence} (score: {r.score:.3f})")
1586
+ lines.append(f" ---")
1587
+
1588
+ # Show a short excerpt from the most relevant part of the chunk.
1589
+ excerpt = _best_excerpt(chunk.text, anchor_text=anchor_text)
1590
+ lines.append(f" \"{excerpt}\"")
1591
+ lines.append("")
1592
+
1593
+ lines.append("=" * 70)
1594
+ return "\n".join(lines)
1595
+
1596
+
1597
+ def annotate_citations(
1598
+ formatted: str,
1599
+ results: list[RetrievalResult],
1600
+ min_keyword_overlap: int = 3,
1601
+ min_overlap_ratio: float = 0.20,
1602
+ ) -> str:
1603
+ """Add inline citation numbers ``[N]`` to formatted analysis lines.
1604
+
1605
+ For each content line in the formatted output, checks keyword overlap
1606
+ with each RAG evidence passage. If a sufficient number of distinctive
1607
+ keywords match, appends ``[N]`` citation tags to the line.
1608
+
1609
+ Parameters
1610
+ ----------
1611
+ formatted:
1612
+ The formatted analysis text (from :class:`AnalysisFormatter`).
1613
+ results:
1614
+ List of :class:`RetrievalResult` from RAG retrieval.
1615
+ min_keyword_overlap:
1616
+ Minimum number of shared keywords between a line and an evidence
1617
+ passage for a citation to be added.
1618
+ min_overlap_ratio:
1619
+ Minimum fraction of the line's keywords that must appear in the
1620
+ evidence passage.
1621
+
1622
+ Returns
1623
+ -------
1624
+ The formatted text with ``[N]`` citations appended to matching lines.
1625
+ """
1626
+ if not results:
1627
+ return formatted
1628
+
1629
+ # Build keyword sets for each evidence passage
1630
+ evidence_keywords: list[set[str]] = []
1631
+ evidence_specific_keywords: list[set[str]] = []
1632
+ for r in results:
1633
+ tokens = set(_tokenise(r.chunk.text))
1634
+ evidence_keywords.append(tokens)
1635
+ evidence_specific_keywords.append(_specific_tokens(tokens))
1636
+
1637
+ # Section headers should not be annotated
1638
+ # Note: separator-only lines (all "=" or "-") are caught separately below
1639
+ _skip_prefixes = (
1640
+ "COUPLING CLASSIFICATION", "SYSTEMS IDENTIFICATION",
1641
+ "FLOWS ANALYSIS", "AGENTS", "CAUSES", "EFFECTS",
1642
+ "RESEARCH GAPS", "PERICOUPLING DATABASE",
1643
+ "METACOUPLING FRAMEWORK", "SUPPORTING EVIDENCE",
1644
+ "RECOMMENDED LITERATURE", "Note:",
1645
+ )
1646
+ _section_headers = (
1647
+ "COUPLING CLASSIFICATION", "SYSTEMS IDENTIFICATION",
1648
+ "FLOWS ANALYSIS", "AGENTS", "CAUSES", "EFFECTS",
1649
+ "RESEARCH GAPS",
1650
+ )
1651
+
1652
+ lines = formatted.split("\n")
1653
+ annotated: list[str] = []
1654
+
1655
+ # Track which section we're in to avoid annotating non-analysis content
1656
+ in_evidence_block = False
1657
+ current_section = ""
1658
+
1659
+ for line in lines:
1660
+ stripped = line.strip()
1661
+
1662
+ matched_header = next(
1663
+ (header for header in _section_headers if stripped.startswith(header)),
1664
+ "",
1665
+ )
1666
+ if matched_header:
1667
+ current_section = matched_header
1668
+
1669
+ # Detect evidence block boundaries — never annotate evidence itself
1670
+ if "SUPPORTING EVIDENCE FROM LITERATURE" in stripped:
1671
+ in_evidence_block = True
1672
+ if in_evidence_block:
1673
+ annotated.append(line)
1674
+ continue
1675
+
1676
+ # Skip empty lines, separators, and section headers
1677
+ if (
1678
+ not stripped
1679
+ or stripped.startswith(tuple(_skip_prefixes))
1680
+ or set(stripped) <= {"=", "-", " "}
1681
+ ):
1682
+ annotated.append(line)
1683
+ continue
1684
+
1685
+ # Skip sub-headings like " [Sending]", " Ecological:", etc.
1686
+ if stripped.startswith("[") and stripped.endswith("]"):
1687
+ annotated.append(line)
1688
+ continue
1689
+ # Skip category headings like "Socioeconomic:" or " **Ecological**"
1690
+ if stripped.endswith(":") and len(stripped.split()) <= 3:
1691
+ annotated.append(line)
1692
+ continue
1693
+
1694
+ # Tokenize the line content
1695
+ line_tokens = set(_tokenise(stripped))
1696
+ if len(line_tokens) < 3:
1697
+ annotated.append(line)
1698
+ continue
1699
+ line_specific_tokens = _specific_tokens(line_tokens)
1700
+ section_min_overlap, section_min_ratio, min_specific_overlap, max_citations = (
1701
+ _citation_policy(current_section)
1702
+ )
1703
+ min_overlap = max(min_keyword_overlap, section_min_overlap)
1704
+ min_ratio = max(min_overlap_ratio, section_min_ratio)
1705
+
1706
+ # Find matching evidence passages
1707
+ candidates: list[tuple[tuple[float, float, float, float], int]] = []
1708
+ for idx, ev_kw in enumerate(evidence_keywords):
1709
+ overlap = line_tokens & ev_kw
1710
+ n_overlap = len(overlap)
1711
+ specific_overlap = line_specific_tokens & evidence_specific_keywords[idx]
1712
+ effective_overlap = (
1713
+ len(specific_overlap) if line_specific_tokens else n_overlap
1714
+ )
1715
+ effective_base = (
1716
+ len(line_specific_tokens) if line_specific_tokens else len(line_tokens)
1717
+ )
1718
+ ratio = effective_overlap / effective_base if effective_base else 0
1719
+
1720
+ if n_overlap < min_overlap or ratio < min_ratio:
1721
+ continue
1722
+ if line_specific_tokens and len(specific_overlap) < min_specific_overlap:
1723
+ continue
1724
+
1725
+ score = (
1726
+ float(len(specific_overlap)),
1727
+ ratio,
1728
+ float(n_overlap),
1729
+ results[idx].score,
1730
+ )
1731
+ candidates.append((score, idx + 1)) # 1-indexed
1732
+
1733
+ if candidates:
1734
+ candidates.sort(reverse=True)
1735
+ citations = sorted({
1736
+ citation
1737
+ for _, citation in candidates[:max_citations]
1738
+ })
1739
+
1740
+ existing = {
1741
+ int(match.group(1))
1742
+ for match in re.finditer(r"\[(\d+)\]", line)
1743
+ }
1744
+ new_citations = [c for c in citations if c not in existing]
1745
+ if not new_citations:
1746
+ annotated.append(line)
1747
+ continue
1748
+
1749
+ cite_str = " " + " ".join(f"[{n}]" for n in new_citations)
1750
+ annotated.append(f"{line}{cite_str}")
1751
+ else:
1752
+ annotated.append(line)
1753
+
1754
+ return "\n".join(annotated)
1755
+
1756
+
1757
+ def _score_to_confidence(score: float, backend: str = "tfidf") -> str:
1758
+ """Convert a similarity score to a human-readable confidence label.
1759
+
1760
+ Parameters
1761
+ ----------
1762
+ score:
1763
+ Cosine similarity score.
1764
+ backend:
1765
+ Either ``"tfidf"`` or ``"embeddings"``. The two backends
1766
+ produce scores in very different ranges, so we use separate
1767
+ threshold sets:
1768
+
1769
+ - TF-IDF: typical range 0.01–0.4 for relevant chunks.
1770
+ Thresholds: 0.15 / 0.08 / 0.03.
1771
+ - Embeddings (BGE-small cosine): typical range 0.5–0.9 for
1772
+ relevant chunks. Thresholds: 0.7 / 0.6 / 0.5.
1773
+ """
1774
+ if backend == "embeddings":
1775
+ if score >= 0.70:
1776
+ return "High"
1777
+ elif score >= 0.60:
1778
+ return "Medium"
1779
+ elif score >= 0.50:
1780
+ return "Low"
1781
+ else:
1782
+ return "Very Low"
1783
+ # TF-IDF (default)
1784
+ if score >= 0.15:
1785
+ return "High"
1786
+ elif score >= 0.08:
1787
+ return "Medium"
1788
+ elif score >= 0.03:
1789
+ return "Low"
1790
+ else:
1791
+ return "Very Low"