MetacouplingLLM 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. metacouplingllm/__init__.py +208 -0
  2. metacouplingllm/core.py +4542 -0
  3. metacouplingllm/data/Papers.zip +0 -0
  4. metacouplingllm/data/PeriTelecoupling_clean.csv +49953 -0
  5. metacouplingllm/data/PeriTelecoupling_subset.csv +11 -0
  6. metacouplingllm/data/chunk_embeddings.manifest.json +11 -0
  7. metacouplingllm/data/chunk_embeddings.npy +0 -0
  8. metacouplingllm/data/pericoupled_adm1_edge_list.csv +8291 -0
  9. metacouplingllm/data/telecoupling_literature.bib +3498 -0
  10. metacouplingllm/knowledge/__init__.py +1 -0
  11. metacouplingllm/knowledge/adm1_pericoupling.py +591 -0
  12. metacouplingllm/knowledge/citations.py +109 -0
  13. metacouplingllm/knowledge/countries.py +557 -0
  14. metacouplingllm/knowledge/examples.py +713 -0
  15. metacouplingllm/knowledge/framework.py +717 -0
  16. metacouplingllm/knowledge/literature.py +518 -0
  17. metacouplingllm/knowledge/pericoupling.py +241 -0
  18. metacouplingllm/knowledge/rag.py +1791 -0
  19. metacouplingllm/knowledge/references.py +112 -0
  20. metacouplingllm/knowledge/websearch.py +1842 -0
  21. metacouplingllm/llm/__init__.py +1 -0
  22. metacouplingllm/llm/client.py +636 -0
  23. metacouplingllm/llm/parser.py +868 -0
  24. metacouplingllm/output/__init__.py +1 -0
  25. metacouplingllm/output/formatter.py +276 -0
  26. metacouplingllm/prompts/__init__.py +1 -0
  27. metacouplingllm/prompts/builder.py +477 -0
  28. metacouplingllm/prompts/templates.py +556 -0
  29. metacouplingllm/visualization/__init__.py +114 -0
  30. metacouplingllm/visualization/adm1_map.py +884 -0
  31. metacouplingllm/visualization/worldmap.py +1372 -0
  32. metacouplingllm-0.1.0.dist-info/METADATA +440 -0
  33. metacouplingllm-0.1.0.dist-info/RECORD +35 -0
  34. metacouplingllm-0.1.0.dist-info/WHEEL +4 -0
  35. metacouplingllm-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1 @@
1
+ """Domain knowledge for the metacoupling and telecoupling frameworks."""
@@ -0,0 +1,591 @@
1
+ """
2
+ ADM1 (subnational) pericoupling database.
3
+
4
+ Loads a curated CSV edge list of first-level administrative divisions (ADM1)
5
+ to determine which subnational regions share a border (pericoupled). The
6
+ database uses World Bank ADM1 codes (e.g., ``"MEX001"``, ``"USA035"``).
7
+
8
+ The edge list contains **8,290 border pairs** covering **3,366 unique ADM1
9
+ regions** across **195 countries**, including both within-country and
10
+ cross-country borders.
11
+
12
+ Source
13
+ ------
14
+ ``pericoupled_adm1_edge_list.csv`` (research dataset using World Bank ADM1
15
+ codes and ISO 3166-1 alpha-3 country codes).
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import csv
21
+ from dataclasses import dataclass
22
+ from enum import Enum
23
+ from importlib import resources
24
+ from pathlib import Path
25
+ import re
26
+
27
+ from metacouplingllm.knowledge.countries import resolve_country_code
28
+
29
+
30
+ # ---------------------------------------------------------------------------
31
+ # Public types
32
+ # ---------------------------------------------------------------------------
33
+
34
+
35
+ class Adm1PairType(str, Enum):
36
+ """Result of an ADM1 pericoupling database lookup."""
37
+
38
+ PERICOUPLED = "pericoupled"
39
+ TELECOUPLED = "telecoupled"
40
+ SAME_REGION = "same_region"
41
+ UNKNOWN = "unknown"
42
+
43
+
44
+ @dataclass(frozen=True)
45
+ class Adm1PericouplingResult:
46
+ """Result of looking up an ADM1 region pair in the pericoupling database.
47
+
48
+ Attributes
49
+ ----------
50
+ pair_type:
51
+ Whether the pair is pericoupled, telecoupled, same_region, or unknown.
52
+ code_a:
53
+ ADM1 code for the first region, or ``None`` if unresolved.
54
+ code_b:
55
+ ADM1 code for the second region, or ``None`` if unresolved.
56
+ cross_country:
57
+ ``True`` if the two regions belong to different countries.
58
+ confidence:
59
+ ``"database"`` if looked up from the database, ``"unresolved"`` if
60
+ one or both ADM1 codes are not in the database, ``"same_region"``
61
+ if both codes are identical.
62
+ """
63
+
64
+ pair_type: Adm1PairType
65
+ code_a: str | None
66
+ code_b: str | None
67
+ cross_country: bool = False
68
+ confidence: str = "database"
69
+
70
+
71
+ # ---------------------------------------------------------------------------
72
+ # Internal cached data structures
73
+ # ---------------------------------------------------------------------------
74
+
75
+ _adm1_pairs: frozenset[frozenset[str]] | None = None
76
+ _adm1_neighbors: dict[str, set[str]] | None = None
77
+ _adm1_country: dict[str, str] | None = None
78
+ _adm1_metadata: dict[str, dict[str, str]] | None = None
79
+
80
+
81
+ # ---------------------------------------------------------------------------
82
+ # CSV loading
83
+ # ---------------------------------------------------------------------------
84
+
85
+
86
+ def _locate_csv() -> Path | None:
87
+ """Find the ADM1 pericoupling CSV data file."""
88
+ try:
89
+ data_pkg = resources.files("metacoupling") / "data"
90
+ except (TypeError, ModuleNotFoundError):
91
+ return None
92
+
93
+ candidate = data_pkg / "pericoupled_adm1_edge_list.csv"
94
+ try:
95
+ path = Path(str(candidate))
96
+ if path.is_file():
97
+ return path
98
+ except Exception:
99
+ pass
100
+ return None
101
+
102
+
103
+ def _load_adm1_data() -> tuple[
104
+ frozenset[frozenset[str]],
105
+ dict[str, set[str]],
106
+ dict[str, str],
107
+ dict[str, dict[str, str]],
108
+ ]:
109
+ """Load all ADM1 data from the CSV file.
110
+
111
+ Returns
112
+ -------
113
+ Tuple of:
114
+ - pairs: frozenset of frozenset pairs (all adjacent ADM1 code pairs)
115
+ - neighbors: adjacency index {code: {neighbor_codes}}
116
+ - country_map: {adm1_code: iso_a3_code}
117
+ - metadata: {adm1_code: {name, country_name, iso_a3, wb_region}}
118
+ """
119
+ csv_path = _locate_csv()
120
+ if csv_path is None:
121
+ return frozenset(), {}, {}, {}
122
+
123
+ pairs: set[frozenset[str]] = set()
124
+ neighbors: dict[str, set[str]] = {}
125
+ country_map: dict[str, str] = {}
126
+ metadata: dict[str, dict[str, str]] = {}
127
+
128
+ with open(csv_path, newline="", encoding="utf-8-sig") as fh:
129
+ reader = csv.DictReader(fh)
130
+ for row in reader:
131
+ code_a = row["ADM1_code_A"].strip()
132
+ code_b = row["ADM1_code_B"].strip()
133
+
134
+ # Record the pair
135
+ pairs.add(frozenset({code_a, code_b}))
136
+
137
+ # Build adjacency index
138
+ neighbors.setdefault(code_a, set()).add(code_b)
139
+ neighbors.setdefault(code_b, set()).add(code_a)
140
+
141
+ # Record country mapping and metadata for both sides
142
+ for suffix, code in [("_A", code_a), ("_B", code_b)]:
143
+ if code not in country_map:
144
+ iso = row[f"ISO_A3{suffix}"].strip()
145
+ country_map[code] = iso
146
+ metadata[code] = {
147
+ "name": row[f"ADM1_name{suffix}"].strip(),
148
+ "country_name": row[f"country{suffix}"].strip(),
149
+ "iso_a3": iso,
150
+ "wb_region": row[f"WB_region{suffix}"].strip(),
151
+ }
152
+
153
+ return frozenset(pairs), neighbors, country_map, metadata
154
+
155
+
156
+ def _ensure_loaded() -> None:
157
+ """Lazily load and cache all ADM1 data."""
158
+ global _adm1_pairs, _adm1_neighbors, _adm1_country, _adm1_metadata
159
+ if _adm1_pairs is None:
160
+ _adm1_pairs, _adm1_neighbors, _adm1_country, _adm1_metadata = (
161
+ _load_adm1_data()
162
+ )
163
+
164
+
165
+ # ---------------------------------------------------------------------------
166
+ # Lookup functions
167
+ # ---------------------------------------------------------------------------
168
+
169
+
170
+ def lookup_adm1_pericoupling(
171
+ adm1_a: str,
172
+ adm1_b: str,
173
+ ) -> Adm1PericouplingResult:
174
+ """Look up whether two ADM1 regions are pericoupled or telecoupled.
175
+
176
+ Parameters
177
+ ----------
178
+ adm1_a:
179
+ ADM1 code of the first region (e.g., ``"MEX001"``).
180
+ adm1_b:
181
+ ADM1 code of the second region (e.g., ``"USA035"``).
182
+
183
+ Returns
184
+ -------
185
+ An :class:`Adm1PericouplingResult` with the lookup outcome.
186
+
187
+ Examples
188
+ --------
189
+ >>> result = lookup_adm1_pericoupling("AFG001", "PAK005")
190
+ >>> result.pair_type
191
+ <Adm1PairType.PERICOUPLED: 'pericoupled'>
192
+ >>> result.cross_country
193
+ True
194
+ """
195
+ _ensure_loaded()
196
+ assert _adm1_pairs is not None
197
+ assert _adm1_country is not None
198
+
199
+ adm1_a = adm1_a.strip()
200
+ adm1_b = adm1_b.strip()
201
+
202
+ # Check if codes are in the database
203
+ a_known = adm1_a in _adm1_country
204
+ b_known = adm1_b in _adm1_country
205
+
206
+ if not a_known or not b_known:
207
+ return Adm1PericouplingResult(
208
+ pair_type=Adm1PairType.UNKNOWN,
209
+ code_a=adm1_a if a_known else None,
210
+ code_b=adm1_b if b_known else None,
211
+ cross_country=False,
212
+ confidence="unresolved",
213
+ )
214
+
215
+ if adm1_a == adm1_b:
216
+ return Adm1PericouplingResult(
217
+ pair_type=Adm1PairType.SAME_REGION,
218
+ code_a=adm1_a,
219
+ code_b=adm1_b,
220
+ cross_country=False,
221
+ confidence="same_region",
222
+ )
223
+
224
+ cross_country = _adm1_country[adm1_a] != _adm1_country[adm1_b]
225
+ pair = frozenset({adm1_a, adm1_b})
226
+
227
+ if pair in _adm1_pairs:
228
+ return Adm1PericouplingResult(
229
+ pair_type=Adm1PairType.PERICOUPLED,
230
+ code_a=adm1_a,
231
+ code_b=adm1_b,
232
+ cross_country=cross_country,
233
+ confidence="database",
234
+ )
235
+
236
+ return Adm1PericouplingResult(
237
+ pair_type=Adm1PairType.TELECOUPLED,
238
+ code_a=adm1_a,
239
+ code_b=adm1_b,
240
+ cross_country=cross_country,
241
+ confidence="database",
242
+ )
243
+
244
+
245
+ def get_adm1_neighbors(adm1_code: str) -> set[str]:
246
+ """Return all ADM1 codes adjacent to the given region.
247
+
248
+ Includes both within-country and cross-border neighbors.
249
+
250
+ Parameters
251
+ ----------
252
+ adm1_code:
253
+ ADM1 code (e.g., ``"AFG001"``).
254
+
255
+ Returns
256
+ -------
257
+ Set of adjacent ADM1 codes. Empty set if the code is not in the
258
+ database.
259
+
260
+ Examples
261
+ --------
262
+ >>> neighbors = get_adm1_neighbors("AFG001")
263
+ >>> "PAK005" in neighbors
264
+ True
265
+ >>> "AFG024" in neighbors
266
+ True
267
+ """
268
+ _ensure_loaded()
269
+ assert _adm1_neighbors is not None
270
+ return set(_adm1_neighbors.get(adm1_code.strip(), set()))
271
+
272
+
273
+ def get_cross_border_neighbors(adm1_code: str) -> set[str]:
274
+ """Return ADM1 codes adjacent to the given region in *different* countries.
275
+
276
+ Parameters
277
+ ----------
278
+ adm1_code:
279
+ ADM1 code (e.g., ``"AFG001"``).
280
+
281
+ Returns
282
+ -------
283
+ Set of adjacent ADM1 codes that belong to a different country.
284
+
285
+ Examples
286
+ --------
287
+ >>> cross = get_cross_border_neighbors("AFG001")
288
+ >>> "PAK005" in cross
289
+ True
290
+ >>> "AFG024" in cross # same country
291
+ False
292
+ """
293
+ _ensure_loaded()
294
+ assert _adm1_neighbors is not None
295
+ assert _adm1_country is not None
296
+
297
+ adm1_code = adm1_code.strip()
298
+ if adm1_code not in _adm1_country:
299
+ return set()
300
+
301
+ focal_iso = _adm1_country[adm1_code]
302
+ result: set[str] = set()
303
+ for neighbor in _adm1_neighbors.get(adm1_code, set()):
304
+ if _adm1_country.get(neighbor) != focal_iso:
305
+ result.add(neighbor)
306
+ return result
307
+
308
+
309
+ def get_adm1_codes_for_country(country: str) -> set[str]:
310
+ """Return all ADM1 codes belonging to a country.
311
+
312
+ Parameters
313
+ ----------
314
+ country:
315
+ Country name (e.g., ``"Afghanistan"``) or ISO alpha-3 code
316
+ (e.g., ``"AFG"``).
317
+
318
+ Returns
319
+ -------
320
+ Set of ADM1 codes for that country. Empty set if the country
321
+ cannot be resolved or has no ADM1 regions in the database.
322
+
323
+ Examples
324
+ --------
325
+ >>> codes = get_adm1_codes_for_country("Afghanistan")
326
+ >>> "AFG001" in codes
327
+ True
328
+ """
329
+ _ensure_loaded()
330
+ assert _adm1_country is not None
331
+
332
+ # Try direct ISO code match first
333
+ country_stripped = country.strip().upper()
334
+ if len(country_stripped) == 3:
335
+ # Check if it's a valid ISO code in our data
336
+ matching = {
337
+ code
338
+ for code, iso in _adm1_country.items()
339
+ if iso == country_stripped
340
+ }
341
+ if matching:
342
+ return matching
343
+
344
+ # Try resolving via countries.py
345
+ iso_code = resolve_country_code(country)
346
+ if iso_code is None:
347
+ return set()
348
+
349
+ return {
350
+ code for code, iso in _adm1_country.items() if iso == iso_code
351
+ }
352
+
353
+
354
+ def get_adm1_info(adm1_code: str) -> dict[str, str] | None:
355
+ """Return metadata for an ADM1 region.
356
+
357
+ Parameters
358
+ ----------
359
+ adm1_code:
360
+ ADM1 code (e.g., ``"AFG001"``).
361
+
362
+ Returns
363
+ -------
364
+ A dictionary with keys ``name``, ``country_name``, ``iso_a3``,
365
+ ``wb_region``, or ``None`` if the code is not in the database.
366
+
367
+ Examples
368
+ --------
369
+ >>> info = get_adm1_info("AFG001")
370
+ >>> info["name"]
371
+ 'Badakhshan'
372
+ >>> info["country_name"]
373
+ 'Afghanistan'
374
+ """
375
+ _ensure_loaded()
376
+ assert _adm1_metadata is not None
377
+ return _adm1_metadata.get(adm1_code.strip())
378
+
379
+
380
+ def is_adm1_pericoupled(adm1_a: str, adm1_b: str) -> bool | None:
381
+ """Convenience function for quick ADM1 pericoupling checks.
382
+
383
+ Returns
384
+ -------
385
+ ``True`` if the two regions share a border, ``False`` if they do
386
+ not, ``None`` if one or both codes are not in the database.
387
+ """
388
+ result = lookup_adm1_pericoupling(adm1_a, adm1_b)
389
+ if result.pair_type == Adm1PairType.PERICOUPLED:
390
+ return True
391
+ if result.pair_type in (Adm1PairType.TELECOUPLED, Adm1PairType.SAME_REGION):
392
+ return False
393
+ return None
394
+
395
+
396
+ def get_adm1_country(adm1_code: str) -> str | None:
397
+ """Return the ISO alpha-3 country code for an ADM1 region.
398
+
399
+ Parameters
400
+ ----------
401
+ adm1_code:
402
+ ADM1 code (e.g., ``"MEX001"``).
403
+
404
+ Returns
405
+ -------
406
+ ISO alpha-3 code (e.g., ``"MEX"``), or ``None`` if the code is
407
+ not in the database.
408
+ """
409
+ _ensure_loaded()
410
+ assert _adm1_country is not None
411
+ return _adm1_country.get(adm1_code.strip())
412
+
413
+
414
+ # ---------------------------------------------------------------------------
415
+ # ADM1 name-to-code resolution
416
+ # ---------------------------------------------------------------------------
417
+
418
+ # Known suffixes in World Bank ADM1 naming conventions.
419
+ # These are stripped during fuzzy matching so "Anhui" matches "Anhui Sheng".
420
+ _ADM1_NAME_SUFFIXES: tuple[str, ...] = (
421
+ "sheng",
422
+ "oblast",
423
+ "okrug",
424
+ "kray",
425
+ "pradesh",
426
+ "laen",
427
+ "novads",
428
+ "maakond",
429
+ "region",
430
+ "province",
431
+ "state",
432
+ "zizhiqu",
433
+ "rep.",
434
+ "giang",
435
+ )
436
+
437
+ _adm1_name_index: dict[str, list[tuple[str, str]]] | None = None
438
+
439
+
440
+ def _build_adm1_name_index() -> dict[str, list[tuple[str, str]]]:
441
+ """Build a reverse index from lowercase name → [(adm1_code, iso_a3), ...].
442
+
443
+ Also indexes suffix-stripped and slash-split variants.
444
+ """
445
+ _ensure_loaded()
446
+ assert _adm1_metadata is not None
447
+
448
+ index: dict[str, list[tuple[str, str]]] = {}
449
+
450
+ for code, meta in _adm1_metadata.items():
451
+ raw_name = meta["name"]
452
+ iso = meta["iso_a3"]
453
+ entry = (code, iso)
454
+
455
+ names_to_index: set[str] = set()
456
+
457
+ # 1. Full name (lowercased)
458
+ names_to_index.add(raw_name.lower())
459
+
460
+ # 2. Suffix-stripped variants
461
+ name_lower = raw_name.lower()
462
+ for suffix in _ADM1_NAME_SUFFIXES:
463
+ if name_lower.endswith(" " + suffix):
464
+ stripped = name_lower[: -(len(suffix) + 1)].strip()
465
+ if stripped and len(stripped) >= 2:
466
+ names_to_index.add(stripped)
467
+
468
+ # 3. Slash-split variants (e.g., "Cataluña/Catalunya")
469
+ if "/" in raw_name:
470
+ for part in raw_name.split("/"):
471
+ part = part.strip().lower()
472
+ if part and len(part) >= 2:
473
+ names_to_index.add(part)
474
+
475
+ for n in names_to_index:
476
+ index.setdefault(n, []).append(entry)
477
+
478
+ return index
479
+
480
+
481
+ def _get_adm1_name_index() -> dict[str, list[tuple[str, str]]]:
482
+ """Lazily build and cache the ADM1 name-to-code index."""
483
+ global _adm1_name_index
484
+ if _adm1_name_index is None:
485
+ _adm1_name_index = _build_adm1_name_index()
486
+ return _adm1_name_index
487
+
488
+
489
+ def resolve_adm1_code(
490
+ name: str,
491
+ country: str | None = None,
492
+ ) -> str | None:
493
+ """Resolve an ADM1 region name to its World Bank ADM1 code.
494
+
495
+ Parameters
496
+ ----------
497
+ name:
498
+ Region name (e.g., ``"Michigan"``, ``"Anhui"``, ``"Catalunya"``).
499
+ country:
500
+ Optional country name or ISO alpha-3 code to disambiguate when
501
+ the name matches regions in multiple countries (e.g., ``"Georgia"``
502
+ exists in both the USA and as a country).
503
+
504
+ Returns
505
+ -------
506
+ The ADM1 code (e.g., ``"USA023"``), or ``None`` if not found.
507
+
508
+ Examples
509
+ --------
510
+ >>> resolve_adm1_code("Michigan")
511
+ 'USA023'
512
+
513
+ >>> resolve_adm1_code("Anhui")
514
+ 'CHN001'
515
+
516
+ >>> resolve_adm1_code("Georgia", country="United States")
517
+ 'USA011'
518
+ """
519
+ if not name or not name.strip():
520
+ return None
521
+
522
+ name_lower = name.strip().lower()
523
+
524
+ # Skip if the name resolves as a country (e.g. "Ethiopia", "China")
525
+ # UNLESS a country filter is provided — that signals the caller knows
526
+ # this is a subnational region (e.g. "Georgia" in "USA").
527
+ if country is None and resolve_country_code(name_lower):
528
+ return None
529
+
530
+ index = _get_adm1_name_index()
531
+
532
+ # Resolve country filter if provided
533
+ country_iso: str | None = None
534
+ if country is not None:
535
+ country_iso = resolve_country_code(country)
536
+
537
+ # --- Strategy 1: Direct index lookup ---
538
+ candidates = index.get(name_lower)
539
+ if candidates:
540
+ result = _pick_best_candidate(candidates, country_iso)
541
+ if result:
542
+ return result
543
+
544
+ def _contains_phrase(haystack: str, needle: str) -> bool:
545
+ pattern = rf"(?<![a-z]){re.escape(needle)}(?![a-z])"
546
+ return re.search(pattern, haystack) is not None
547
+
548
+ # --- Strategy 2: Substring match ---
549
+ # Only if name is at least 4 chars and the matching DB name is also
550
+ # at least 4 chars, to avoid false positives from short fragments.
551
+ if len(name_lower) >= 4:
552
+ for db_name, entries in index.items():
553
+ if len(db_name) < 4:
554
+ continue
555
+ if (
556
+ _contains_phrase(db_name, name_lower)
557
+ or _contains_phrase(name_lower, db_name)
558
+ ):
559
+ result = _pick_best_candidate(entries, country_iso)
560
+ if result:
561
+ return result
562
+
563
+ return None
564
+
565
+
566
+ def _pick_best_candidate(
567
+ candidates: list[tuple[str, str]],
568
+ country_iso: str | None,
569
+ ) -> str | None:
570
+ """Pick the best ADM1 code from a list of candidates.
571
+
572
+ If a country filter is given, only candidates matching that country
573
+ are considered. Otherwise returns the first candidate.
574
+ """
575
+ if country_iso:
576
+ for code, iso in candidates:
577
+ if iso == country_iso:
578
+ return code
579
+ return None # No match for the specified country
580
+
581
+ if len(candidates) == 1:
582
+ return candidates[0][0]
583
+
584
+ # Multiple candidates, no country filter — return first but only
585
+ # if all candidates are in the same country (unambiguous)
586
+ isos = {iso for _, iso in candidates}
587
+ if len(isos) == 1:
588
+ return candidates[0][0]
589
+
590
+ # Ambiguous — cannot determine without country hint
591
+ return None
@@ -0,0 +1,109 @@
1
+ """
2
+ Citation sanitization utilities for the pre-retrieval RAG pipeline.
3
+
4
+ When the LLM is given a `<retrieved_literature>` block with N passages
5
+ labeled `[1]..[N]` and asked to cite them inline, it may occasionally
6
+ emit out-of-range tokens like `[99]` (a hallucinated citation) or copy a
7
+ stale `[1]` from a previous turn that no longer corresponds to the same
8
+ paper. This module provides:
9
+
10
+ - :func:`sanitize_citations` strips bracket-number tokens whose ID falls
11
+ outside ``[1, n_valid]``, logs a warning naming the stripped IDs, and
12
+ runs an idempotent whitespace/punctuation cleanup pass so the result
13
+ reads naturally.
14
+ - :func:`extract_cited_ids` returns the set of citation numbers used in a
15
+ block of text — handy for tests and downstream validation.
16
+
17
+ Both functions only touch numeric bracket tokens like ``[1]`` or ``[42]``;
18
+ they leave alphanumeric tokens like ``[W1]`` (web search citations) alone.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import logging
24
+ import re
25
+
26
+ logger = logging.getLogger(__name__)
27
+
28
+ # Match [N] where N is one or more digits.
29
+ # Does NOT match [W1] (web citations) or other alphanumeric tokens.
30
+ CITATION_PATTERN = re.compile(r"\[(\d+)\]")
31
+
32
+
33
+ def _cleanup_whitespace(text: str) -> str:
34
+ """Idempotent cleanup of leftover whitespace and punctuation spacing.
35
+
36
+ Run after citation stripping so that an input like
37
+ ``"claim [99] and more"`` does not leave a double space, and
38
+ ``"claim [99]."`` does not leave an orphan space before the period.
39
+
40
+ Only horizontal whitespace and punctuation-adjacent spaces are
41
+ touched — newlines are preserved exactly so list/paragraph structure
42
+ survives.
43
+ """
44
+ # Collapse runs of horizontal whitespace (tabs + spaces); keep newlines
45
+ text = re.sub(r"[ \t]+", " ", text)
46
+ # Remove any space immediately before sentence punctuation
47
+ text = re.sub(r" +([.,;:!?)])", r"\1", text)
48
+ # Strip trailing horizontal whitespace on every line
49
+ text = re.sub(r" +$", "", text, flags=re.MULTILINE)
50
+ return text
51
+
52
+
53
+ def sanitize_citations(
54
+ text: str, n_valid: int
55
+ ) -> tuple[str, set[int]]:
56
+ """Strip numeric citation tokens whose ID is outside ``[1, n_valid]``.
57
+
58
+ Parameters
59
+ ----------
60
+ text:
61
+ The LLM-generated text to sanitize. Typically the raw response
62
+ ``content`` before any structured parsing.
63
+ n_valid:
64
+ The number of valid citation labels in the most recent
65
+ ``<retrieved_literature>`` block. Tokens ``[1]..[n_valid]`` are
66
+ kept; everything else (e.g., ``[0]``, ``[99]``) is stripped.
67
+
68
+ Returns
69
+ -------
70
+ A tuple ``(sanitized_text, invalid_ids)``:
71
+
72
+ - ``sanitized_text`` is the cleaned text with invalid tokens removed
73
+ and whitespace/punctuation normalized.
74
+ - ``invalid_ids`` is the set of out-of-range integers that were
75
+ found and stripped (empty if input was already clean).
76
+
77
+ A WARNING-level log message naming the stripped IDs is emitted via
78
+ the module logger when at least one invalid token is found.
79
+ """
80
+ invalid_ids: set[int] = set()
81
+
82
+ def _replacer(match: re.Match[str]) -> str:
83
+ cid = int(match.group(1))
84
+ if 1 <= cid <= n_valid:
85
+ return match.group(0)
86
+ invalid_ids.add(cid)
87
+ return ""
88
+
89
+ sanitized = CITATION_PATTERN.sub(_replacer, text)
90
+ if invalid_ids:
91
+ logger.warning(
92
+ "Stripped %d invalid citation ID(s): %s (valid range: 1..%d)",
93
+ len(invalid_ids),
94
+ sorted(invalid_ids),
95
+ n_valid,
96
+ )
97
+ # Always run cleanup — it is idempotent on already-clean input and
98
+ # only matters when something was actually stripped.
99
+ sanitized = _cleanup_whitespace(sanitized)
100
+ return sanitized, invalid_ids
101
+
102
+
103
+ def extract_cited_ids(text: str) -> set[int]:
104
+ """Return the set of numeric citation IDs used in ``text``.
105
+
106
+ Useful for downstream validation and tests. Web citations like
107
+ ``[W1]`` are NOT included — only purely numeric brackets.
108
+ """
109
+ return {int(m) for m in CITATION_PATTERN.findall(text)}