MetacouplingLLM 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metacouplingllm/__init__.py +208 -0
- metacouplingllm/core.py +4542 -0
- metacouplingllm/data/Papers.zip +0 -0
- metacouplingllm/data/PeriTelecoupling_clean.csv +49953 -0
- metacouplingllm/data/PeriTelecoupling_subset.csv +11 -0
- metacouplingllm/data/chunk_embeddings.manifest.json +11 -0
- metacouplingllm/data/chunk_embeddings.npy +0 -0
- metacouplingllm/data/pericoupled_adm1_edge_list.csv +8291 -0
- metacouplingllm/data/telecoupling_literature.bib +3498 -0
- metacouplingllm/knowledge/__init__.py +1 -0
- metacouplingllm/knowledge/adm1_pericoupling.py +591 -0
- metacouplingllm/knowledge/citations.py +109 -0
- metacouplingllm/knowledge/countries.py +557 -0
- metacouplingllm/knowledge/examples.py +713 -0
- metacouplingllm/knowledge/framework.py +717 -0
- metacouplingllm/knowledge/literature.py +518 -0
- metacouplingllm/knowledge/pericoupling.py +241 -0
- metacouplingllm/knowledge/rag.py +1791 -0
- metacouplingllm/knowledge/references.py +112 -0
- metacouplingllm/knowledge/websearch.py +1842 -0
- metacouplingllm/llm/__init__.py +1 -0
- metacouplingllm/llm/client.py +636 -0
- metacouplingllm/llm/parser.py +868 -0
- metacouplingllm/output/__init__.py +1 -0
- metacouplingllm/output/formatter.py +276 -0
- metacouplingllm/prompts/__init__.py +1 -0
- metacouplingllm/prompts/builder.py +477 -0
- metacouplingllm/prompts/templates.py +556 -0
- metacouplingllm/visualization/__init__.py +114 -0
- metacouplingllm/visualization/adm1_map.py +884 -0
- metacouplingllm/visualization/worldmap.py +1372 -0
- metacouplingllm-0.1.0.dist-info/METADATA +440 -0
- metacouplingllm-0.1.0.dist-info/RECORD +35 -0
- metacouplingllm-0.1.0.dist-info/WHEEL +4 -0
- metacouplingllm-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Domain knowledge for the metacoupling and telecoupling frameworks."""
|
|
@@ -0,0 +1,591 @@
|
|
|
1
|
+
"""
|
|
2
|
+
ADM1 (subnational) pericoupling database.
|
|
3
|
+
|
|
4
|
+
Loads a curated CSV edge list of first-level administrative divisions (ADM1)
|
|
5
|
+
to determine which subnational regions share a border (pericoupled). The
|
|
6
|
+
database uses World Bank ADM1 codes (e.g., ``"MEX001"``, ``"USA035"``).
|
|
7
|
+
|
|
8
|
+
The edge list contains **8,290 border pairs** covering **3,366 unique ADM1
|
|
9
|
+
regions** across **195 countries**, including both within-country and
|
|
10
|
+
cross-country borders.
|
|
11
|
+
|
|
12
|
+
Source
|
|
13
|
+
------
|
|
14
|
+
``pericoupled_adm1_edge_list.csv`` (research dataset using World Bank ADM1
|
|
15
|
+
codes and ISO 3166-1 alpha-3 country codes).
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import csv
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
from enum import Enum
|
|
23
|
+
from importlib import resources
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
import re
|
|
26
|
+
|
|
27
|
+
from metacouplingllm.knowledge.countries import resolve_country_code
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
# ---------------------------------------------------------------------------
|
|
31
|
+
# Public types
|
|
32
|
+
# ---------------------------------------------------------------------------
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class Adm1PairType(str, Enum):
|
|
36
|
+
"""Result of an ADM1 pericoupling database lookup."""
|
|
37
|
+
|
|
38
|
+
PERICOUPLED = "pericoupled"
|
|
39
|
+
TELECOUPLED = "telecoupled"
|
|
40
|
+
SAME_REGION = "same_region"
|
|
41
|
+
UNKNOWN = "unknown"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(frozen=True)
|
|
45
|
+
class Adm1PericouplingResult:
|
|
46
|
+
"""Result of looking up an ADM1 region pair in the pericoupling database.
|
|
47
|
+
|
|
48
|
+
Attributes
|
|
49
|
+
----------
|
|
50
|
+
pair_type:
|
|
51
|
+
Whether the pair is pericoupled, telecoupled, same_region, or unknown.
|
|
52
|
+
code_a:
|
|
53
|
+
ADM1 code for the first region, or ``None`` if unresolved.
|
|
54
|
+
code_b:
|
|
55
|
+
ADM1 code for the second region, or ``None`` if unresolved.
|
|
56
|
+
cross_country:
|
|
57
|
+
``True`` if the two regions belong to different countries.
|
|
58
|
+
confidence:
|
|
59
|
+
``"database"`` if looked up from the database, ``"unresolved"`` if
|
|
60
|
+
one or both ADM1 codes are not in the database, ``"same_region"``
|
|
61
|
+
if both codes are identical.
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
pair_type: Adm1PairType
|
|
65
|
+
code_a: str | None
|
|
66
|
+
code_b: str | None
|
|
67
|
+
cross_country: bool = False
|
|
68
|
+
confidence: str = "database"
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
# ---------------------------------------------------------------------------
|
|
72
|
+
# Internal cached data structures
|
|
73
|
+
# ---------------------------------------------------------------------------
|
|
74
|
+
|
|
75
|
+
_adm1_pairs: frozenset[frozenset[str]] | None = None
|
|
76
|
+
_adm1_neighbors: dict[str, set[str]] | None = None
|
|
77
|
+
_adm1_country: dict[str, str] | None = None
|
|
78
|
+
_adm1_metadata: dict[str, dict[str, str]] | None = None
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
# ---------------------------------------------------------------------------
|
|
82
|
+
# CSV loading
|
|
83
|
+
# ---------------------------------------------------------------------------
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _locate_csv() -> Path | None:
|
|
87
|
+
"""Find the ADM1 pericoupling CSV data file."""
|
|
88
|
+
try:
|
|
89
|
+
data_pkg = resources.files("metacoupling") / "data"
|
|
90
|
+
except (TypeError, ModuleNotFoundError):
|
|
91
|
+
return None
|
|
92
|
+
|
|
93
|
+
candidate = data_pkg / "pericoupled_adm1_edge_list.csv"
|
|
94
|
+
try:
|
|
95
|
+
path = Path(str(candidate))
|
|
96
|
+
if path.is_file():
|
|
97
|
+
return path
|
|
98
|
+
except Exception:
|
|
99
|
+
pass
|
|
100
|
+
return None
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _load_adm1_data() -> tuple[
|
|
104
|
+
frozenset[frozenset[str]],
|
|
105
|
+
dict[str, set[str]],
|
|
106
|
+
dict[str, str],
|
|
107
|
+
dict[str, dict[str, str]],
|
|
108
|
+
]:
|
|
109
|
+
"""Load all ADM1 data from the CSV file.
|
|
110
|
+
|
|
111
|
+
Returns
|
|
112
|
+
-------
|
|
113
|
+
Tuple of:
|
|
114
|
+
- pairs: frozenset of frozenset pairs (all adjacent ADM1 code pairs)
|
|
115
|
+
- neighbors: adjacency index {code: {neighbor_codes}}
|
|
116
|
+
- country_map: {adm1_code: iso_a3_code}
|
|
117
|
+
- metadata: {adm1_code: {name, country_name, iso_a3, wb_region}}
|
|
118
|
+
"""
|
|
119
|
+
csv_path = _locate_csv()
|
|
120
|
+
if csv_path is None:
|
|
121
|
+
return frozenset(), {}, {}, {}
|
|
122
|
+
|
|
123
|
+
pairs: set[frozenset[str]] = set()
|
|
124
|
+
neighbors: dict[str, set[str]] = {}
|
|
125
|
+
country_map: dict[str, str] = {}
|
|
126
|
+
metadata: dict[str, dict[str, str]] = {}
|
|
127
|
+
|
|
128
|
+
with open(csv_path, newline="", encoding="utf-8-sig") as fh:
|
|
129
|
+
reader = csv.DictReader(fh)
|
|
130
|
+
for row in reader:
|
|
131
|
+
code_a = row["ADM1_code_A"].strip()
|
|
132
|
+
code_b = row["ADM1_code_B"].strip()
|
|
133
|
+
|
|
134
|
+
# Record the pair
|
|
135
|
+
pairs.add(frozenset({code_a, code_b}))
|
|
136
|
+
|
|
137
|
+
# Build adjacency index
|
|
138
|
+
neighbors.setdefault(code_a, set()).add(code_b)
|
|
139
|
+
neighbors.setdefault(code_b, set()).add(code_a)
|
|
140
|
+
|
|
141
|
+
# Record country mapping and metadata for both sides
|
|
142
|
+
for suffix, code in [("_A", code_a), ("_B", code_b)]:
|
|
143
|
+
if code not in country_map:
|
|
144
|
+
iso = row[f"ISO_A3{suffix}"].strip()
|
|
145
|
+
country_map[code] = iso
|
|
146
|
+
metadata[code] = {
|
|
147
|
+
"name": row[f"ADM1_name{suffix}"].strip(),
|
|
148
|
+
"country_name": row[f"country{suffix}"].strip(),
|
|
149
|
+
"iso_a3": iso,
|
|
150
|
+
"wb_region": row[f"WB_region{suffix}"].strip(),
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
return frozenset(pairs), neighbors, country_map, metadata
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _ensure_loaded() -> None:
|
|
157
|
+
"""Lazily load and cache all ADM1 data."""
|
|
158
|
+
global _adm1_pairs, _adm1_neighbors, _adm1_country, _adm1_metadata
|
|
159
|
+
if _adm1_pairs is None:
|
|
160
|
+
_adm1_pairs, _adm1_neighbors, _adm1_country, _adm1_metadata = (
|
|
161
|
+
_load_adm1_data()
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
# ---------------------------------------------------------------------------
|
|
166
|
+
# Lookup functions
|
|
167
|
+
# ---------------------------------------------------------------------------
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def lookup_adm1_pericoupling(
|
|
171
|
+
adm1_a: str,
|
|
172
|
+
adm1_b: str,
|
|
173
|
+
) -> Adm1PericouplingResult:
|
|
174
|
+
"""Look up whether two ADM1 regions are pericoupled or telecoupled.
|
|
175
|
+
|
|
176
|
+
Parameters
|
|
177
|
+
----------
|
|
178
|
+
adm1_a:
|
|
179
|
+
ADM1 code of the first region (e.g., ``"MEX001"``).
|
|
180
|
+
adm1_b:
|
|
181
|
+
ADM1 code of the second region (e.g., ``"USA035"``).
|
|
182
|
+
|
|
183
|
+
Returns
|
|
184
|
+
-------
|
|
185
|
+
An :class:`Adm1PericouplingResult` with the lookup outcome.
|
|
186
|
+
|
|
187
|
+
Examples
|
|
188
|
+
--------
|
|
189
|
+
>>> result = lookup_adm1_pericoupling("AFG001", "PAK005")
|
|
190
|
+
>>> result.pair_type
|
|
191
|
+
<Adm1PairType.PERICOUPLED: 'pericoupled'>
|
|
192
|
+
>>> result.cross_country
|
|
193
|
+
True
|
|
194
|
+
"""
|
|
195
|
+
_ensure_loaded()
|
|
196
|
+
assert _adm1_pairs is not None
|
|
197
|
+
assert _adm1_country is not None
|
|
198
|
+
|
|
199
|
+
adm1_a = adm1_a.strip()
|
|
200
|
+
adm1_b = adm1_b.strip()
|
|
201
|
+
|
|
202
|
+
# Check if codes are in the database
|
|
203
|
+
a_known = adm1_a in _adm1_country
|
|
204
|
+
b_known = adm1_b in _adm1_country
|
|
205
|
+
|
|
206
|
+
if not a_known or not b_known:
|
|
207
|
+
return Adm1PericouplingResult(
|
|
208
|
+
pair_type=Adm1PairType.UNKNOWN,
|
|
209
|
+
code_a=adm1_a if a_known else None,
|
|
210
|
+
code_b=adm1_b if b_known else None,
|
|
211
|
+
cross_country=False,
|
|
212
|
+
confidence="unresolved",
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
if adm1_a == adm1_b:
|
|
216
|
+
return Adm1PericouplingResult(
|
|
217
|
+
pair_type=Adm1PairType.SAME_REGION,
|
|
218
|
+
code_a=adm1_a,
|
|
219
|
+
code_b=adm1_b,
|
|
220
|
+
cross_country=False,
|
|
221
|
+
confidence="same_region",
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
cross_country = _adm1_country[adm1_a] != _adm1_country[adm1_b]
|
|
225
|
+
pair = frozenset({adm1_a, adm1_b})
|
|
226
|
+
|
|
227
|
+
if pair in _adm1_pairs:
|
|
228
|
+
return Adm1PericouplingResult(
|
|
229
|
+
pair_type=Adm1PairType.PERICOUPLED,
|
|
230
|
+
code_a=adm1_a,
|
|
231
|
+
code_b=adm1_b,
|
|
232
|
+
cross_country=cross_country,
|
|
233
|
+
confidence="database",
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
return Adm1PericouplingResult(
|
|
237
|
+
pair_type=Adm1PairType.TELECOUPLED,
|
|
238
|
+
code_a=adm1_a,
|
|
239
|
+
code_b=adm1_b,
|
|
240
|
+
cross_country=cross_country,
|
|
241
|
+
confidence="database",
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def get_adm1_neighbors(adm1_code: str) -> set[str]:
|
|
246
|
+
"""Return all ADM1 codes adjacent to the given region.
|
|
247
|
+
|
|
248
|
+
Includes both within-country and cross-border neighbors.
|
|
249
|
+
|
|
250
|
+
Parameters
|
|
251
|
+
----------
|
|
252
|
+
adm1_code:
|
|
253
|
+
ADM1 code (e.g., ``"AFG001"``).
|
|
254
|
+
|
|
255
|
+
Returns
|
|
256
|
+
-------
|
|
257
|
+
Set of adjacent ADM1 codes. Empty set if the code is not in the
|
|
258
|
+
database.
|
|
259
|
+
|
|
260
|
+
Examples
|
|
261
|
+
--------
|
|
262
|
+
>>> neighbors = get_adm1_neighbors("AFG001")
|
|
263
|
+
>>> "PAK005" in neighbors
|
|
264
|
+
True
|
|
265
|
+
>>> "AFG024" in neighbors
|
|
266
|
+
True
|
|
267
|
+
"""
|
|
268
|
+
_ensure_loaded()
|
|
269
|
+
assert _adm1_neighbors is not None
|
|
270
|
+
return set(_adm1_neighbors.get(adm1_code.strip(), set()))
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def get_cross_border_neighbors(adm1_code: str) -> set[str]:
|
|
274
|
+
"""Return ADM1 codes adjacent to the given region in *different* countries.
|
|
275
|
+
|
|
276
|
+
Parameters
|
|
277
|
+
----------
|
|
278
|
+
adm1_code:
|
|
279
|
+
ADM1 code (e.g., ``"AFG001"``).
|
|
280
|
+
|
|
281
|
+
Returns
|
|
282
|
+
-------
|
|
283
|
+
Set of adjacent ADM1 codes that belong to a different country.
|
|
284
|
+
|
|
285
|
+
Examples
|
|
286
|
+
--------
|
|
287
|
+
>>> cross = get_cross_border_neighbors("AFG001")
|
|
288
|
+
>>> "PAK005" in cross
|
|
289
|
+
True
|
|
290
|
+
>>> "AFG024" in cross # same country
|
|
291
|
+
False
|
|
292
|
+
"""
|
|
293
|
+
_ensure_loaded()
|
|
294
|
+
assert _adm1_neighbors is not None
|
|
295
|
+
assert _adm1_country is not None
|
|
296
|
+
|
|
297
|
+
adm1_code = adm1_code.strip()
|
|
298
|
+
if adm1_code not in _adm1_country:
|
|
299
|
+
return set()
|
|
300
|
+
|
|
301
|
+
focal_iso = _adm1_country[adm1_code]
|
|
302
|
+
result: set[str] = set()
|
|
303
|
+
for neighbor in _adm1_neighbors.get(adm1_code, set()):
|
|
304
|
+
if _adm1_country.get(neighbor) != focal_iso:
|
|
305
|
+
result.add(neighbor)
|
|
306
|
+
return result
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def get_adm1_codes_for_country(country: str) -> set[str]:
|
|
310
|
+
"""Return all ADM1 codes belonging to a country.
|
|
311
|
+
|
|
312
|
+
Parameters
|
|
313
|
+
----------
|
|
314
|
+
country:
|
|
315
|
+
Country name (e.g., ``"Afghanistan"``) or ISO alpha-3 code
|
|
316
|
+
(e.g., ``"AFG"``).
|
|
317
|
+
|
|
318
|
+
Returns
|
|
319
|
+
-------
|
|
320
|
+
Set of ADM1 codes for that country. Empty set if the country
|
|
321
|
+
cannot be resolved or has no ADM1 regions in the database.
|
|
322
|
+
|
|
323
|
+
Examples
|
|
324
|
+
--------
|
|
325
|
+
>>> codes = get_adm1_codes_for_country("Afghanistan")
|
|
326
|
+
>>> "AFG001" in codes
|
|
327
|
+
True
|
|
328
|
+
"""
|
|
329
|
+
_ensure_loaded()
|
|
330
|
+
assert _adm1_country is not None
|
|
331
|
+
|
|
332
|
+
# Try direct ISO code match first
|
|
333
|
+
country_stripped = country.strip().upper()
|
|
334
|
+
if len(country_stripped) == 3:
|
|
335
|
+
# Check if it's a valid ISO code in our data
|
|
336
|
+
matching = {
|
|
337
|
+
code
|
|
338
|
+
for code, iso in _adm1_country.items()
|
|
339
|
+
if iso == country_stripped
|
|
340
|
+
}
|
|
341
|
+
if matching:
|
|
342
|
+
return matching
|
|
343
|
+
|
|
344
|
+
# Try resolving via countries.py
|
|
345
|
+
iso_code = resolve_country_code(country)
|
|
346
|
+
if iso_code is None:
|
|
347
|
+
return set()
|
|
348
|
+
|
|
349
|
+
return {
|
|
350
|
+
code for code, iso in _adm1_country.items() if iso == iso_code
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def get_adm1_info(adm1_code: str) -> dict[str, str] | None:
|
|
355
|
+
"""Return metadata for an ADM1 region.
|
|
356
|
+
|
|
357
|
+
Parameters
|
|
358
|
+
----------
|
|
359
|
+
adm1_code:
|
|
360
|
+
ADM1 code (e.g., ``"AFG001"``).
|
|
361
|
+
|
|
362
|
+
Returns
|
|
363
|
+
-------
|
|
364
|
+
A dictionary with keys ``name``, ``country_name``, ``iso_a3``,
|
|
365
|
+
``wb_region``, or ``None`` if the code is not in the database.
|
|
366
|
+
|
|
367
|
+
Examples
|
|
368
|
+
--------
|
|
369
|
+
>>> info = get_adm1_info("AFG001")
|
|
370
|
+
>>> info["name"]
|
|
371
|
+
'Badakhshan'
|
|
372
|
+
>>> info["country_name"]
|
|
373
|
+
'Afghanistan'
|
|
374
|
+
"""
|
|
375
|
+
_ensure_loaded()
|
|
376
|
+
assert _adm1_metadata is not None
|
|
377
|
+
return _adm1_metadata.get(adm1_code.strip())
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def is_adm1_pericoupled(adm1_a: str, adm1_b: str) -> bool | None:
|
|
381
|
+
"""Convenience function for quick ADM1 pericoupling checks.
|
|
382
|
+
|
|
383
|
+
Returns
|
|
384
|
+
-------
|
|
385
|
+
``True`` if the two regions share a border, ``False`` if they do
|
|
386
|
+
not, ``None`` if one or both codes are not in the database.
|
|
387
|
+
"""
|
|
388
|
+
result = lookup_adm1_pericoupling(adm1_a, adm1_b)
|
|
389
|
+
if result.pair_type == Adm1PairType.PERICOUPLED:
|
|
390
|
+
return True
|
|
391
|
+
if result.pair_type in (Adm1PairType.TELECOUPLED, Adm1PairType.SAME_REGION):
|
|
392
|
+
return False
|
|
393
|
+
return None
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def get_adm1_country(adm1_code: str) -> str | None:
|
|
397
|
+
"""Return the ISO alpha-3 country code for an ADM1 region.
|
|
398
|
+
|
|
399
|
+
Parameters
|
|
400
|
+
----------
|
|
401
|
+
adm1_code:
|
|
402
|
+
ADM1 code (e.g., ``"MEX001"``).
|
|
403
|
+
|
|
404
|
+
Returns
|
|
405
|
+
-------
|
|
406
|
+
ISO alpha-3 code (e.g., ``"MEX"``), or ``None`` if the code is
|
|
407
|
+
not in the database.
|
|
408
|
+
"""
|
|
409
|
+
_ensure_loaded()
|
|
410
|
+
assert _adm1_country is not None
|
|
411
|
+
return _adm1_country.get(adm1_code.strip())
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
# ---------------------------------------------------------------------------
|
|
415
|
+
# ADM1 name-to-code resolution
|
|
416
|
+
# ---------------------------------------------------------------------------
|
|
417
|
+
|
|
418
|
+
# Known suffixes in World Bank ADM1 naming conventions.
|
|
419
|
+
# These are stripped during fuzzy matching so "Anhui" matches "Anhui Sheng".
|
|
420
|
+
_ADM1_NAME_SUFFIXES: tuple[str, ...] = (
|
|
421
|
+
"sheng",
|
|
422
|
+
"oblast",
|
|
423
|
+
"okrug",
|
|
424
|
+
"kray",
|
|
425
|
+
"pradesh",
|
|
426
|
+
"laen",
|
|
427
|
+
"novads",
|
|
428
|
+
"maakond",
|
|
429
|
+
"region",
|
|
430
|
+
"province",
|
|
431
|
+
"state",
|
|
432
|
+
"zizhiqu",
|
|
433
|
+
"rep.",
|
|
434
|
+
"giang",
|
|
435
|
+
)
|
|
436
|
+
|
|
437
|
+
_adm1_name_index: dict[str, list[tuple[str, str]]] | None = None
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def _build_adm1_name_index() -> dict[str, list[tuple[str, str]]]:
|
|
441
|
+
"""Build a reverse index from lowercase name → [(adm1_code, iso_a3), ...].
|
|
442
|
+
|
|
443
|
+
Also indexes suffix-stripped and slash-split variants.
|
|
444
|
+
"""
|
|
445
|
+
_ensure_loaded()
|
|
446
|
+
assert _adm1_metadata is not None
|
|
447
|
+
|
|
448
|
+
index: dict[str, list[tuple[str, str]]] = {}
|
|
449
|
+
|
|
450
|
+
for code, meta in _adm1_metadata.items():
|
|
451
|
+
raw_name = meta["name"]
|
|
452
|
+
iso = meta["iso_a3"]
|
|
453
|
+
entry = (code, iso)
|
|
454
|
+
|
|
455
|
+
names_to_index: set[str] = set()
|
|
456
|
+
|
|
457
|
+
# 1. Full name (lowercased)
|
|
458
|
+
names_to_index.add(raw_name.lower())
|
|
459
|
+
|
|
460
|
+
# 2. Suffix-stripped variants
|
|
461
|
+
name_lower = raw_name.lower()
|
|
462
|
+
for suffix in _ADM1_NAME_SUFFIXES:
|
|
463
|
+
if name_lower.endswith(" " + suffix):
|
|
464
|
+
stripped = name_lower[: -(len(suffix) + 1)].strip()
|
|
465
|
+
if stripped and len(stripped) >= 2:
|
|
466
|
+
names_to_index.add(stripped)
|
|
467
|
+
|
|
468
|
+
# 3. Slash-split variants (e.g., "Cataluña/Catalunya")
|
|
469
|
+
if "/" in raw_name:
|
|
470
|
+
for part in raw_name.split("/"):
|
|
471
|
+
part = part.strip().lower()
|
|
472
|
+
if part and len(part) >= 2:
|
|
473
|
+
names_to_index.add(part)
|
|
474
|
+
|
|
475
|
+
for n in names_to_index:
|
|
476
|
+
index.setdefault(n, []).append(entry)
|
|
477
|
+
|
|
478
|
+
return index
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def _get_adm1_name_index() -> dict[str, list[tuple[str, str]]]:
|
|
482
|
+
"""Lazily build and cache the ADM1 name-to-code index."""
|
|
483
|
+
global _adm1_name_index
|
|
484
|
+
if _adm1_name_index is None:
|
|
485
|
+
_adm1_name_index = _build_adm1_name_index()
|
|
486
|
+
return _adm1_name_index
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def resolve_adm1_code(
|
|
490
|
+
name: str,
|
|
491
|
+
country: str | None = None,
|
|
492
|
+
) -> str | None:
|
|
493
|
+
"""Resolve an ADM1 region name to its World Bank ADM1 code.
|
|
494
|
+
|
|
495
|
+
Parameters
|
|
496
|
+
----------
|
|
497
|
+
name:
|
|
498
|
+
Region name (e.g., ``"Michigan"``, ``"Anhui"``, ``"Catalunya"``).
|
|
499
|
+
country:
|
|
500
|
+
Optional country name or ISO alpha-3 code to disambiguate when
|
|
501
|
+
the name matches regions in multiple countries (e.g., ``"Georgia"``
|
|
502
|
+
exists in both the USA and as a country).
|
|
503
|
+
|
|
504
|
+
Returns
|
|
505
|
+
-------
|
|
506
|
+
The ADM1 code (e.g., ``"USA023"``), or ``None`` if not found.
|
|
507
|
+
|
|
508
|
+
Examples
|
|
509
|
+
--------
|
|
510
|
+
>>> resolve_adm1_code("Michigan")
|
|
511
|
+
'USA023'
|
|
512
|
+
|
|
513
|
+
>>> resolve_adm1_code("Anhui")
|
|
514
|
+
'CHN001'
|
|
515
|
+
|
|
516
|
+
>>> resolve_adm1_code("Georgia", country="United States")
|
|
517
|
+
'USA011'
|
|
518
|
+
"""
|
|
519
|
+
if not name or not name.strip():
|
|
520
|
+
return None
|
|
521
|
+
|
|
522
|
+
name_lower = name.strip().lower()
|
|
523
|
+
|
|
524
|
+
# Skip if the name resolves as a country (e.g. "Ethiopia", "China")
|
|
525
|
+
# UNLESS a country filter is provided — that signals the caller knows
|
|
526
|
+
# this is a subnational region (e.g. "Georgia" in "USA").
|
|
527
|
+
if country is None and resolve_country_code(name_lower):
|
|
528
|
+
return None
|
|
529
|
+
|
|
530
|
+
index = _get_adm1_name_index()
|
|
531
|
+
|
|
532
|
+
# Resolve country filter if provided
|
|
533
|
+
country_iso: str | None = None
|
|
534
|
+
if country is not None:
|
|
535
|
+
country_iso = resolve_country_code(country)
|
|
536
|
+
|
|
537
|
+
# --- Strategy 1: Direct index lookup ---
|
|
538
|
+
candidates = index.get(name_lower)
|
|
539
|
+
if candidates:
|
|
540
|
+
result = _pick_best_candidate(candidates, country_iso)
|
|
541
|
+
if result:
|
|
542
|
+
return result
|
|
543
|
+
|
|
544
|
+
def _contains_phrase(haystack: str, needle: str) -> bool:
|
|
545
|
+
pattern = rf"(?<![a-z]){re.escape(needle)}(?![a-z])"
|
|
546
|
+
return re.search(pattern, haystack) is not None
|
|
547
|
+
|
|
548
|
+
# --- Strategy 2: Substring match ---
|
|
549
|
+
# Only if name is at least 4 chars and the matching DB name is also
|
|
550
|
+
# at least 4 chars, to avoid false positives from short fragments.
|
|
551
|
+
if len(name_lower) >= 4:
|
|
552
|
+
for db_name, entries in index.items():
|
|
553
|
+
if len(db_name) < 4:
|
|
554
|
+
continue
|
|
555
|
+
if (
|
|
556
|
+
_contains_phrase(db_name, name_lower)
|
|
557
|
+
or _contains_phrase(name_lower, db_name)
|
|
558
|
+
):
|
|
559
|
+
result = _pick_best_candidate(entries, country_iso)
|
|
560
|
+
if result:
|
|
561
|
+
return result
|
|
562
|
+
|
|
563
|
+
return None
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def _pick_best_candidate(
|
|
567
|
+
candidates: list[tuple[str, str]],
|
|
568
|
+
country_iso: str | None,
|
|
569
|
+
) -> str | None:
|
|
570
|
+
"""Pick the best ADM1 code from a list of candidates.
|
|
571
|
+
|
|
572
|
+
If a country filter is given, only candidates matching that country
|
|
573
|
+
are considered. Otherwise returns the first candidate.
|
|
574
|
+
"""
|
|
575
|
+
if country_iso:
|
|
576
|
+
for code, iso in candidates:
|
|
577
|
+
if iso == country_iso:
|
|
578
|
+
return code
|
|
579
|
+
return None # No match for the specified country
|
|
580
|
+
|
|
581
|
+
if len(candidates) == 1:
|
|
582
|
+
return candidates[0][0]
|
|
583
|
+
|
|
584
|
+
# Multiple candidates, no country filter — return first but only
|
|
585
|
+
# if all candidates are in the same country (unambiguous)
|
|
586
|
+
isos = {iso for _, iso in candidates}
|
|
587
|
+
if len(isos) == 1:
|
|
588
|
+
return candidates[0][0]
|
|
589
|
+
|
|
590
|
+
# Ambiguous — cannot determine without country hint
|
|
591
|
+
return None
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Citation sanitization utilities for the pre-retrieval RAG pipeline.
|
|
3
|
+
|
|
4
|
+
When the LLM is given a `<retrieved_literature>` block with N passages
|
|
5
|
+
labeled `[1]..[N]` and asked to cite them inline, it may occasionally
|
|
6
|
+
emit out-of-range tokens like `[99]` (a hallucinated citation) or copy a
|
|
7
|
+
stale `[1]` from a previous turn that no longer corresponds to the same
|
|
8
|
+
paper. This module provides:
|
|
9
|
+
|
|
10
|
+
- :func:`sanitize_citations` strips bracket-number tokens whose ID falls
|
|
11
|
+
outside ``[1, n_valid]``, logs a warning naming the stripped IDs, and
|
|
12
|
+
runs an idempotent whitespace/punctuation cleanup pass so the result
|
|
13
|
+
reads naturally.
|
|
14
|
+
- :func:`extract_cited_ids` returns the set of citation numbers used in a
|
|
15
|
+
block of text — handy for tests and downstream validation.
|
|
16
|
+
|
|
17
|
+
Both functions only touch numeric bracket tokens like ``[1]`` or ``[42]``;
|
|
18
|
+
they leave alphanumeric tokens like ``[W1]`` (web search citations) alone.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import logging
|
|
24
|
+
import re
|
|
25
|
+
|
|
26
|
+
logger = logging.getLogger(__name__)
|
|
27
|
+
|
|
28
|
+
# Match [N] where N is one or more digits.
|
|
29
|
+
# Does NOT match [W1] (web citations) or other alphanumeric tokens.
|
|
30
|
+
CITATION_PATTERN = re.compile(r"\[(\d+)\]")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _cleanup_whitespace(text: str) -> str:
|
|
34
|
+
"""Idempotent cleanup of leftover whitespace and punctuation spacing.
|
|
35
|
+
|
|
36
|
+
Run after citation stripping so that an input like
|
|
37
|
+
``"claim [99] and more"`` does not leave a double space, and
|
|
38
|
+
``"claim [99]."`` does not leave an orphan space before the period.
|
|
39
|
+
|
|
40
|
+
Only horizontal whitespace and punctuation-adjacent spaces are
|
|
41
|
+
touched — newlines are preserved exactly so list/paragraph structure
|
|
42
|
+
survives.
|
|
43
|
+
"""
|
|
44
|
+
# Collapse runs of horizontal whitespace (tabs + spaces); keep newlines
|
|
45
|
+
text = re.sub(r"[ \t]+", " ", text)
|
|
46
|
+
# Remove any space immediately before sentence punctuation
|
|
47
|
+
text = re.sub(r" +([.,;:!?)])", r"\1", text)
|
|
48
|
+
# Strip trailing horizontal whitespace on every line
|
|
49
|
+
text = re.sub(r" +$", "", text, flags=re.MULTILINE)
|
|
50
|
+
return text
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def sanitize_citations(
|
|
54
|
+
text: str, n_valid: int
|
|
55
|
+
) -> tuple[str, set[int]]:
|
|
56
|
+
"""Strip numeric citation tokens whose ID is outside ``[1, n_valid]``.
|
|
57
|
+
|
|
58
|
+
Parameters
|
|
59
|
+
----------
|
|
60
|
+
text:
|
|
61
|
+
The LLM-generated text to sanitize. Typically the raw response
|
|
62
|
+
``content`` before any structured parsing.
|
|
63
|
+
n_valid:
|
|
64
|
+
The number of valid citation labels in the most recent
|
|
65
|
+
``<retrieved_literature>`` block. Tokens ``[1]..[n_valid]`` are
|
|
66
|
+
kept; everything else (e.g., ``[0]``, ``[99]``) is stripped.
|
|
67
|
+
|
|
68
|
+
Returns
|
|
69
|
+
-------
|
|
70
|
+
A tuple ``(sanitized_text, invalid_ids)``:
|
|
71
|
+
|
|
72
|
+
- ``sanitized_text`` is the cleaned text with invalid tokens removed
|
|
73
|
+
and whitespace/punctuation normalized.
|
|
74
|
+
- ``invalid_ids`` is the set of out-of-range integers that were
|
|
75
|
+
found and stripped (empty if input was already clean).
|
|
76
|
+
|
|
77
|
+
A WARNING-level log message naming the stripped IDs is emitted via
|
|
78
|
+
the module logger when at least one invalid token is found.
|
|
79
|
+
"""
|
|
80
|
+
invalid_ids: set[int] = set()
|
|
81
|
+
|
|
82
|
+
def _replacer(match: re.Match[str]) -> str:
|
|
83
|
+
cid = int(match.group(1))
|
|
84
|
+
if 1 <= cid <= n_valid:
|
|
85
|
+
return match.group(0)
|
|
86
|
+
invalid_ids.add(cid)
|
|
87
|
+
return ""
|
|
88
|
+
|
|
89
|
+
sanitized = CITATION_PATTERN.sub(_replacer, text)
|
|
90
|
+
if invalid_ids:
|
|
91
|
+
logger.warning(
|
|
92
|
+
"Stripped %d invalid citation ID(s): %s (valid range: 1..%d)",
|
|
93
|
+
len(invalid_ids),
|
|
94
|
+
sorted(invalid_ids),
|
|
95
|
+
n_valid,
|
|
96
|
+
)
|
|
97
|
+
# Always run cleanup — it is idempotent on already-clean input and
|
|
98
|
+
# only matters when something was actually stripped.
|
|
99
|
+
sanitized = _cleanup_whitespace(sanitized)
|
|
100
|
+
return sanitized, invalid_ids
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def extract_cited_ids(text: str) -> set[int]:
|
|
104
|
+
"""Return the set of numeric citation IDs used in ``text``.
|
|
105
|
+
|
|
106
|
+
Useful for downstream validation and tests. Web citations like
|
|
107
|
+
``[W1]`` are NOT included — only purely numeric brackets.
|
|
108
|
+
"""
|
|
109
|
+
return {int(m) for m in CITATION_PATTERN.findall(text)}
|