MetacouplingLLM 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. metacouplingllm/__init__.py +208 -0
  2. metacouplingllm/core.py +4542 -0
  3. metacouplingllm/data/Papers.zip +0 -0
  4. metacouplingllm/data/PeriTelecoupling_clean.csv +49953 -0
  5. metacouplingllm/data/PeriTelecoupling_subset.csv +11 -0
  6. metacouplingllm/data/chunk_embeddings.manifest.json +11 -0
  7. metacouplingllm/data/chunk_embeddings.npy +0 -0
  8. metacouplingllm/data/pericoupled_adm1_edge_list.csv +8291 -0
  9. metacouplingllm/data/telecoupling_literature.bib +3498 -0
  10. metacouplingllm/knowledge/__init__.py +1 -0
  11. metacouplingllm/knowledge/adm1_pericoupling.py +591 -0
  12. metacouplingllm/knowledge/citations.py +109 -0
  13. metacouplingllm/knowledge/countries.py +557 -0
  14. metacouplingllm/knowledge/examples.py +713 -0
  15. metacouplingllm/knowledge/framework.py +717 -0
  16. metacouplingllm/knowledge/literature.py +518 -0
  17. metacouplingllm/knowledge/pericoupling.py +241 -0
  18. metacouplingllm/knowledge/rag.py +1791 -0
  19. metacouplingllm/knowledge/references.py +112 -0
  20. metacouplingllm/knowledge/websearch.py +1842 -0
  21. metacouplingllm/llm/__init__.py +1 -0
  22. metacouplingllm/llm/client.py +636 -0
  23. metacouplingllm/llm/parser.py +868 -0
  24. metacouplingllm/output/__init__.py +1 -0
  25. metacouplingllm/output/formatter.py +276 -0
  26. metacouplingllm/prompts/__init__.py +1 -0
  27. metacouplingllm/prompts/builder.py +477 -0
  28. metacouplingllm/prompts/templates.py +556 -0
  29. metacouplingllm/visualization/__init__.py +114 -0
  30. metacouplingllm/visualization/adm1_map.py +884 -0
  31. metacouplingllm/visualization/worldmap.py +1372 -0
  32. metacouplingllm-0.1.0.dist-info/METADATA +440 -0
  33. metacouplingllm-0.1.0.dist-info/RECORD +35 -0
  34. metacouplingllm-0.1.0.dist-info/WHEEL +4 -0
  35. metacouplingllm-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,4542 @@
1
+ """
2
+ Main user-facing interface for the Metacoupling package.
3
+
4
+ :class:`MetacouplingAssistant` is the primary entry point. Researchers
5
+ create an advisor with their LLM client, then call :meth:`analyze` and
6
+ :meth:`refine` to iteratively build a telecoupling/metacoupling analysis.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import logging
12
+ import re
13
+ from dataclasses import dataclass
14
+ from pathlib import Path
15
+ from typing import TYPE_CHECKING
16
+
17
+ from metacouplingllm.knowledge.citations import sanitize_citations
18
+ from metacouplingllm.knowledge.countries import get_country_name, resolve_country_code
19
+ from metacouplingllm.knowledge.literature import Paper
20
+ from metacouplingllm.knowledge.pericoupling import (
21
+ PairCouplingType,
22
+ lookup_pericoupling,
23
+ )
24
+ from metacouplingllm.llm.client import (
25
+ AnthropicAdapter,
26
+ GeminiAdapter,
27
+ GrokAdapter,
28
+ LLMClient,
29
+ LLMResponse,
30
+ Message,
31
+ OpenAIAdapter,
32
+ )
33
+ from metacouplingllm.llm.parser import ParsedAnalysis, parse_analysis
34
+ from metacouplingllm.output.formatter import AnalysisFormatter
35
+ from metacouplingllm.prompts.builder import PromptBuilder
36
+
37
+ if TYPE_CHECKING:
38
+ from matplotlib.figure import Figure
39
+
40
+ from metacouplingllm.knowledge.rag import RetrievalResult
41
+
42
+
43
+ logger = logging.getLogger(__name__)
44
+
45
+
46
+ # Allowed values for the rag_mode parameter on MetacouplingAssistant.
47
+ _VALID_RAG_MODES = frozenset({"pre_retrieval", "post_hoc"})
48
+
49
+ # Public built-in RAG corpus names. Use these when the bundled journal
50
+ # corpus should be used; ``rag_papers_dir`` is reserved for custom folders.
51
+ JOURNAL_ARTICLES_2025 = "journal_articles_2025"
52
+ _BUILTIN_RAG_CORPUS_ALIASES = frozenset({
53
+ JOURNAL_ARTICLES_2025,
54
+ "journal_articles",
55
+ "bundled",
56
+ })
57
+ _BUILTIN_RAG_SENTINEL = "__metacoupling_builtin_journal_articles_2025__"
58
+
59
+ # Canonical regex for splitting flow direction strings at arrow symbols.
60
+ # Used across flow resolution and validation to ensure consistency.
61
+ _FLOW_ARROW_RE = re.compile(r"\s*(?:\u2192|\u2194|<->|<=>|->|=>)\s*")
62
+ # Detect whether a flow direction contains a connector (arrow or "between X and Y").
63
+ _FLOW_HAS_CONNECTOR_RE = re.compile(
64
+ r"\bbetween\b.+\band\b|\u2192|\u2194|<->|<=>|->|=>",
65
+ re.IGNORECASE,
66
+ )
67
+ _UNSUPPORTED_AUTOMAP_SCOPE_RE = re.compile(
68
+ r"\b("
69
+ r"city|cities|municipalit(?:y|ies)|municipal|town|village|"
70
+ r"watershed|watersheds|river\s+basin|basin|"
71
+ r"protected\s+area|nature\s+reserve|national\s+nature\s+reserve|"
72
+ r"national\s+park|park|reserve|conservation\s+area"
73
+ r")\b",
74
+ re.IGNORECASE,
75
+ )
76
+
77
+
78
+ @dataclass
79
+ class AnalysisResult:
80
+ """The result of an analysis or refinement turn.
81
+
82
+ Attributes
83
+ ----------
84
+ parsed:
85
+ Structured representation (best-effort parsed).
86
+ formatted:
87
+ Human-readable text ready to display.
88
+ raw:
89
+ The unprocessed LLM response.
90
+ turn_number:
91
+ Which conversational turn produced this result (1-indexed).
92
+ usage:
93
+ Token-usage information from the LLM (if available).
94
+ map:
95
+ Auto-generated matplotlib Figure (country-level or ADM1), or
96
+ ``None`` when ``auto_map=False`` or visualization deps are missing.
97
+ structured_supplement:
98
+ When ``rag_structured_extraction=True`` and the pre-retrieval
99
+ RAG pipeline produced retrieval hits, a second LLM pass
100
+ returns a dict with keys ``additional_{sending,receiving,
101
+ spillover}_mentions``, ``{sending,receiving,spillover}_subsystem_fills``,
102
+ and ``supplementary_flows``. Each entry carries evidence
103
+ passage IDs pointing back at the ``[N]`` references in the
104
+ ``SUPPORTING EVIDENCE FROM LITERATURE`` block. Also rendered
105
+ visibly in ``formatted`` under a ``SUPPLEMENTARY STRUCTURED
106
+ EXTRACTION`` block. ``None`` when the feature is disabled or
107
+ extraction failed.
108
+ map_notice:
109
+ Human-readable map status note appended to ``formatted`` when
110
+ ``auto_map=True``. This explains whether a map was generated or
111
+ why no map was produced.
112
+ """
113
+
114
+ parsed: ParsedAnalysis
115
+ formatted: str
116
+ raw: str
117
+ turn_number: int
118
+ usage: dict[str, int] | None = None
119
+ map: Figure | None = None
120
+ web_map_signals: dict[str, object] | None = None
121
+ structured_supplement: dict[str, object] | None = None
122
+ map_notice: str | None = None
123
+
124
+ def __repr__(self) -> str:
125
+ parts = [f"turn={self.turn_number}"]
126
+ if self.parsed.is_parsed:
127
+ parts.append(f"parsed=True")
128
+ if self.parsed.coupling_classification:
129
+ cc = self.parsed.coupling_classification[:30]
130
+ parts.append(f"classification='{cc}'")
131
+ parts.append(f"formatted={len(self.formatted)} chars")
132
+ if self.map is not None:
133
+ parts.append("map=Figure")
134
+ if self.usage:
135
+ total = self.usage.get(
136
+ "total_tokens",
137
+ sum(self.usage.values()),
138
+ )
139
+ parts.append(f"tokens={total}")
140
+ return f"AnalysisResult({', '.join(parts)})"
141
+
142
+
143
+ @dataclass
144
+ class RAGResult:
145
+ """The result of a single RAG-only Q&A turn (``coupling_analysis=False``).
146
+
147
+ Attributes
148
+ ----------
149
+ answer:
150
+ The LLM's narrative response with ``[N]`` citation markers as
151
+ the LLM emitted them. Markers may be sparse (e.g. ``[1]``,
152
+ ``[3]``, ``[7]``) because they refer to positions in
153
+ ``retrieved_passages`` and the LLM doesn't cite every passage.
154
+ For a display-friendly version with re-numbered sequential
155
+ ``[1]``, ``[2]``, ``[3]`` markers plus a bibliography, use
156
+ :attr:`formatted` instead.
157
+ Out-of-range markers (e.g. ``[99]`` when only 8 passages were
158
+ retrieved) have already been sanitized.
159
+ references:
160
+ Papers actually cited by the LLM, deduplicated by ``key`` and
161
+ ordered by first appearance in the answer.
162
+ retrieved_passages:
163
+ All passages shown to the LLM for this turn (length ``rag_top_k``).
164
+ web_sources:
165
+ Web search results used to supplement the answer, or ``None`` if
166
+ ``web_search=False``.
167
+ turn_number:
168
+ Which conversational turn produced this result (1-indexed within
169
+ the current RAG-mode session).
170
+ usage:
171
+ Token-usage information from the LLM (if available).
172
+ raw:
173
+ The unsanitized LLM response text.
174
+ """
175
+
176
+ answer: str
177
+ references: list[Paper]
178
+ retrieved_passages: list[RetrievalResult]
179
+ web_sources: list[dict[str, str]] | None = None
180
+ turn_number: int = 1
181
+ usage: dict[str, int] | None = None
182
+ raw: str = ""
183
+ retrieval_backend: str = "embeddings"
184
+
185
+ @property
186
+ def formatted(self) -> str:
187
+ """Display-friendly answer with sequential ``[1], [2], ...``
188
+ markers, a bibliography of cited papers, and (when
189
+ ``web_search=True``) a web-sources block.
190
+
191
+ Re-numbers the LLM's possibly-sparse ``[N]`` markers (which
192
+ refer to passage positions in ``retrieved_passages``) into
193
+ sequential ``[1], [2], [3], ...`` ordered by first appearance,
194
+ appends a ``REFERENCES`` block listing only the cited papers in
195
+ the same order, and appends a ``WEB SOURCES`` block (with
196
+ ``[w1], [w2], ...`` IDs) when web search returned hits. Use
197
+ this in place of ``answer`` when you want a single
198
+ ``print(result.formatted)`` to show everything a user needs.
199
+ """
200
+ renumbered = _renumber_citations_sequentially(
201
+ self.answer, len(self.references)
202
+ )
203
+ parts = [renumbered]
204
+ if self.references:
205
+ parts.append(_format_references_block(
206
+ self.references,
207
+ self.retrieved_passages,
208
+ backend=self.retrieval_backend,
209
+ ))
210
+ if self.web_sources:
211
+ parts.append(_format_web_sources_block(self.web_sources))
212
+ return "\n\n".join(parts)
213
+
214
+ def __repr__(self) -> str:
215
+ parts = [f"turn={self.turn_number}"]
216
+ parts.append(f"answer={len(self.answer)} chars")
217
+ parts.append(f"refs={len(self.references)}")
218
+ parts.append(f"passages={len(self.retrieved_passages)}")
219
+ if self.web_sources:
220
+ parts.append(f"web={len(self.web_sources)}")
221
+ if self.usage:
222
+ total = self.usage.get(
223
+ "total_tokens", sum(self.usage.values())
224
+ )
225
+ parts.append(f"tokens={total}")
226
+ return f"RAGResult({', '.join(parts)})"
227
+
228
+
229
+ def _renumber_citations_sequentially(answer: str, n_refs: int) -> str:
230
+ """Map possibly-sparse ``[N]`` tokens to ``[1], [2], [3], ...``
231
+ by first appearance in the text. Tokens whose original IDs were
232
+ already stripped by sanitization are left alone (they shouldn't
233
+ appear). Tokens past ``n_refs`` are also passed through unchanged
234
+ as a safety net.
235
+ """
236
+ if n_refs <= 0:
237
+ return answer
238
+ mapping: dict[int, int] = {}
239
+ next_id = 1
240
+
241
+ def _replace(match: re.Match[str]) -> str:
242
+ nonlocal next_id
243
+ original = int(match.group(1))
244
+ if original not in mapping:
245
+ if next_id > n_refs:
246
+ return match.group(0) # unexpected; leave as-is
247
+ mapping[original] = next_id
248
+ next_id += 1
249
+ return f"[{mapping[original]}]"
250
+
251
+ return _CITATION_TOKEN_RE.sub(_replace, answer)
252
+
253
+
254
+ def _format_references_block(
255
+ refs: list[Paper],
256
+ passages: list[RetrievalResult] | None = None,
257
+ backend: str = "embeddings",
258
+ ) -> str:
259
+ """Render the cited-papers list as a printable bibliography.
260
+
261
+ When ``passages`` is provided, each reference also shows the
262
+ retrieval confidence (High / Medium / Low / Very Low) and the raw
263
+ similarity score, taken from the highest-scoring passage of that
264
+ paper in the retrieval set. ``backend`` ("embeddings" or "tfidf")
265
+ selects the appropriate confidence thresholds.
266
+ """
267
+ # Map paper_key -> best score across all retrieved passages of that paper.
268
+ score_by_key: dict[str, float] = {}
269
+ if passages:
270
+ for r in passages:
271
+ key = r.chunk.paper_key
272
+ if key not in score_by_key or r.score > score_by_key[key]:
273
+ score_by_key[key] = r.score
274
+
275
+ # Lazy import to avoid a hard dependency at import time.
276
+ from metacouplingllm.knowledge.rag import _score_to_confidence
277
+
278
+ lines: list[str] = []
279
+ lines.append("=" * 70)
280
+ lines.append(" REFERENCES (cited in this answer)")
281
+ lines.append("=" * 70)
282
+ for i, paper in enumerate(refs, start=1):
283
+ # Trim very long author strings the same way format_evidence does.
284
+ authors = paper.authors
285
+ if len(authors) > 60:
286
+ parts = authors.split(" and ")
287
+ if len(parts) > 2:
288
+ authors = parts[0] + " et al."
289
+ lines.append("")
290
+ lines.append(f" [{i}] {paper.title}")
291
+ lines.append(f" {authors} ({paper.year})")
292
+ score = score_by_key.get(paper.key)
293
+ if score is not None:
294
+ confidence = _score_to_confidence(score, backend=backend)
295
+ lines.append(
296
+ f" Confidence: {confidence} (score: {score:.3f})"
297
+ )
298
+ lines.append(f" key: {paper.key}")
299
+ return "\n".join(lines)
300
+
301
+
302
+ def _format_web_sources_block(web_sources: list[dict[str, str]]) -> str:
303
+ """Render the web search hits as a printable block.
304
+
305
+ Uses ``[w1], [w2], ...`` IDs to visually distinguish them from
306
+ literature citations (which use ``[1], [2], ...``). Snippets are
307
+ truncated to ~140 chars to keep the block readable.
308
+ """
309
+ lines: list[str] = []
310
+ lines.append("=" * 70)
311
+ lines.append(" WEB SOURCES (background context, not used for citations)")
312
+ lines.append("=" * 70)
313
+ for i, src in enumerate(web_sources, start=1):
314
+ title = (src.get("title") or "(no title)").strip()
315
+ url = (src.get("url") or "").strip()
316
+ snippet = (src.get("snippet") or "").strip().replace("\n", " ")
317
+ if len(snippet) > 140:
318
+ snippet = snippet[:137].rstrip() + "..."
319
+ lines.append("")
320
+ lines.append(f" [w{i}] {title}")
321
+ if url:
322
+ lines.append(f" {url}")
323
+ if snippet:
324
+ lines.append(f" {snippet}")
325
+ return "\n".join(lines)
326
+
327
+
328
+ # System prompt for the RAG-only Q&A mode (coupling_analysis=False).
329
+ # Multi-turn: the advisor maintains _rag_history across analyze() calls
330
+ # so users can ask follow-up questions naturally.
331
+ _RAG_ONLY_SYSTEM_PROMPT = """\
332
+ You are a literature assistant for the metacoupling and telecoupling
333
+ research community. The user already understands the framework — do
334
+ not lecture them about it.
335
+
336
+ This is a multi-turn conversation. Use the prior turns as context to
337
+ interpret follow-up questions, but ground every factual claim in the
338
+ literature passages of the CURRENT turn.
339
+
340
+ Answer using ONLY the retrieved literature passages provided in this
341
+ turn's <retrieved_literature> block. Each passage is numbered
342
+ [1]..[K] for THIS turn. Cite every factual claim with one or more [N]
343
+ markers referring to this turn's passages. Do not invent citations.
344
+ Do not introduce facts that are not supported by the passages.
345
+
346
+ Note: [N] in your prior assistant turns referred to that turn's
347
+ passages, not this one. If the user asks you to recall something from
348
+ a prior turn, refer to it narratively rather than re-citing it by
349
+ number.
350
+
351
+ If the literature does not cover the question — or covers only part
352
+ of it — say so explicitly and describe the gap. A short, well-cited
353
+ answer is better than a long, weakly-supported one.
354
+
355
+ If <web_search_results> is provided, you may use it for context but
356
+ prefer the literature passages for any citable claim.
357
+ """
358
+
359
+ # Regex for extracting cited passage IDs from a sanitized answer.
360
+ _CITATION_TOKEN_RE = re.compile(r"\[(\d+)\]")
361
+
362
+
363
+ class MetacouplingAssistant:
364
+ """Advisor that applies the metacoupling framework to user research.
365
+
366
+ Parameters
367
+ ----------
368
+ llm_client:
369
+ Any object satisfying the :class:`~metacouplingllm.llm.client.LLMClient`
370
+ protocol (e.g., :class:`OpenAIAdapter`, :class:`AnthropicAdapter`,
371
+ or a custom implementation).
372
+ temperature:
373
+ Sampling temperature for the LLM.
374
+ max_tokens:
375
+ Maximum response tokens. ``None`` uses the model default.
376
+ max_examples:
377
+ Number of curated examples to inject into the system prompt.
378
+ verbose:
379
+ If ``True``, print diagnostic information during execution.
380
+ recommend_papers:
381
+ If ``True``, append literature recommendations to ``formatted``.
382
+ max_recommendations:
383
+ Maximum number of paper recommendations to include.
384
+ auto_map:
385
+ If ``True``, automatically generate a map from the analysis.
386
+
387
+ - **Subnational research** (e.g., "Michigan's pork exports"):
388
+ generates an ADM1 map where the focal region is intracoupling,
389
+ adjacent ADM1 regions are pericoupling, and distant regions are
390
+ telecoupling. Uses *adm1_shapefile* when provided, otherwise
391
+ tries the cached / auto-downloaded ADM1 basemap.
392
+ - **National research** (e.g., "USA's pork exports"):
393
+ generates a country-level world map. Requires geopandas.
394
+
395
+ The map is stored on ``AnalysisResult.map``. If visualisation
396
+ dependencies are not installed the field is silently ``None``.
397
+ adm1_shapefile:
398
+ Path to a World Bank Admin 1 GeoPackage (``.gpkg``) or shapefile.
399
+ Optional override for subnational (ADM1) maps. When omitted,
400
+ metacoupling first looks for a cached/local file and then tries
401
+ the hosted ADM1 download mirror.
402
+ adm0_shapefile:
403
+ Path to a World Bank Admin 0 GeoPackage for country-level maps.
404
+ When omitted, metacoupling first looks for a local
405
+ ``Admin 0_all_layers.gpkg`` before trying the hosted mirror and
406
+ official World Bank fallback download.
407
+ rag_papers_dir:
408
+ Path to a custom directory containing markdown papers. Use this
409
+ only when you want to override the bundled RAG corpus with your
410
+ own files.
411
+ rag_corpus:
412
+ Name of a built-in RAG corpus. Recommended:
413
+ ``"journal_articles_2025"``. This uses the bundled rebuilt
414
+ corpus of metacoupling/telecoupling journal-paper records with
415
+ BGE-base embeddings. Leave both ``rag_corpus`` and
416
+ ``rag_papers_dir`` unset to disable RAG.
417
+ rag_top_k:
418
+ Maximum number of evidence passages to retrieve per analysis.
419
+ Default ``8``. In ``rag_mode="pre_retrieval"`` this also bounds
420
+ the number of ``<retrieved_literature>`` passages injected into
421
+ the user message and therefore the number of valid ``[1]..[N]``
422
+ citation labels for the LLM.
423
+ rag_mode:
424
+ Selects how RAG integrates with the LLM call. Two options:
425
+
426
+ - ``"pre_retrieval"`` (default): retrieve corpus passages
427
+ **before** the LLM runs, embed them in the user message as
428
+ a labeled ``<retrieved_literature>`` XML block, and instruct
429
+ the LLM (via a citation-rules layer in the system prompt) to
430
+ cite them inline as ``[1]..[N]``. After the LLM responds,
431
+ out-of-range citation tokens are stripped (with a logged
432
+ warning) by
433
+ :func:`~metacouplingllm.knowledge.citations.sanitize_citations`,
434
+ and the same evidence block as the post-hoc path is appended
435
+ to the formatted output. This is the recommended mode and
436
+ gives the LLM literature to ground its analysis in rather
437
+ than relying purely on training memory.
438
+ - ``"post_hoc"`` (alternative): the LLM generates from training
439
+ memory, then a keyword-overlap pass annotates ``[N]``
440
+ citations onto sentences that match retrieved passages. Use
441
+ this mode when downstream tooling expects citations to be
442
+ assigned by post-hoc keyword matching rather than inline by
443
+ the LLM.
444
+
445
+ On ``refine()`` in pre_retrieval mode, the merged retrieval
446
+ query is the **original** research description (anchored at
447
+ ``analyze()`` time and never overwritten) plus the new
448
+ refinement text, in a labeled structure — see
449
+ :meth:`refine` for details.
450
+ rag_max_chunks_per_paper:
451
+ Maximum number of chunks from the same paper that may appear
452
+ in a single retrieval result. Default ``3``. Set to ``1`` for
453
+ the legacy one-chunk-per-paper behavior, or raise to ``5``
454
+ (or higher) when a single paper is expected to be strongly
455
+ relevant and its key evidence is spread across many sections
456
+ — for example, a systematic framework paper whose results
457
+ section has been split by the chunker into several distinct
458
+ sub-topic chunks (inbound vs outbound, aggregate vs bilateral).
459
+ Higher values improve coverage at the cost of reduced
460
+ paper-level diversity in the retrieved set.
461
+ rag_structured_extraction:
462
+ If ``True`` and ``rag_mode="pre_retrieval"`` with a RAG engine
463
+ configured, runs a second LLM pass over the already-retrieved
464
+ passages to extract systems and flows that may have been missed
465
+ or under-specified in the free-form analysis. Covers all three
466
+ system roles (sending, receiving, spillover) and produces a
467
+ schema-validated list of supplementary flows. Results are
468
+ rendered as a clearly labelled ``SUPPLEMENTARY STRUCTURED
469
+ EXTRACTION`` block in ``AnalysisResult.formatted`` and exposed
470
+ programmatically as ``AnalysisResult.structured_supplement``
471
+ (a dict keyed by ``additional_sending_mentions``,
472
+ ``additional_receiving_mentions``,
473
+ ``additional_spillover_mentions``,
474
+ ``sending_subsystem_fills``, ``receiving_subsystem_fills``,
475
+ ``spillover_subsystem_fills``, and ``supplementary_flows``).
476
+ The main analysis body is not modified — the supplement is
477
+ purely additive so the reader can always tell LLM-authored
478
+ content apart from RAG-extracted content. Defaults to
479
+ ``False`` because this adds one extra LLM call per
480
+ ``analyze()`` / ``refine()`` turn.
481
+ web_structured_extraction:
482
+ If ``True`` and ``web_search=True``, runs a second LLM pass over the
483
+ web snippets to extract validated map-ready countries and flows.
484
+ These signals are used as conservative hints for map generation and
485
+ are exposed on ``AnalysisResult.web_map_signals``. Recommended when
486
+ ``web_search=True`` and ``auto_map=True`` are both enabled.
487
+
488
+ Example
489
+ -------
490
+ >>> from openai import OpenAI
491
+ >>> from metacouplingllm import (
492
+ ... JOURNAL_ARTICLES_2025,
493
+ ... MetacouplingAssistant,
494
+ ... OpenAIAdapter,
495
+ ... )
496
+ >>> advisor = MetacouplingAssistant(
497
+ ... OpenAIAdapter(OpenAI(), model="gpt-4o"),
498
+ ... rag_corpus=JOURNAL_ARTICLES_2025,
499
+ ... web_search=True,
500
+ ... web_structured_extraction=True,
501
+ ... auto_map=True,
502
+ ... )
503
+ >>> result = advisor.analyze("My study examines Michigan's pork exports...")
504
+ >>> print(result.formatted)
505
+ >>> if result.map:
506
+ ... result.map.savefig("map.png", dpi=150)
507
+ """
508
+
509
+ def __init__(
510
+ self,
511
+ llm_client: LLMClient,
512
+ temperature: float = 0.7,
513
+ max_tokens: int | None = None,
514
+ max_examples: int = 2,
515
+ verbose: bool = False,
516
+ recommend_papers: bool = False,
517
+ max_recommendations: int = 5,
518
+ auto_map: bool = False,
519
+ adm1_shapefile: str | Path | None = None,
520
+ adm0_shapefile: str | Path | None = None,
521
+ rag_papers_dir: str | Path | None = None,
522
+ rag_corpus: str | None = None,
523
+ rag_top_k: int = 8,
524
+ rag_backend: str = "auto",
525
+ rag_mode: str = "pre_retrieval",
526
+ rag_max_chunks_per_paper: int = 3,
527
+ rag_structured_extraction: bool = False,
528
+ web_search: bool = False,
529
+ web_search_max_results: int = 5,
530
+ web_structured_extraction: bool = False,
531
+ web_structured_min_confidence: float = 0.7,
532
+ web_structured_max_targets: int = 6,
533
+ rag_min_score: float | None = None,
534
+ coupling_analysis: bool = True,
535
+ ) -> None:
536
+ if rag_mode not in _VALID_RAG_MODES:
537
+ raise ValueError(
538
+ f"rag_mode must be one of {sorted(_VALID_RAG_MODES)}, "
539
+ f"got {rag_mode!r}"
540
+ )
541
+ self._client = llm_client
542
+ self._temperature = temperature
543
+ self._max_tokens = max_tokens
544
+ self._verbose = verbose
545
+ self._recommend_papers = recommend_papers
546
+ self._max_recommendations = max_recommendations
547
+ self._auto_map = auto_map
548
+ self._adm1_shapefile = adm1_shapefile
549
+ self._adm0_shapefile = adm0_shapefile
550
+ self._rag_top_k = rag_top_k
551
+ self._rag_backend = rag_backend
552
+ self._rag_mode = rag_mode
553
+ self._rag_max_chunks_per_paper = max(1, int(rag_max_chunks_per_paper))
554
+ self._rag_structured_extraction = rag_structured_extraction
555
+ self._web_search = web_search
556
+ self._web_search_max_results = web_search_max_results
557
+ # Auto-enable structured web extraction when both web_search and
558
+ # auto_map are on — the map needs structured data to work well.
559
+ if web_structured_extraction is False and web_search and auto_map:
560
+ web_structured_extraction = True
561
+ self._web_structured_extraction = web_structured_extraction
562
+ self._web_structured_min_confidence = web_structured_min_confidence
563
+ self._web_structured_max_targets = web_structured_max_targets
564
+ self._rag_min_score = rag_min_score
565
+ self._last_web_results: list[dict[str, str]] = []
566
+ self._last_web_map_signals: dict[str, object] | None = None
567
+ self._last_map_notice: str | None = None
568
+ self._prompt_builder = PromptBuilder(max_examples=max_examples)
569
+ self._formatter = AnalysisFormatter()
570
+
571
+ # Pre-retrieval RAG state. Set in analyze() / refine().
572
+ # `_original_query` is anchored to the very first analyze() call
573
+ # and survives across refine()s so the merged retrieval query
574
+ # always preserves the original topic. `_last_rag_hits` is the
575
+ # most recent list of retrieved passages — used both for prompt
576
+ # injection and for the post-LLM evidence block.
577
+ self._last_rag_hits: list[RetrievalResult] | None = None
578
+ self._original_query: str | None = None
579
+
580
+ rag_source = self._resolve_rag_source(
581
+ rag_papers_dir=rag_papers_dir,
582
+ rag_corpus=rag_corpus,
583
+ )
584
+
585
+ # Initialise RAG engine if a custom directory or built-in corpus
586
+ # is selected.
587
+ self._rag_engine = None
588
+ if rag_source is not None:
589
+ try:
590
+ from metacouplingllm.knowledge.rag import RAGEngine
591
+
592
+ self._rag_engine = RAGEngine(
593
+ papers_dir=rag_source,
594
+ verbose=verbose,
595
+ backend=rag_backend,
596
+ )
597
+ self._rag_engine.load()
598
+
599
+ if self._rag_engine.total_chunks == 0:
600
+ print(
601
+ "[MetacouplingAssistant] WARNING: RAG engine loaded "
602
+ "but has 0 chunks. Evidence retrieval will return "
603
+ "no results."
604
+ )
605
+ except Exception as exc:
606
+ print(
607
+ f"[MetacouplingAssistant] WARNING: RAG initialisation "
608
+ f"failed: {exc}"
609
+ )
610
+ self._rag_engine = None
611
+ self._history: list[Message] = []
612
+ self._turn: int = 0
613
+
614
+ # RAG-only mode (coupling_analysis=False). Maintains a separate
615
+ # multi-turn conversation history; framework `_history` is left
616
+ # untouched so the two modes don't interfere.
617
+ self._coupling_analysis = bool(coupling_analysis)
618
+ self._rag_history: list[Message] = []
619
+ self._rag_turn: int = 0
620
+ if not self._coupling_analysis:
621
+ self._warn_disabled_framework_options(
622
+ auto_map=auto_map,
623
+ recommend_papers=recommend_papers,
624
+ rag_structured_extraction=rag_structured_extraction,
625
+ web_structured_extraction=web_structured_extraction,
626
+ )
627
+
628
+ @staticmethod
629
+ def _warn_disabled_framework_options(
630
+ *,
631
+ auto_map: bool,
632
+ recommend_papers: bool,
633
+ rag_structured_extraction: bool,
634
+ web_structured_extraction: bool,
635
+ ) -> None:
636
+ """Print a notice for framework-only options ignored in RAG mode."""
637
+ disabled = []
638
+ if auto_map:
639
+ disabled.append("auto_map")
640
+ if recommend_papers:
641
+ disabled.append("recommend_papers")
642
+ if rag_structured_extraction:
643
+ disabled.append("rag_structured_extraction")
644
+ if web_structured_extraction:
645
+ disabled.append("web_structured_extraction")
646
+ if disabled:
647
+ print(
648
+ "[MetacouplingAssistant] coupling_analysis=False — the "
649
+ "following framework-only options are ignored: "
650
+ + ", ".join(disabled)
651
+ )
652
+
653
+ # ------------------------------------------------------------------
654
+ # Public API
655
+ # ------------------------------------------------------------------
656
+
657
+ @staticmethod
658
+ def _resolve_rag_source(
659
+ *,
660
+ rag_papers_dir: str | Path | None,
661
+ rag_corpus: str | None,
662
+ ) -> str | Path | None:
663
+ """Resolve user-facing RAG corpus options to a RAGEngine source."""
664
+ if rag_papers_dir is not None and rag_corpus is not None:
665
+ raise ValueError(
666
+ "Pass either rag_corpus or rag_papers_dir, not both. "
667
+ "Use rag_corpus='journal_articles_2025' for the bundled "
668
+ "corpus, or rag_papers_dir=... for a custom folder."
669
+ )
670
+ if rag_papers_dir is not None:
671
+ return rag_papers_dir
672
+ if rag_corpus is None:
673
+ return None
674
+
675
+ normalized = str(rag_corpus).strip().lower().replace("-", "_")
676
+ if normalized in _BUILTIN_RAG_CORPUS_ALIASES:
677
+ return _BUILTIN_RAG_SENTINEL
678
+ raise ValueError(
679
+ f"Unknown rag_corpus={rag_corpus!r}. Supported built-in "
680
+ f"corpora: {sorted(_BUILTIN_RAG_CORPUS_ALIASES)}"
681
+ )
682
+
683
+ @property
684
+ def conversation_turns(self) -> int:
685
+ """Number of completed RAG-mode turns in the current session.
686
+
687
+ Returns 0 in framework mode (``coupling_analysis=True``) or
688
+ before the first :meth:`analyze` call in RAG mode. Reset by
689
+ :meth:`clear_history`.
690
+ """
691
+ return self._rag_turn if not self._coupling_analysis else 0
692
+
693
+ def clear_history(self) -> None:
694
+ """Reset all conversation state.
695
+
696
+ Clears both the framework-mode history (used by
697
+ :meth:`refine`) and the RAG-mode conversation history. Safe to
698
+ call from either mode.
699
+ """
700
+ self._history.clear()
701
+ self._turn = 0
702
+ self._last_rag_hits = None
703
+ self._original_query = None
704
+ self._rag_history.clear()
705
+ self._rag_turn = 0
706
+
707
+ def _analyze_rag_only(self, query: str) -> RAGResult:
708
+ """RAG-only Q&A path used when ``coupling_analysis=False``.
709
+
710
+ Multi-turn: appends to ``self._rag_history`` so follow-up
711
+ questions can use prior turns as context.
712
+ """
713
+ if not query or not query.strip():
714
+ raise ValueError("query must be a non-empty string")
715
+
716
+ # Optional web search (per-turn). The map-signal extraction is
717
+ # framework-only, so we deliberately skip extract_web_map_signals.
718
+ web_context: str | None = None
719
+ web_sources: list[dict[str, str]] | None = None
720
+ if self._web_search:
721
+ try:
722
+ from metacouplingllm.knowledge.websearch import (
723
+ AnthropicWebSearchBackend,
724
+ GeminiWebSearchBackend,
725
+ GrokWebSearchBackend,
726
+ OpenAIWebSearchBackend,
727
+ format_web_context,
728
+ search_web,
729
+ )
730
+
731
+ print("[MetacouplingAssistant] (RAG mode) Searching the web...")
732
+ web_backend = None
733
+ if isinstance(self._client, OpenAIAdapter):
734
+ web_backend = OpenAIWebSearchBackend(
735
+ client=self._client.raw_client,
736
+ model=self._client.model,
737
+ reasoning="default",
738
+ )
739
+ elif isinstance(self._client, AnthropicAdapter):
740
+ web_backend = AnthropicWebSearchBackend(
741
+ client=self._client.raw_client,
742
+ model=self._client.model,
743
+ )
744
+ elif isinstance(self._client, GeminiAdapter):
745
+ web_backend = GeminiWebSearchBackend(
746
+ client=self._client.raw_client,
747
+ model=self._client.model,
748
+ )
749
+ elif isinstance(self._client, GrokAdapter):
750
+ web_backend = GrokWebSearchBackend(
751
+ client=self._client.raw_client,
752
+ model=self._client.model,
753
+ )
754
+ web_search_metadata: dict[str, object] = {}
755
+ web_sources = search_web(
756
+ query.strip(),
757
+ max_results=self._web_search_max_results,
758
+ backend=web_backend,
759
+ metadata=web_search_metadata,
760
+ )
761
+ backend_used = web_search_metadata.get("backend_used")
762
+ if backend_used:
763
+ fallback_from = web_search_metadata.get("fallback_from")
764
+ if fallback_from:
765
+ print(
766
+ f"[MetacouplingAssistant] (RAG mode) Web search via "
767
+ f"{backend_used} — fallback after "
768
+ f"{fallback_from}"
769
+ )
770
+ else:
771
+ print(
772
+ f"[MetacouplingAssistant] (RAG mode) Web search via "
773
+ f"{backend_used}"
774
+ )
775
+ if web_sources:
776
+ web_context = format_web_context(web_sources)
777
+ print(
778
+ f"[MetacouplingAssistant] (RAG mode) Web search returned "
779
+ f"{len(web_sources)} results."
780
+ )
781
+ else:
782
+ print(
783
+ "[MetacouplingAssistant] (RAG mode) Web search returned "
784
+ "0 results."
785
+ )
786
+ except Exception as exc:
787
+ print(
788
+ "[MetacouplingAssistant] (RAG mode) Web search failed: "
789
+ f"{type(exc).__name__}: {exc}"
790
+ )
791
+ web_sources = None
792
+ web_context = None
793
+
794
+ # RAG retrieval — fresh on every turn so follow-ups get the
795
+ # most relevant passages for *their* specific question.
796
+ passages: list[RetrievalResult] = []
797
+ if self._rag_engine is not None:
798
+ try:
799
+ passages = list(
800
+ self._rag_engine.retrieve(
801
+ query,
802
+ top_k=self._rag_top_k,
803
+ min_score=self._rag_min_score,
804
+ max_chunks_per_paper=self._rag_max_chunks_per_paper,
805
+ )
806
+ )
807
+ if self._verbose:
808
+ print(
809
+ f"[MetacouplingAssistant] (RAG mode) Retrieved "
810
+ f"{len(passages)} passages."
811
+ )
812
+ except Exception as exc:
813
+ logger.warning(
814
+ "RAG retrieval failed in _analyze_rag_only(): %s. "
815
+ "Continuing with empty literature block.",
816
+ exc,
817
+ )
818
+ passages = []
819
+
820
+ # Build this turn's user message: query + literature block
821
+ # (+ optional web block).
822
+ from metacouplingllm.knowledge.rag import format_evidence
823
+
824
+ literature_block = (
825
+ format_evidence(passages, anchor_text=query)
826
+ if passages
827
+ else "No literature passages were retrieved for this query."
828
+ )
829
+ user_parts = [
830
+ query.strip(),
831
+ "<retrieved_literature>",
832
+ literature_block,
833
+ "</retrieved_literature>",
834
+ ]
835
+ if web_context:
836
+ user_parts.extend([
837
+ "",
838
+ "<web_search_results>",
839
+ web_context,
840
+ "</web_search_results>",
841
+ ])
842
+ user_msg = "\n\n".join(user_parts)
843
+
844
+ # On turn 1, seed the conversation with the system prompt.
845
+ if not self._rag_history:
846
+ self._rag_history.append(
847
+ Message(role="system", content=_RAG_ONLY_SYSTEM_PROMPT)
848
+ )
849
+ self._rag_history.append(Message(role="user", content=user_msg))
850
+
851
+ # Token-budget heuristic: warn (verbose only) when the projected
852
+ # message size approaches typical context windows. ~4 chars/token
853
+ # rule of thumb; warn at ~120k tokens (480k chars).
854
+ if self._verbose:
855
+ total_chars = sum(len(m.content) for m in self._rag_history)
856
+ if total_chars > 480_000:
857
+ print(
858
+ "[MetacouplingAssistant] (RAG mode) conversation is "
859
+ f"~{total_chars // 4:,} tokens — consider calling "
860
+ "advisor.clear_history() before the next turn."
861
+ )
862
+
863
+ # LLM call (uses full RAG history, NOT framework `_history`).
864
+ response = self._client.chat(
865
+ messages=list(self._rag_history),
866
+ temperature=self._temperature,
867
+ max_tokens=self._max_tokens,
868
+ )
869
+
870
+ # Sanitize citations: strip any [N] where N > number of passages
871
+ # for THIS turn.
872
+ sanitized, dropped = sanitize_citations(
873
+ response.content, n_valid=len(passages)
874
+ )
875
+ if dropped and self._verbose:
876
+ print(
877
+ "[MetacouplingAssistant] (RAG mode) Sanitized "
878
+ f"{len(dropped)} hallucinated citation(s): "
879
+ f"{sorted(dropped)}"
880
+ )
881
+
882
+ # Append the (sanitized) assistant response to history so the
883
+ # next turn sees it as context.
884
+ self._rag_history.append(
885
+ Message(role="assistant", content=sanitized)
886
+ )
887
+ self._rag_turn += 1
888
+
889
+ # Extract cited references from the sanitized answer, dedup'd
890
+ # by paper key and ordered by first appearance.
891
+ references = self._build_references_from_citations(
892
+ sanitized, passages
893
+ )
894
+
895
+ retrieval_backend = "embeddings"
896
+ if self._rag_engine is not None:
897
+ try:
898
+ retrieval_backend = str(
899
+ getattr(self._rag_engine, "backend", "embeddings")
900
+ )
901
+ except Exception:
902
+ retrieval_backend = "embeddings"
903
+
904
+ return RAGResult(
905
+ answer=sanitized,
906
+ references=references,
907
+ retrieved_passages=passages,
908
+ web_sources=web_sources,
909
+ turn_number=self._rag_turn,
910
+ usage=response.usage,
911
+ raw=response.content,
912
+ retrieval_backend=retrieval_backend,
913
+ )
914
+
915
+ @staticmethod
916
+ def _build_references_from_citations(
917
+ answer: str,
918
+ passages: list[RetrievalResult],
919
+ ) -> list[Paper]:
920
+ """Return cited papers in order of first appearance, dedup'd by key."""
921
+ if not passages:
922
+ return []
923
+ seen: set[str] = set()
924
+ out: list[Paper] = []
925
+ for match in _CITATION_TOKEN_RE.finditer(answer):
926
+ try:
927
+ idx = int(match.group(1))
928
+ except ValueError:
929
+ continue
930
+ if not (1 <= idx <= len(passages)):
931
+ continue
932
+ chunk = passages[idx - 1].chunk
933
+ if chunk.paper_key in seen:
934
+ continue
935
+ seen.add(chunk.paper_key)
936
+ try:
937
+ year_int = int(chunk.year)
938
+ except (TypeError, ValueError):
939
+ year_int = 0
940
+ out.append(
941
+ Paper(
942
+ key=chunk.paper_key,
943
+ title=chunk.paper_title,
944
+ authors=chunk.authors,
945
+ year=year_int,
946
+ )
947
+ )
948
+ return out
949
+
950
+ def analyze(
951
+ self, research_description: str
952
+ ) -> AnalysisResult | RAGResult:
953
+ """Run an analysis on a research description.
954
+
955
+ Behaviour depends on the ``coupling_analysis`` flag set at
956
+ construction:
957
+
958
+ - ``coupling_analysis=True`` (default): runs the full framework
959
+ pipeline and returns an :class:`AnalysisResult`. Resets any
960
+ existing framework conversation history and starts fresh.
961
+ - ``coupling_analysis=False``: runs a RAG-only literature Q&A
962
+ and returns a :class:`RAGResult`. Multi-turn — each call
963
+ appends to the running RAG conversation; call
964
+ :meth:`clear_history` to reset.
965
+
966
+ Parameters
967
+ ----------
968
+ research_description:
969
+ In framework mode, the researcher's description of their
970
+ study, research idea, or draft. In RAG mode, a free-form
971
+ literature question (or follow-up).
972
+
973
+ Returns
974
+ -------
975
+ :class:`AnalysisResult` (framework mode) or :class:`RAGResult`
976
+ (RAG-only mode).
977
+
978
+ Notes
979
+ -----
980
+ Framework mode: when ``rag_mode="pre_retrieval"`` (the default)
981
+ and a RAG engine is configured, this method retrieves the top
982
+ ``rag_top_k`` passages from the corpus *before* calling the LLM
983
+ and embeds them in the user message as a
984
+ ``<retrieved_literature>`` block, so the LLM can cite them
985
+ inline as ``[1]..[N]``. The retrieved hits and the original
986
+ research description are stored on the instance for later reuse
987
+ by :meth:`refine`.
988
+ """
989
+ if not self._coupling_analysis:
990
+ return self._analyze_rag_only(research_description)
991
+
992
+ # Reset for a new framework analysis
993
+ self._history.clear()
994
+ self._turn = 0
995
+ # Reset pre-retrieval state — even if RAG is disabled or fails,
996
+ # leaving stale hits from a previous session would be wrong.
997
+ self._last_rag_hits = None
998
+ self._original_query = research_description
999
+
1000
+ # Optional: pre-search web context
1001
+ web_context: str | None = None
1002
+ self._last_web_results: list[dict[str, str]] = []
1003
+ self._last_web_map_signals = None
1004
+ if self._web_search:
1005
+ try:
1006
+ from metacouplingllm.knowledge.websearch import (
1007
+ AnthropicWebSearchBackend,
1008
+ GeminiWebSearchBackend,
1009
+ GrokWebSearchBackend,
1010
+ OpenAIWebSearchBackend,
1011
+ extract_web_map_signals,
1012
+ format_web_context,
1013
+ format_web_map_signals_context,
1014
+ search_web,
1015
+ )
1016
+
1017
+ print("[MetacouplingAssistant] Searching the web...")
1018
+ # Auto-wire the native web-search tool for the matching
1019
+ # provider. If the backend fails or returns no results,
1020
+ # search_web() falls back to the DuckDuckGo cascade.
1021
+ web_backend = None
1022
+ if isinstance(self._client, OpenAIAdapter):
1023
+ web_backend = OpenAIWebSearchBackend(
1024
+ client=self._client.raw_client,
1025
+ model=self._client.model,
1026
+ reasoning="default",
1027
+ )
1028
+ elif isinstance(self._client, AnthropicAdapter):
1029
+ web_backend = AnthropicWebSearchBackend(
1030
+ client=self._client.raw_client,
1031
+ model=self._client.model,
1032
+ )
1033
+ elif isinstance(self._client, GeminiAdapter):
1034
+ web_backend = GeminiWebSearchBackend(
1035
+ client=self._client.raw_client,
1036
+ model=self._client.model,
1037
+ )
1038
+ elif isinstance(self._client, GrokAdapter):
1039
+ web_backend = GrokWebSearchBackend(
1040
+ client=self._client.raw_client,
1041
+ model=self._client.model,
1042
+ )
1043
+ web_search_metadata: dict[str, object] = {}
1044
+ self._last_web_results = search_web(
1045
+ research_description.strip(),
1046
+ max_results=self._web_search_max_results,
1047
+ backend=web_backend,
1048
+ metadata=web_search_metadata,
1049
+ )
1050
+ backend_used = web_search_metadata.get("backend_used")
1051
+ if backend_used:
1052
+ fallback_from = web_search_metadata.get("fallback_from")
1053
+ if fallback_from:
1054
+ print(
1055
+ f"[MetacouplingAssistant] Web search via "
1056
+ f"{backend_used} \u2014 fallback after "
1057
+ f"{fallback_from}"
1058
+ )
1059
+ else:
1060
+ print(
1061
+ f"[MetacouplingAssistant] Web search via "
1062
+ f"{backend_used}"
1063
+ )
1064
+ if self._last_web_results:
1065
+ web_context = format_web_context(self._last_web_results)
1066
+ if self._web_structured_extraction:
1067
+ try:
1068
+ self._last_web_map_signals = (
1069
+ extract_web_map_signals(
1070
+ research_description.strip(),
1071
+ self._last_web_results,
1072
+ self._client,
1073
+ min_confidence=(
1074
+ self._web_structured_min_confidence
1075
+ ),
1076
+ max_targets=self._web_structured_max_targets,
1077
+ )
1078
+ )
1079
+ except Exception as exc:
1080
+ print(
1081
+ "[MetacouplingAssistant] Structured web "
1082
+ f"extraction failed: {exc}"
1083
+ )
1084
+ self._last_web_map_signals = None
1085
+ if self._last_web_map_signals:
1086
+ structured_context = (
1087
+ format_web_map_signals_context(
1088
+ self._last_web_map_signals,
1089
+ )
1090
+ )
1091
+ if structured_context:
1092
+ web_context += "\n\n" + structured_context
1093
+ if self._verbose:
1094
+ structured_flows = self._last_web_map_signals.get(
1095
+ "flows", []
1096
+ )
1097
+ structured_receiving = (
1098
+ self._last_web_map_signals.get(
1099
+ "receiving_systems", []
1100
+ )
1101
+ )
1102
+ print(
1103
+ "[MetacouplingAssistant] Structured web "
1104
+ "extraction accepted "
1105
+ f"{len(structured_receiving)} receiving "
1106
+ f"systems and {len(structured_flows)} "
1107
+ "flows."
1108
+ )
1109
+ elif self._verbose:
1110
+ print(
1111
+ "[MetacouplingAssistant] Structured web "
1112
+ "extraction found no validated map signals."
1113
+ )
1114
+ print(
1115
+ f"[MetacouplingAssistant] Web search returned "
1116
+ f"{len(self._last_web_results)} results."
1117
+ )
1118
+ else:
1119
+ print(
1120
+ "[MetacouplingAssistant] Web search returned 0 results."
1121
+ )
1122
+ except Exception as exc:
1123
+ if self._verbose:
1124
+ print(f"[MetacouplingAssistant] Web search failed: {exc}")
1125
+
1126
+ # Pre-retrieval RAG: fetch corpus passages BEFORE calling the LLM
1127
+ # so we can inject them into the user message and the LLM can
1128
+ # cite them inline. Disabled in post_hoc mode (which retrieves
1129
+ # and annotates AFTER the LLM responds) and when no RAG engine
1130
+ # is configured.
1131
+ if self._rag_mode == "pre_retrieval" and self._rag_engine is not None:
1132
+ try:
1133
+ results = self._rag_engine.retrieve(
1134
+ research_description,
1135
+ top_k=self._rag_top_k,
1136
+ min_score=self._rag_min_score,
1137
+ max_chunks_per_paper=self._rag_max_chunks_per_paper,
1138
+ )
1139
+ self._last_rag_hits = list(results)
1140
+ if self._verbose:
1141
+ print(
1142
+ f"[MetacouplingAssistant] Pre-retrieval RAG: "
1143
+ f"{len(self._last_rag_hits)} passages."
1144
+ )
1145
+ except Exception as exc:
1146
+ logger.warning(
1147
+ "Pre-retrieval RAG failed in analyze(): %s. "
1148
+ "Continuing with empty literature block.",
1149
+ exc,
1150
+ )
1151
+ self._last_rag_hits = []
1152
+
1153
+ # Build the system prompt (examples selected based on context).
1154
+ # The citation rules layer is only injected when pre_retrieval
1155
+ # mode is active so post_hoc users see no behavioral change.
1156
+ system_prompt = self._prompt_builder.build_system_prompt(
1157
+ research_context=research_description,
1158
+ web_context=web_context,
1159
+ include_citation_rules=(self._rag_mode == "pre_retrieval"),
1160
+ )
1161
+ self._history.append(Message(role="system", content=system_prompt))
1162
+
1163
+ # Build the user message. In pre_retrieval mode the literature
1164
+ # passages are prepended as a <retrieved_literature> XML block.
1165
+ # In post_hoc mode (or when RAG is disabled) literature_passages
1166
+ # is None and the original behavior is preserved exactly.
1167
+ literature_for_msg = (
1168
+ self._last_rag_hits
1169
+ if self._rag_mode == "pre_retrieval"
1170
+ and self._rag_engine is not None
1171
+ else None
1172
+ )
1173
+ user_msg = self._prompt_builder.build_initial_message(
1174
+ research_description,
1175
+ literature_passages=literature_for_msg,
1176
+ )
1177
+ self._history.append(Message(role="user", content=user_msg))
1178
+
1179
+ if self._verbose:
1180
+ print(f"[MetacouplingAssistant] System prompt: {len(system_prompt)} chars")
1181
+ print(f"[MetacouplingAssistant] Sending initial analysis request...")
1182
+
1183
+ # Call the LLM
1184
+ response = self._call_llm()
1185
+
1186
+ # Record assistant response in history
1187
+ self._history.append(Message(role="assistant", content=response.content))
1188
+ self._turn += 1
1189
+
1190
+ return self._build_result(response)
1191
+
1192
+ def refine(
1193
+ self,
1194
+ additional_info: str,
1195
+ focus_component: str | None = None,
1196
+ ) -> AnalysisResult:
1197
+ """Refine the analysis with additional information or focus.
1198
+
1199
+ Must be called after :meth:`analyze`.
1200
+
1201
+ Parameters
1202
+ ----------
1203
+ additional_info:
1204
+ Follow-up questions, corrections, or additional context.
1205
+ focus_component:
1206
+ Optional component to elaborate on (e.g., ``"systems"``,
1207
+ ``"flows"``, ``"agents"``, ``"causes"``, ``"effects"``).
1208
+
1209
+ Returns
1210
+ -------
1211
+ An :class:`AnalysisResult` with the refined analysis.
1212
+
1213
+ Raises
1214
+ ------
1215
+ RuntimeError
1216
+ If called before :meth:`analyze`.
1217
+
1218
+ Notes
1219
+ -----
1220
+ When ``rag_mode="pre_retrieval"``, this method always re-runs
1221
+ retrieval using a **labeled merged query** that combines the
1222
+ original research description (anchored at :meth:`analyze` time
1223
+ and never overwritten) with the new ``additional_info``::
1224
+
1225
+ Original research question:
1226
+ <original research_description>
1227
+
1228
+ Refinement request:
1229
+ <additional_info>
1230
+
1231
+ The labeled structure gives the embedding model clearer
1232
+ structural cues than raw concatenation and prevents the
1233
+ refinement text from "drifting" the retrieval away from the
1234
+ original topic. Fresh hits replace ``self._last_rag_hits`` and
1235
+ are injected into the new user message.
1236
+
1237
+ .. warning::
1238
+
1239
+ **Stale citation tokens across turns (Phase 1 limitation).**
1240
+ Because the conversation history persists across turns,
1241
+ ``[1]`` in turn 2 may refer to a different paper than
1242
+ ``[1]`` in turn 1 (because re-retrieval can return a
1243
+ different ranking). The system prompt instructs the LLM to
1244
+ cite only from the most recent ``<retrieved_literature>``
1245
+ block, and the post-LLM sanitizer strips out-of-range
1246
+ tokens, but neither can detect a token that is in-range yet
1247
+ semantically wrong. Phase 2 will address this with
1248
+ turn-scoped citation markers. See ``CHANGELOG.md`` for the
1249
+ current workaround.
1250
+ """
1251
+ if self._turn == 0:
1252
+ raise RuntimeError(
1253
+ "Cannot refine before an initial analysis. "
1254
+ "Call `analyze()` first."
1255
+ )
1256
+
1257
+ # Pre-retrieval RAG on refinement: re-retrieve with the merged
1258
+ # query so the LLM gets fresh evidence for the refined ask.
1259
+ # TODO(phase2): citation labels [1]..[N] are turn-local. After
1260
+ # this re-retrieval the same number may now refer to a
1261
+ # different paper than it did on a previous turn. The system
1262
+ # prompt rule + sanitizer mitigate but cannot fully eliminate
1263
+ # the "stale [1]" failure mode. Plan: prefix labels with a
1264
+ # turn marker (e.g. [T2:1]) or strip prior-turn citations from
1265
+ # history before each refine.
1266
+ if (
1267
+ self._rag_mode == "pre_retrieval"
1268
+ and self._rag_engine is not None
1269
+ and self._original_query is not None
1270
+ ):
1271
+ merged_query = (
1272
+ f"Original research question:\n{self._original_query}\n\n"
1273
+ f"Refinement request:\n{additional_info}"
1274
+ )
1275
+ try:
1276
+ results = self._rag_engine.retrieve(
1277
+ merged_query,
1278
+ top_k=self._rag_top_k,
1279
+ min_score=self._rag_min_score,
1280
+ max_chunks_per_paper=self._rag_max_chunks_per_paper,
1281
+ )
1282
+ self._last_rag_hits = list(results)
1283
+ if self._verbose:
1284
+ print(
1285
+ f"[MetacouplingAssistant] Refine pre-retrieval: "
1286
+ f"{len(self._last_rag_hits)} passages."
1287
+ )
1288
+ except Exception as exc:
1289
+ logger.warning(
1290
+ "Pre-retrieval RAG failed in refine(): %s. "
1291
+ "Continuing with empty literature block.",
1292
+ exc,
1293
+ )
1294
+ self._last_rag_hits = []
1295
+
1296
+ literature_for_msg = (
1297
+ self._last_rag_hits
1298
+ if self._rag_mode == "pre_retrieval"
1299
+ and self._rag_engine is not None
1300
+ else None
1301
+ )
1302
+ user_msg = self._prompt_builder.build_refinement_message(
1303
+ additional_info,
1304
+ focus_component=focus_component,
1305
+ literature_passages=literature_for_msg,
1306
+ )
1307
+ self._history.append(Message(role="user", content=user_msg))
1308
+
1309
+ if self._verbose:
1310
+ print(f"[MetacouplingAssistant] Refining (turn {self._turn + 1})...")
1311
+
1312
+ response = self._call_llm()
1313
+ self._history.append(Message(role="assistant", content=response.content))
1314
+ self._turn += 1
1315
+
1316
+ return self._build_result(response)
1317
+
1318
+ def reset(self) -> None:
1319
+ """Clear conversation history and pre-retrieval state."""
1320
+ self._history.clear()
1321
+ self._turn = 0
1322
+ self._last_rag_hits = None
1323
+ self._original_query = None
1324
+ if self._verbose:
1325
+ print("[MetacouplingAssistant] Conversation reset.")
1326
+
1327
+ @property
1328
+ def history(self) -> list[Message]:
1329
+ """Return a copy of the conversation history."""
1330
+ return list(self._history)
1331
+
1332
+ @property
1333
+ def turn_count(self) -> int:
1334
+ """Return the number of completed turns."""
1335
+ return self._turn
1336
+
1337
+ # ------------------------------------------------------------------
1338
+ # Internal helpers
1339
+ # ------------------------------------------------------------------
1340
+
1341
+ def _call_llm(self) -> LLMResponse:
1342
+ """Send the current history to the LLM client."""
1343
+ return self._client.chat(
1344
+ messages=list(self._history),
1345
+ temperature=self._temperature,
1346
+ max_tokens=self._max_tokens,
1347
+ )
1348
+
1349
+ # ------------------------------------------------------------------
1350
+ # Auto-map helpers
1351
+ # ------------------------------------------------------------------
1352
+
1353
+ @staticmethod
1354
+ def _resolve_adm1_from_analysis(parsed: ParsedAnalysis) -> str | None:
1355
+ """Try to resolve a subnational ADM1 code from the parsed analysis.
1356
+
1357
+ Scans the sending, receiving, and spillover systems' ``geographic_scope``
1358
+ and ``name`` fields for subnational region names (e.g., "Michigan,
1359
+ United States") and attempts to resolve them to ADM1 codes.
1360
+
1361
+ Falls back to scanning flow directions and coupling classification
1362
+ if systems don't contain resolvable region names.
1363
+
1364
+ Returns the first successfully resolved ADM1 code, or ``None``.
1365
+ """
1366
+ from metacouplingllm.knowledge.adm1_pericoupling import resolve_adm1_code
1367
+
1368
+ def _clean_candidate_text(text: str) -> str:
1369
+ text = re.sub(r"\[[^\]]+\]", " ", text)
1370
+ text = text.replace("*", " ").replace("`", " ")
1371
+ text = re.sub(r"\s+", " ", text)
1372
+ return text.strip().rstrip(".,;:")
1373
+
1374
+ def _looks_like_direct_location(text: str) -> bool:
1375
+ lowered = text.lower().strip()
1376
+ if not lowered:
1377
+ return False
1378
+ if any(
1379
+ marker in lowered
1380
+ for marker in (
1381
+ "such as",
1382
+ "for example",
1383
+ "for instance",
1384
+ "e.g.",
1385
+ "including",
1386
+ "especially",
1387
+ "depending on",
1388
+ "potentially",
1389
+ "plausibly",
1390
+ "likely ",
1391
+ "possible ",
1392
+ )
1393
+ ):
1394
+ return False
1395
+ words = re.findall(r"[A-Za-z][A-Za-z'’-]*", lowered)
1396
+ return 0 < len(words) <= 8
1397
+
1398
+ def _try_resolve_text(text: str) -> str | None:
1399
+ """Try resolving a text string to an ADM1 code."""
1400
+ text = _clean_candidate_text(text)
1401
+ if not text:
1402
+ return None
1403
+ # Split on comma to separate region from country hint
1404
+ parts = [p.strip() for p in text.split(",")]
1405
+ region = parts[0]
1406
+ country_hint: str | None = None
1407
+ if len(parts) > 1:
1408
+ country_hint = parts[-1]
1409
+ hint_code = resolve_country_code(country_hint)
1410
+ if hint_code:
1411
+ country_hint = hint_code
1412
+ else:
1413
+ country_hint = None
1414
+ if _looks_like_direct_location(region):
1415
+ code = resolve_adm1_code(region, country=country_hint)
1416
+ if code:
1417
+ return code
1418
+ # Also try each comma-separated part individually
1419
+ for part in parts:
1420
+ part = _clean_candidate_text(part)
1421
+ if part and len(part) > 2 and _looks_like_direct_location(part):
1422
+ code = resolve_adm1_code(part)
1423
+ if code:
1424
+ return code
1425
+ return None
1426
+
1427
+ # --- Relevance guard ------------------------------------------------
1428
+ # After resolving an ADM1 code, verify the corresponding country
1429
+ # actually appears in the analysis text. This prevents false
1430
+ # positives like "Gulf" (US Gulf Coast) → PNG008 (Papua New
1431
+ # Guinea) when the analysis is about US corn exports.
1432
+ from metacouplingllm.knowledge.adm1_pericoupling import get_adm1_country
1433
+
1434
+ def _build_analysis_country_mentions() -> set[str]:
1435
+ """Collect ISO codes of all countries mentioned in the analysis."""
1436
+ codes: set[str] = set()
1437
+ combined = " ".join(parsed.iter_text_fragments())
1438
+ # Try to resolve country names found in the text
1439
+ for word_group in re.findall(
1440
+ r"[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*", combined
1441
+ ):
1442
+ code = resolve_country_code(word_group)
1443
+ if code:
1444
+ codes.add(code)
1445
+ # Also pick up explicit 3-letter ISO codes like "USA", "BRA"
1446
+ for m in re.findall(r"\b([A-Z]{3})\b", combined):
1447
+ name = get_country_name(m)
1448
+ if name:
1449
+ codes.add(m)
1450
+ return codes
1451
+
1452
+ _mentioned_countries: set[str] | None = None
1453
+
1454
+ def _is_relevant_adm1(code: str) -> bool:
1455
+ """Check if the ADM1 code's country appears in the analysis."""
1456
+ nonlocal _mentioned_countries
1457
+ if _mentioned_countries is None:
1458
+ try:
1459
+ _mentioned_countries = _build_analysis_country_mentions()
1460
+ except Exception:
1461
+ _mentioned_countries = set() # Fallback: accept all
1462
+ adm1_country = get_adm1_country(code)
1463
+ if not adm1_country:
1464
+ return True # Can't verify — accept optimistically
1465
+ return adm1_country in _mentioned_countries
1466
+
1467
+ # Wrap _try_resolve_text with the relevance guard
1468
+ _orig_try_resolve = _try_resolve_text
1469
+
1470
+ def _try_resolve_text_guarded(text: str) -> str | None:
1471
+ code = _orig_try_resolve(text)
1472
+ if code and not _is_relevant_adm1(code):
1473
+ return None
1474
+ return code
1475
+
1476
+ # --- Step 1: Scan system dict entries (nested format) ---
1477
+ for role in ("focal", "adjacent", "sending", "receiving", "spillover"):
1478
+ for entry in parsed.get_system_entries(role):
1479
+ for key in ("geographic_scope", "name"):
1480
+ text = entry.get(key, "")
1481
+ if text:
1482
+ code = _try_resolve_text_guarded(text)
1483
+ if code:
1484
+ return code
1485
+
1486
+ # --- Step 2: Scan flow directions ---
1487
+ for flow in parsed.iter_flow_entries():
1488
+ direction = flow.get("direction", "")
1489
+ if direction:
1490
+ # Split arrows and try each endpoint
1491
+ for part in _FLOW_ARROW_RE.split(direction):
1492
+ part = part.strip()
1493
+ # Remove parenthesized annotations like "(sending)"
1494
+ part = re.sub(r"\([^)]*\)", "", part).strip()
1495
+ if part:
1496
+ code = _try_resolve_text_guarded(part)
1497
+ if code:
1498
+ return code
1499
+
1500
+ # --- Step 3: Scan coupling classification text ---
1501
+ if parsed.coupling_classification:
1502
+ # Look for quoted or parenthesized topic descriptions
1503
+ # e.g., '"impact of Michigan\'s pork exports"'
1504
+ for m in re.finditer(r'["\u201c]([^"\u201d]+)["\u201d]',
1505
+ parsed.coupling_classification):
1506
+ topic = m.group(1)
1507
+ # Extract potential region names by splitting on possessives and prepositions
1508
+ for chunk in re.split(r"[''']s\s+|\s+of\s+|\s+in\s+|\s+from\s+", topic):
1509
+ chunk = chunk.strip().rstrip(".,;:")
1510
+ if chunk and len(chunk) > 2:
1511
+ code = _try_resolve_text_guarded(chunk)
1512
+ if code:
1513
+ return code
1514
+
1515
+ return None
1516
+
1517
+ @staticmethod
1518
+ def _extract_mentioned_adm1_from_text(
1519
+ parsed: ParsedAnalysis,
1520
+ ) -> set[str]:
1521
+ """Return ALL ADM1 codes mentioned in the substantive analysis.
1522
+
1523
+ Used as a fallback when the LLM's ``mentioned_adm1_regions``
1524
+ list is empty. Scans parsed systems (sending/receiving/spillover
1525
+ names + geographic_scope), flow directions/descriptions, and
1526
+ causes/effects. Does NOT scan the coupling classification text
1527
+ (which often describes the focal region), and never scans any
1528
+ pericoupling database validation text.
1529
+
1530
+ Each candidate is validated against the ADM1 database and
1531
+ guarded against false positives (e.g., "Gulf" resolving to
1532
+ Papua New Guinea) by requiring the resolved region's country
1533
+ to appear in the analysis text.
1534
+ """
1535
+ from metacouplingllm.knowledge.adm1_pericoupling import (
1536
+ get_adm1_country,
1537
+ get_adm1_info,
1538
+ resolve_adm1_code,
1539
+ )
1540
+
1541
+ # Build the set of countries mentioned in the analysis for the
1542
+ # relevance guard.
1543
+ country_mentions: set[str] = set()
1544
+ combined = " ".join(parsed.iter_text_fragments())
1545
+ for word_group in re.findall(
1546
+ r"[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*", combined
1547
+ ):
1548
+ code = resolve_country_code(word_group)
1549
+ if code:
1550
+ country_mentions.add(code)
1551
+ for m in re.findall(r"\b([A-Z]{3})\b", combined):
1552
+ if get_country_name(m):
1553
+ country_mentions.add(m)
1554
+
1555
+ def _resolve_candidate(text: str) -> str | None:
1556
+ cleaned = re.sub(r"\[[^\]]+\]", " ", text)
1557
+ cleaned = cleaned.replace("*", " ").replace("`", " ")
1558
+ cleaned = re.sub(r"\s+", " ", cleaned).strip().rstrip(".,;:")
1559
+ if not cleaned or len(cleaned) < 3:
1560
+ return None
1561
+ # Bail out on fuzzy language
1562
+ lowered = cleaned.lower()
1563
+ if any(
1564
+ marker in lowered
1565
+ for marker in (
1566
+ "such as", "for example", "e.g.", "including",
1567
+ "especially", "depending", "likely ", "possible ",
1568
+ )
1569
+ ):
1570
+ return None
1571
+ parts = [p.strip() for p in cleaned.split(",")]
1572
+ country_hint: str | None = None
1573
+ if len(parts) > 1:
1574
+ hint_code = resolve_country_code(parts[-1])
1575
+ if hint_code:
1576
+ country_hint = hint_code
1577
+ for part in parts:
1578
+ if not part or len(part) < 3:
1579
+ continue
1580
+ code = resolve_adm1_code(part, country=country_hint)
1581
+ if code is None:
1582
+ continue
1583
+ # Relevance guard: the ADM1 region's country must
1584
+ # appear in the analysis text.
1585
+ adm1_country = get_adm1_country(code)
1586
+ if adm1_country and country_mentions and (
1587
+ adm1_country not in country_mentions
1588
+ ):
1589
+ continue
1590
+ if get_adm1_info(code) is not None:
1591
+ return code
1592
+ return None
1593
+
1594
+ found: set[str] = set()
1595
+
1596
+ # Scan substantive system fields only — NOT `name` or
1597
+ # `geographic_scope`, which typically hold list-style
1598
+ # enumerations of neighbouring regions (e.g., "Para,
1599
+ # Rondonia, Tocantins, Amazonas") that the LLM echoed from
1600
+ # the adjacency database hint. Such lists are not evidence
1601
+ # of actual interaction.
1602
+ for role in ("focal", "adjacent", "sending", "receiving", "spillover"):
1603
+ for entry in parsed.get_system_entries(role):
1604
+ for key in ("human_subsystem", "natural_subsystem", "description"):
1605
+ text = entry.get(key, "")
1606
+ if not text:
1607
+ continue
1608
+ # Try the full text first
1609
+ code = _resolve_candidate(text)
1610
+ if code:
1611
+ found.add(code)
1612
+ # Also scan comma-separated sub-chunks
1613
+ for chunk in re.split(r"[;\u2022]|\s+and\s+", text):
1614
+ code = _resolve_candidate(chunk)
1615
+ if code:
1616
+ found.add(code)
1617
+ # Flat string values are skipped for the same reason
1618
+ # — they rarely describe substantive interactions.
1619
+
1620
+ # Scan flow directions and descriptions
1621
+ for flow in parsed.iter_flow_entries():
1622
+ for key in ("direction", "description"):
1623
+ text = flow.get(key, "")
1624
+ if not text:
1625
+ continue
1626
+ # Arrow-split for directions
1627
+ for part in _FLOW_ARROW_RE.split(text):
1628
+ part = re.sub(r"\([^)]*\)", "", part).strip()
1629
+ if part:
1630
+ code = _resolve_candidate(part)
1631
+ if code:
1632
+ found.add(code)
1633
+ # Also try chunk-splitting the description
1634
+ for chunk in re.split(r"[;\u2022]|\s+and\s+", text):
1635
+ code = _resolve_candidate(chunk)
1636
+ if code:
1637
+ found.add(code)
1638
+
1639
+ # Scan causes and effects lists
1640
+ for kind in ("causes", "effects"):
1641
+ for _section_name, _category, item in parsed.iter_category_items(kind):
1642
+ if not isinstance(item, str):
1643
+ continue
1644
+ for chunk in re.split(r"[;\u2022]|\s+and\s+", item):
1645
+ code = _resolve_candidate(chunk)
1646
+ if code:
1647
+ found.add(code)
1648
+
1649
+ return found
1650
+
1651
+ @staticmethod
1652
+ def _resolve_flows_for_map_legacy(
1653
+ parsed: ParsedAnalysis,
1654
+ focal_country: str,
1655
+ ) -> list[dict[str, str]]:
1656
+ """Resolve flow directions to country-level endpoints for map arrows.
1657
+
1658
+ LLMs often produce generic directions like
1659
+ ``"Michigan (sending) → Receiving regions"`` instead of specific
1660
+ country names. This method resolves such references by:
1661
+
1662
+ 1. Trying direct country-name resolution.
1663
+ 2. Trying ADM1 region-name → country resolution (e.g., Michigan → USA).
1664
+ 3. Falling back to system-role matching (``"Receiving regions"`` →
1665
+ receiving-system country codes from the analysis).
1666
+
1667
+ Flows that are purely internal (``"within Michigan"``) are skipped.
1668
+
1669
+ Returns a list of flows whose ``direction`` fields contain resolvable
1670
+ country names (e.g., ``"United States → China"``).
1671
+ """
1672
+ from metacouplingllm.knowledge.adm1_pericoupling import (
1673
+ get_adm1_country,
1674
+ resolve_adm1_code,
1675
+ )
1676
+ from metacouplingllm.knowledge.countries import (
1677
+ get_country_name,
1678
+ resolve_country_code,
1679
+ )
1680
+
1681
+ def _extract_country_list_codes(
1682
+ text: str,
1683
+ *,
1684
+ allow_soft_markers: bool = False,
1685
+ ) -> set[str]:
1686
+ """Extract country codes from list-like text."""
1687
+ if not text or not text.strip():
1688
+ return set()
1689
+
1690
+ clean = re.sub(r"\[[^\]]+\]", "", text).strip()
1691
+ lowered = clean.lower()
1692
+ hard_blockers = (
1693
+ "for example", "e.g.", "depending on", "depending upon",
1694
+ "another market", "another destination",
1695
+ "does not specify", "not specify", "not yet",
1696
+ "uncertain", "unknown", "would specify whether",
1697
+ "should be confirmed", "to be confirmed",
1698
+ )
1699
+ if any(marker in lowered for marker in hard_blockers):
1700
+ return set()
1701
+
1702
+ soft_markers = (
1703
+ "such as", "likely", "may ", " may", "might", "could",
1704
+ "plausible", "potential", "possible", "include",
1705
+ "includes", "including", "proxy",
1706
+ )
1707
+ if (
1708
+ not allow_soft_markers
1709
+ and any(marker in lowered for marker in soft_markers)
1710
+ ):
1711
+ return set()
1712
+
1713
+ chunks = re.split(r"[,;/]+|\band\b|\bor\b", clean)
1714
+ codes: set[str] = set()
1715
+ resolved_chunks: list[str] = []
1716
+
1717
+ for chunk in chunks:
1718
+ candidate = re.sub(r"\([^)]*\)", "", chunk).strip().rstrip(".:")
1719
+ if not candidate:
1720
+ continue
1721
+ code = resolve_country_code(candidate)
1722
+ if not code:
1723
+ adm1 = resolve_adm1_code(candidate)
1724
+ if adm1:
1725
+ code = get_adm1_country(adm1)
1726
+ if code:
1727
+ codes.add(code)
1728
+ resolved_chunks.append(candidate.lower())
1729
+
1730
+ if not codes:
1731
+ return set()
1732
+
1733
+ residue = lowered
1734
+ for chunk in resolved_chunks:
1735
+ residue = residue.replace(chunk, " ")
1736
+ residue = re.sub(r"\([^)]*\)", " ", residue)
1737
+ residue = re.sub(
1738
+ r"\b(and|or|countries?|country|markets?|market|regions?|region|"
1739
+ r"systems?|system|states?|state|destinations?|destination|"
1740
+ r"importers?|exporters?|trade|trading|importing|exporting|"
1741
+ r"distant|domestic|foreign|international|major|key|selected|"
1742
+ r"target|receiving|sending|spillover|focal|adjacent|"
1743
+ r"nonadjacent|other|main|principal|likely|possible|potential|"
1744
+ r"plausible|connected|connect|linked|link|including|include|"
1745
+ r"includes|such|proxy|confirmed|top)\b",
1746
+ " ",
1747
+ residue,
1748
+ )
1749
+ residue = re.sub(r"[^a-z]+", " ", residue)
1750
+ leftover_tokens = [tok for tok in residue.split() if len(tok) > 2]
1751
+ if len(leftover_tokens) > 1:
1752
+ return set()
1753
+
1754
+ return codes
1755
+
1756
+ def _extract_explicit_codes(text: str) -> set[str]:
1757
+ """Extract country codes only from explicit, low-ambiguity text."""
1758
+ return _extract_country_list_codes(
1759
+ text,
1760
+ allow_soft_markers=False,
1761
+ )
1762
+
1763
+ def _extract_proxy_receiving_codes(text: str) -> set[str]:
1764
+ """Extract softened receiving-market lists for proxy arrows."""
1765
+ lowered = text.lower()
1766
+ if not any(
1767
+ term in lowered
1768
+ for term in (
1769
+ "receiving", "market", "markets", "destination",
1770
+ "destinations", "export", "exports", "import",
1771
+ "imports",
1772
+ )
1773
+ ):
1774
+ return set()
1775
+ return _extract_country_list_codes(
1776
+ text,
1777
+ allow_soft_markers=True,
1778
+ )
1779
+
1780
+ role_codes: dict[str, set[str]] = {
1781
+ "focal": set(),
1782
+ "adjacent": set(),
1783
+ "sending": set(),
1784
+ "receiving": set(),
1785
+ "spillover": set(),
1786
+ }
1787
+ proxy_role_codes: dict[str, set[str]] = {
1788
+ "sending": set(),
1789
+ "receiving": set(),
1790
+ "spillover": set(),
1791
+ "adjacent": set(),
1792
+ }
1793
+ for role in role_codes:
1794
+ for entry in parsed.get_system_entries(role):
1795
+ texts: list[str] = []
1796
+ for key in ("name", "geographic_scope"):
1797
+ candidate = entry.get(key, "")
1798
+ if candidate:
1799
+ texts.append(candidate)
1800
+ for text in texts:
1801
+ codes = _extract_explicit_codes(text)
1802
+ role_codes[role].update(codes)
1803
+ if entry.get("system_scope") == "adjacent":
1804
+ role_codes["adjacent"].update(codes)
1805
+ if role == "receiving":
1806
+ proxy_role_codes[role].update(
1807
+ _extract_proxy_receiving_codes(text),
1808
+ )
1809
+ role_codes["sending"].update(role_codes["focal"])
1810
+
1811
+ focal_name = get_country_name(focal_country) or focal_country
1812
+
1813
+ def _try_resolve(text: str) -> str | None:
1814
+ """Try to resolve text to a country code."""
1815
+ clean = re.sub(r"\([^)]*\)", "", text).strip()
1816
+ if not clean:
1817
+ return None
1818
+ # Direct country resolution
1819
+ code = resolve_country_code(clean)
1820
+ if code:
1821
+ return code
1822
+ # ADM1 region → country
1823
+ adm1 = resolve_adm1_code(clean)
1824
+ if adm1:
1825
+ return get_adm1_country(adm1)
1826
+ return None
1827
+
1828
+ def _resolve_role_reference(text: str) -> set[str]:
1829
+ """Resolve generic role references such as receiving markets."""
1830
+ t = text.lower()
1831
+ codes: set[str] = set()
1832
+ if "receiving" in t:
1833
+ codes.update(role_codes.get("receiving", set()))
1834
+ if not codes:
1835
+ codes.update(proxy_role_codes.get("receiving", set()))
1836
+ if "spillover" in t:
1837
+ codes.update(role_codes.get("spillover", set()))
1838
+ if any(
1839
+ term in t
1840
+ for term in (
1841
+ "adjacent",
1842
+ "neighboring",
1843
+ "neighbouring",
1844
+ "pericoupled",
1845
+ "nearby",
1846
+ )
1847
+ ):
1848
+ codes.update(role_codes.get("adjacent", set()))
1849
+ if "sending" in t:
1850
+ codes.update(role_codes.get("sending", set()))
1851
+ return codes
1852
+
1853
+ def _resolve_source(text: str) -> set[str]:
1854
+ """Resolve a source endpoint, falling back to system roles."""
1855
+ explicit_codes = _extract_explicit_codes(text)
1856
+ if explicit_codes:
1857
+ return explicit_codes
1858
+ code = _try_resolve(text)
1859
+ if code:
1860
+ return {code}
1861
+ return _resolve_role_reference(text)
1862
+
1863
+ def _resolve_target(text: str) -> set[str]:
1864
+ """Resolve a target endpoint, falling back to system roles."""
1865
+ explicit_codes = _extract_explicit_codes(text)
1866
+ if explicit_codes:
1867
+ return explicit_codes
1868
+ code = _try_resolve(text)
1869
+ if code:
1870
+ return {code}
1871
+ return _resolve_role_reference(text)
1872
+
1873
+ resolved: list[dict[str, str]] = []
1874
+ seen_pairs: set[tuple[str, str, str]] = set() # (cat, src, tgt)
1875
+
1876
+ for flow in parsed.iter_flow_entries():
1877
+ direction = flow.get("direction", "")
1878
+ category = flow.get("category", "")
1879
+ if not direction:
1880
+ continue
1881
+ # Skip internal flows
1882
+ if re.search(r"\bwithin\b", direction, re.IGNORECASE):
1883
+ continue
1884
+
1885
+ is_bidir = (
1886
+ "bidirectional" in direction.lower() or "↔" in direction
1887
+ )
1888
+
1889
+ src_codes: set[str] = set()
1890
+ tgt_codes: set[str] = set()
1891
+
1892
+ # Pattern: "between X and Y"
1893
+ between_m = re.search(
1894
+ r"[Bb](?:etween|idirectional\s+between)\s+(.+?)\s+and\s+(.+)",
1895
+ direction,
1896
+ )
1897
+ if between_m:
1898
+ src_codes = _resolve_source(between_m.group(1))
1899
+ tgt_codes = _resolve_target(between_m.group(2))
1900
+ is_bidir = True
1901
+ else:
1902
+ # Pattern: "A → B"
1903
+ parts = _FLOW_ARROW_RE.split(direction)
1904
+ if len(parts) >= 2:
1905
+ src_codes = _resolve_source(parts[0])
1906
+ tgt_codes = _resolve_target(parts[1])
1907
+
1908
+ # Fallback source: use focal country
1909
+ if not src_codes:
1910
+ src_codes = {focal_country}
1911
+
1912
+ if not tgt_codes:
1913
+ continue
1914
+
1915
+ for src_code in sorted(src_codes):
1916
+ target_codes = set(tgt_codes)
1917
+ target_codes.discard(src_code)
1918
+ if not target_codes:
1919
+ continue
1920
+
1921
+ for tgt in sorted(target_codes):
1922
+ key = (category, src_code, tgt)
1923
+ if key in seen_pairs:
1924
+ continue
1925
+ seen_pairs.add(key)
1926
+
1927
+ src_name = get_country_name(src_code) or src_code
1928
+ tgt_name = get_country_name(tgt) or tgt
1929
+ if is_bidir:
1930
+ dir_str = f"Bidirectional ({src_name} ↔ {tgt_name})"
1931
+ else:
1932
+ dir_str = f"{src_name} → {tgt_name}"
1933
+ resolved.append({
1934
+ "category": category,
1935
+ "direction": dir_str,
1936
+ "description": flow.get("description", ""),
1937
+ })
1938
+
1939
+ return resolved
1940
+
1941
+ @staticmethod
1942
+ def _resolve_flows_for_map(
1943
+ parsed: ParsedAnalysis,
1944
+ focal_country: str,
1945
+ ) -> list[dict[str, str]]:
1946
+ """Resolve flow directions to country-level endpoints for map arrows.
1947
+
1948
+ This version is conservative about speculative examples, but still
1949
+ allows softened receiving-market lists such as
1950
+ ``"likely distant foreign markets ... such as Mexico, China..."``
1951
+ to inform map arrows when no confirmed subnational partner list is
1952
+ available.
1953
+ """
1954
+ from metacouplingllm.knowledge.adm1_pericoupling import (
1955
+ get_adm1_country,
1956
+ resolve_adm1_code,
1957
+ )
1958
+ from metacouplingllm.knowledge.countries import (
1959
+ get_country_name,
1960
+ resolve_country_code,
1961
+ )
1962
+
1963
+ def _extract_country_list_codes(
1964
+ text: str,
1965
+ *,
1966
+ allow_soft_markers: bool = False,
1967
+ ) -> set[str]:
1968
+ if not text or not text.strip():
1969
+ return set()
1970
+
1971
+ clean = re.sub(r"\[[^\]]+\]", "", text).strip()
1972
+ lowered = clean.lower()
1973
+ hard_blockers = (
1974
+ "for example", "e.g.", "depending on", "depending upon",
1975
+ "another market", "another destination",
1976
+ "does not specify", "not specify", "not yet",
1977
+ "uncertain", "unknown", "would specify whether",
1978
+ "should be confirmed", "to be confirmed",
1979
+ )
1980
+ if any(marker in lowered for marker in hard_blockers):
1981
+ return set()
1982
+
1983
+ soft_markers = (
1984
+ "such as", "likely", "may ", " may", "might", "could",
1985
+ "plausible", "potential", "possible", "include",
1986
+ "includes", "including", "proxy",
1987
+ )
1988
+ if (
1989
+ not allow_soft_markers
1990
+ and any(marker in lowered for marker in soft_markers)
1991
+ ):
1992
+ return set()
1993
+
1994
+ chunks = re.split(r"[,;/]+|\band\b|\bor\b", clean)
1995
+ codes: set[str] = set()
1996
+ resolved_chunks: list[str] = []
1997
+
1998
+ for chunk in chunks:
1999
+ candidate = re.sub(r"\([^)]*\)", "", chunk).strip().rstrip(".:")
2000
+ if not candidate:
2001
+ continue
2002
+ code = resolve_country_code(candidate)
2003
+ if not code:
2004
+ adm1 = resolve_adm1_code(candidate)
2005
+ if adm1:
2006
+ code = get_adm1_country(adm1)
2007
+ if code:
2008
+ codes.add(code)
2009
+ resolved_chunks.append(candidate.lower())
2010
+
2011
+ if not codes:
2012
+ return set()
2013
+
2014
+ residue = lowered
2015
+ for chunk in resolved_chunks:
2016
+ residue = residue.replace(chunk, " ")
2017
+ residue = re.sub(r"\([^)]*\)", " ", residue)
2018
+ residue = re.sub(
2019
+ r"\b(and|or|countries?|country|markets?|market|regions?|region|"
2020
+ r"systems?|system|states?|state|destinations?|destination|"
2021
+ r"importers?|exporters?|trade|trading|importing|exporting|"
2022
+ r"distant|domestic|foreign|international|major|key|selected|"
2023
+ r"target|receiving|sending|spillover|focal|adjacent|"
2024
+ r"nonadjacent|other|main|principal|likely|possible|potential|"
2025
+ r"plausible|connected|connect|linked|link|including|include|"
2026
+ r"includes|such|proxy|confirmed|top)\b",
2027
+ " ",
2028
+ residue,
2029
+ )
2030
+ residue = re.sub(r"[^a-z]+", " ", residue)
2031
+ leftover_tokens = [tok for tok in residue.split() if len(tok) > 2]
2032
+ if len(leftover_tokens) > 1:
2033
+ return set()
2034
+
2035
+ return codes
2036
+
2037
+ def _extract_explicit_codes(text: str) -> set[str]:
2038
+ return _extract_country_list_codes(
2039
+ text,
2040
+ allow_soft_markers=False,
2041
+ )
2042
+
2043
+ def _extract_proxy_receiving_codes(text: str) -> set[str]:
2044
+ lowered = text.lower()
2045
+ if not any(
2046
+ term in lowered
2047
+ for term in (
2048
+ "receiving", "market", "markets", "destination",
2049
+ "destinations", "export", "exports", "import",
2050
+ "imports",
2051
+ )
2052
+ ):
2053
+ return set()
2054
+ return _extract_country_list_codes(
2055
+ text,
2056
+ allow_soft_markers=True,
2057
+ )
2058
+
2059
+ role_codes: dict[str, set[str]] = {
2060
+ "focal": set(),
2061
+ "adjacent": set(),
2062
+ "sending": set(),
2063
+ "receiving": set(),
2064
+ "spillover": set(),
2065
+ }
2066
+ proxy_role_codes: dict[str, set[str]] = {
2067
+ "sending": set(),
2068
+ "receiving": set(),
2069
+ "spillover": set(),
2070
+ "adjacent": set(),
2071
+ }
2072
+ for role in role_codes:
2073
+ for entry in parsed.get_system_entries(role):
2074
+ texts: list[str] = []
2075
+ for key in ("name", "geographic_scope"):
2076
+ candidate = entry.get(key, "")
2077
+ if candidate:
2078
+ texts.append(candidate)
2079
+ for text in texts:
2080
+ codes = _extract_explicit_codes(text)
2081
+ role_codes[role].update(codes)
2082
+ if entry.get("system_scope") == "adjacent":
2083
+ role_codes["adjacent"].update(codes)
2084
+ if role == "receiving":
2085
+ proxy_role_codes[role].update(
2086
+ _extract_proxy_receiving_codes(text),
2087
+ )
2088
+ role_codes["sending"].update(role_codes["focal"])
2089
+
2090
+ def _try_resolve(text: str) -> str | None:
2091
+ clean = re.sub(r"\([^)]*\)", "", text).strip()
2092
+ if not clean:
2093
+ return None
2094
+
2095
+ candidates = [clean]
2096
+ clause_parts = [
2097
+ part.strip()
2098
+ for part in re.split(r"[;:]", clean)
2099
+ if part.strip()
2100
+ ]
2101
+ if clause_parts:
2102
+ candidates.extend(clause_parts)
2103
+
2104
+ normalised: list[str] = []
2105
+ for candidate in candidates:
2106
+ simplified = re.sub(
2107
+ r"^(?:effectively|primarily|mainly|mostly|especially|"
2108
+ r"roughly|approximately|about|notably)\s+",
2109
+ "",
2110
+ candidate,
2111
+ flags=re.IGNORECASE,
2112
+ ).strip()
2113
+ if simplified:
2114
+ normalised.append(simplified)
2115
+
2116
+ for candidate in normalised:
2117
+ code = resolve_country_code(candidate)
2118
+ if code:
2119
+ return code
2120
+ adm1 = resolve_adm1_code(candidate)
2121
+ if adm1:
2122
+ return get_adm1_country(adm1)
2123
+ return None
2124
+
2125
+ def _resolve_role_reference(text: str) -> set[str]:
2126
+ lowered = text.lower()
2127
+ codes: set[str] = set()
2128
+ if any(
2129
+ term in lowered
2130
+ for term in (
2131
+ "receiving",
2132
+ "importing",
2133
+ "importer",
2134
+ "importers",
2135
+ "destination",
2136
+ "destinations",
2137
+ )
2138
+ ):
2139
+ codes.update(role_codes.get("receiving", set()))
2140
+ codes.update(proxy_role_codes.get("receiving", set()))
2141
+ if any(
2142
+ term in lowered
2143
+ for term in (
2144
+ "spillover",
2145
+ "competing",
2146
+ "competitor",
2147
+ "competitors",
2148
+ "indirectly affected",
2149
+ )
2150
+ ):
2151
+ codes.update(role_codes.get("spillover", set()))
2152
+ if any(
2153
+ term in lowered
2154
+ for term in (
2155
+ "adjacent",
2156
+ "neighboring",
2157
+ "neighbouring",
2158
+ "pericoupled",
2159
+ "nearby",
2160
+ )
2161
+ ):
2162
+ codes.update(role_codes.get("adjacent", set()))
2163
+ if any(
2164
+ term in lowered
2165
+ for term in (
2166
+ "sending",
2167
+ "exporting",
2168
+ "exporter",
2169
+ "exporters",
2170
+ "origin",
2171
+ "origins",
2172
+ )
2173
+ ):
2174
+ codes.update(role_codes.get("sending", set()))
2175
+ return codes
2176
+
2177
+ def _resolve_source(text: str) -> set[str]:
2178
+ explicit_codes = _extract_explicit_codes(text)
2179
+ if explicit_codes:
2180
+ return explicit_codes
2181
+ code = _try_resolve(text)
2182
+ if code:
2183
+ return {code}
2184
+ return _resolve_role_reference(text)
2185
+
2186
+ def _resolve_target(text: str) -> set[str]:
2187
+ explicit_codes = _extract_explicit_codes(text)
2188
+ if explicit_codes:
2189
+ return explicit_codes
2190
+ code = _try_resolve(text)
2191
+ if code:
2192
+ return {code}
2193
+ return _resolve_role_reference(text)
2194
+
2195
+ resolved: list[dict[str, str]] = []
2196
+ seen_pairs: set[tuple[str, str, str]] = set()
2197
+
2198
+ for flow in parsed.iter_flow_entries():
2199
+ direction = flow.get("direction", "")
2200
+ category = flow.get("category", "")
2201
+ if not direction:
2202
+ continue
2203
+ has_connector = bool(_FLOW_HAS_CONNECTOR_RE.search(direction))
2204
+ if (
2205
+ not has_connector
2206
+ and re.search(r"\bwithin\b", direction, re.IGNORECASE)
2207
+ ):
2208
+ continue
2209
+
2210
+ is_bidir = (
2211
+ "bidirectional" in direction.lower()
2212
+ or "\u2194" in direction
2213
+ or "<->" in direction
2214
+ or "<=>" in direction
2215
+ )
2216
+
2217
+ src_codes: set[str] = set()
2218
+ tgt_codes: set[str] = set()
2219
+
2220
+ between_m = re.search(
2221
+ r"[Bb](?:etween|idirectional\s+between)\s+(.+?)\s+and\s+(.+)",
2222
+ direction,
2223
+ )
2224
+ if between_m:
2225
+ src_codes = _resolve_source(between_m.group(1))
2226
+ tgt_codes = _resolve_target(between_m.group(2))
2227
+ is_bidir = True
2228
+ else:
2229
+ parts = _FLOW_ARROW_RE.split(direction)
2230
+ if len(parts) >= 2:
2231
+ src_codes = _resolve_source(parts[0])
2232
+ tgt_codes = _resolve_target(parts[1])
2233
+
2234
+ # Fallback: when the LLM used a generic label like
2235
+ # "importing countries" instead of specific country names,
2236
+ # harvest concrete countries from the Systems section.
2237
+ if not tgt_codes:
2238
+ try:
2239
+ from metacouplingllm.visualization.worldmap import (
2240
+ _extract_all_analysis_countries,
2241
+ )
2242
+
2243
+ receiving = _extract_all_analysis_countries(parsed).get(
2244
+ "receiving", set()
2245
+ )
2246
+ if receiving:
2247
+ tgt_codes = receiving
2248
+ except (ImportError, AttributeError, KeyError):
2249
+ pass # Viz deps unavailable — use default resolution
2250
+
2251
+ if not src_codes:
2252
+ # When source is generic (e.g., "Importing countries")
2253
+ # and target IS the focal country, the unresolved source
2254
+ # is the receiving systems (e.g., China sends capital
2255
+ # TO Brazil). Only use receiving — never spillover,
2256
+ # which contains competing exporters, not trade partners.
2257
+ if tgt_codes and focal_country in tgt_codes:
2258
+ try:
2259
+ from metacouplingllm.visualization.worldmap import (
2260
+ _extract_all_analysis_countries,
2261
+ )
2262
+
2263
+ receiving = _extract_all_analysis_countries(
2264
+ parsed,
2265
+ ).get("receiving", set())
2266
+ partner_codes = receiving - tgt_codes
2267
+ if partner_codes:
2268
+ src_codes = partner_codes
2269
+ except (ImportError, AttributeError, KeyError):
2270
+ pass # Viz deps unavailable
2271
+ if not src_codes:
2272
+ src_codes = {focal_country}
2273
+ if not tgt_codes:
2274
+ continue
2275
+
2276
+ # Safety net: avoid self-loops where src == tgt.
2277
+ if src_codes == tgt_codes:
2278
+ try:
2279
+ from metacouplingllm.visualization.worldmap import (
2280
+ _extract_all_analysis_countries,
2281
+ )
2282
+
2283
+ receiving = _extract_all_analysis_countries(
2284
+ parsed,
2285
+ ).get("receiving", set())
2286
+ partners = receiving - src_codes
2287
+ if partners:
2288
+ tgt_codes = partners
2289
+ except (ImportError, AttributeError, KeyError):
2290
+ pass # Viz deps unavailable
2291
+
2292
+ for src_code in sorted(src_codes):
2293
+ target_codes = set(tgt_codes)
2294
+ target_codes.discard(src_code)
2295
+ if not target_codes:
2296
+ continue
2297
+
2298
+ for tgt in sorted(target_codes):
2299
+ key = (category, src_code, tgt)
2300
+ if key in seen_pairs:
2301
+ continue
2302
+ seen_pairs.add(key)
2303
+
2304
+ src_name = get_country_name(src_code) or src_code
2305
+ tgt_name = get_country_name(tgt) or tgt
2306
+ if is_bidir:
2307
+ dir_str = (
2308
+ f"Bidirectional ({src_name} \u2194 {tgt_name})"
2309
+ )
2310
+ else:
2311
+ dir_str = f"{src_name} \u2192 {tgt_name}"
2312
+ resolved.append({
2313
+ "category": category,
2314
+ "direction": dir_str,
2315
+ "description": flow.get("description", ""),
2316
+ })
2317
+
2318
+ return resolved
2319
+
2320
+ @staticmethod
2321
+ def _resolve_flows_for_adm1_map(
2322
+ parsed: ParsedAnalysis,
2323
+ focal_adm1: str,
2324
+ focal_country: str,
2325
+ ) -> list[dict[str, str]]:
2326
+ """Resolve mixed ADM1/country flow arrows for ADM1 maps.
2327
+
2328
+ International or cross-country flows remain country-level arrows.
2329
+ Same-country nearby flows involving adjacent ADM1 neighbors are
2330
+ emitted as ADM1-to-ADM1 arrows so they can be visualized locally.
2331
+ """
2332
+ from metacouplingllm.knowledge.adm1_pericoupling import (
2333
+ get_adm1_country,
2334
+ get_adm1_info,
2335
+ get_adm1_neighbors,
2336
+ )
2337
+
2338
+ resolved = MetacouplingAssistant._resolve_flows_for_map(
2339
+ parsed,
2340
+ focal_country,
2341
+ )
2342
+
2343
+ focal_info = get_adm1_info(focal_adm1)
2344
+ focal_name = focal_info["name"] if focal_info else focal_adm1
2345
+ domestic_neighbors = sorted(
2346
+ code
2347
+ for code in get_adm1_neighbors(focal_adm1)
2348
+ if get_adm1_country(code) == focal_country
2349
+ )
2350
+ neighbor_name_to_code = {
2351
+ get_adm1_info(code)["name"].lower(): code
2352
+ for code in domestic_neighbors
2353
+ if get_adm1_info(code) is not None
2354
+ }
2355
+ seen_domestic: set[tuple[str, str, str]] = set()
2356
+
2357
+ def _has_local_marker(text: str) -> bool:
2358
+ lowered = text.lower()
2359
+ return any(
2360
+ marker in lowered
2361
+ for marker in (
2362
+ "adjacent state",
2363
+ "adjacent states",
2364
+ "neighboring state",
2365
+ "neighboring states",
2366
+ "pericoupled",
2367
+ "pericoupling",
2368
+ "regional flow",
2369
+ "regional flows",
2370
+ )
2371
+ )
2372
+
2373
+ def _mentioned_neighbor_codes(text: str) -> set[str]:
2374
+ lowered = text.lower()
2375
+ found: set[str] = set()
2376
+ for name, code in neighbor_name_to_code.items():
2377
+ if re.search(rf"\b{re.escape(name)}\b", lowered):
2378
+ found.add(code)
2379
+ return found
2380
+
2381
+ for flow in parsed.iter_flow_entries():
2382
+ direction = flow.get("direction", "")
2383
+ description = flow.get("description", "")
2384
+ category = flow.get("category", "")
2385
+ combined = f"{direction} {description}".strip()
2386
+ if not combined:
2387
+ continue
2388
+
2389
+ mentioned_neighbors = _mentioned_neighbor_codes(combined)
2390
+ if not mentioned_neighbors and _has_local_marker(combined):
2391
+ mentioned_neighbors = set(domestic_neighbors)
2392
+ if not mentioned_neighbors:
2393
+ continue
2394
+
2395
+ lowered_direction = direction.lower()
2396
+ is_bidir = (
2397
+ "bidirectional" in lowered_direction
2398
+ or "\u2194" in direction
2399
+ or "<->" in direction
2400
+ or "<=>" in direction
2401
+ )
2402
+
2403
+ src_is_neighbor = False
2404
+ tgt_is_neighbor = False
2405
+ parts = _FLOW_ARROW_RE.split(direction)
2406
+ if len(parts) >= 2:
2407
+ src_text = parts[0].lower()
2408
+ tgt_text = parts[1].lower()
2409
+ src_is_neighbor = bool(_mentioned_neighbor_codes(src_text)) or (
2410
+ _has_local_marker(src_text) and "michigan" not in src_text
2411
+ )
2412
+ tgt_is_neighbor = bool(_mentioned_neighbor_codes(tgt_text)) or (
2413
+ _has_local_marker(tgt_text) and "michigan" not in tgt_text
2414
+ )
2415
+
2416
+ for neighbor_code in sorted(mentioned_neighbors):
2417
+ key = (category, focal_adm1, neighbor_code)
2418
+ if key in seen_domestic:
2419
+ continue
2420
+ seen_domestic.add(key)
2421
+
2422
+ neighbor_info = get_adm1_info(neighbor_code)
2423
+ if neighbor_info is None:
2424
+ continue
2425
+ neighbor_name = neighbor_info["name"]
2426
+
2427
+ source_code = focal_adm1
2428
+ target_code = neighbor_code
2429
+ if src_is_neighbor and not tgt_is_neighbor:
2430
+ source_code = neighbor_code
2431
+ target_code = focal_adm1
2432
+
2433
+ if is_bidir:
2434
+ direction_label = (
2435
+ f"Bidirectional ({focal_name} \u2194 {neighbor_name})"
2436
+ )
2437
+ elif source_code == focal_adm1:
2438
+ direction_label = f"{focal_name} \u2192 {neighbor_name}"
2439
+ else:
2440
+ direction_label = f"{neighbor_name} \u2192 {focal_name}"
2441
+
2442
+ resolved.append({
2443
+ "category": category,
2444
+ "direction": direction_label,
2445
+ "description": description,
2446
+ "source_adm1": source_code,
2447
+ "target_adm1": target_code,
2448
+ "is_bidirectional": is_bidir,
2449
+ })
2450
+
2451
+ return resolved
2452
+
2453
+ def _build_adm1_reference_for_prompt(
2454
+ self,
2455
+ parsed: ParsedAnalysis,
2456
+ get_adm1_codes_for_country,
2457
+ get_adm1_info,
2458
+ ) -> str:
2459
+ """Build an ADM1 code reference block for the extraction prompt.
2460
+
2461
+ Collects countries mentioned in the analysis, looks up their
2462
+ ADM1 codes in the pericoupling database, and formats them as a
2463
+ closed-set reference list. This prevents the LLM from
2464
+ hallucinating codes like ``BRA014`` (which doesn't exist) when
2465
+ the correct code is ``BRA011`` (Mato Grosso).
2466
+
2467
+ Returns an empty string when no mentioned countries have ADM1
2468
+ data in the database.
2469
+ """
2470
+ # Gather country ISO codes mentioned anywhere in the analysis
2471
+ mentioned_iso: set[str] = set()
2472
+ combined = " ".join(parsed.iter_text_fragments())
2473
+
2474
+ # Resolve country names found in the text
2475
+ for word_group in re.findall(
2476
+ r"[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*", combined
2477
+ ):
2478
+ code = resolve_country_code(word_group)
2479
+ if code:
2480
+ mentioned_iso.add(code)
2481
+ # Pick up explicit 3-letter ISO codes
2482
+ for m in re.findall(r"\b([A-Z]{3})\b", combined):
2483
+ if get_country_name(m):
2484
+ mentioned_iso.add(m)
2485
+
2486
+ if not mentioned_iso:
2487
+ return ""
2488
+
2489
+ # Build the reference list — one line per country with all
2490
+ # its ADM1 codes. Cap at a reasonable size to keep the prompt
2491
+ # manageable.
2492
+ lines: list[str] = []
2493
+ for iso in sorted(mentioned_iso):
2494
+ codes = get_adm1_codes_for_country(iso)
2495
+ if not codes:
2496
+ continue
2497
+ country_name = get_country_name(iso) or iso
2498
+ pairs: list[tuple[str, str]] = []
2499
+ for code in sorted(codes):
2500
+ info = get_adm1_info(code)
2501
+ if info:
2502
+ name = info.get("name", "")
2503
+ if name:
2504
+ pairs.append((code, name))
2505
+ if not pairs:
2506
+ continue
2507
+ pair_str = ", ".join(f"{c}={n}" for c, n in pairs)
2508
+ lines.append(f"- {country_name} ({iso}): {pair_str}")
2509
+
2510
+ if not lines:
2511
+ return ""
2512
+
2513
+ return (
2514
+ "VALID ADM1 CODES (closed set — only use codes from this "
2515
+ "list, never invent new ones):\n"
2516
+ + "\n".join(lines)
2517
+ + "\n\n"
2518
+ )
2519
+
2520
+ def _extract_map_data_from_analysis(
2521
+ self,
2522
+ parsed: ParsedAnalysis,
2523
+ ) -> dict[str, object] | None:
2524
+ """Second LLM call: extract structured map data from the analysis.
2525
+
2526
+ Sends the parsed analysis text to the LLM with a focused JSON
2527
+ schema, validates the response, and returns a dict with
2528
+ ``focal_country``, ``adm1_region``, ``receiving_countries``,
2529
+ ``spillover_countries``, and ``flows``.
2530
+
2531
+ Returns ``None`` if extraction fails.
2532
+ """
2533
+ import json
2534
+
2535
+ from metacouplingllm.knowledge.adm1_pericoupling import (
2536
+ get_adm1_codes_for_country,
2537
+ get_adm1_info,
2538
+ )
2539
+ from metacouplingllm.knowledge.countries import (
2540
+ get_country_name,
2541
+ resolve_country_code,
2542
+ )
2543
+ from metacouplingllm.knowledge.websearch import _extract_json_object
2544
+ from metacouplingllm.llm.client import Message
2545
+
2546
+ # Build a concise summary of the analysis for the extraction call
2547
+ parts: list[str] = []
2548
+ if parsed.coupling_classification:
2549
+ parts.append(
2550
+ "Coupling classification:\n"
2551
+ + parsed.coupling_classification[:600]
2552
+ )
2553
+ for role in ("focal", "adjacent", "sending", "receiving", "spillover"):
2554
+ entries = parsed.get_system_entries(role)
2555
+ for idx, entry in enumerate(entries[:2], 1):
2556
+ text = " | ".join(
2557
+ f"{k}: {v}"
2558
+ for k, v in entry.items()
2559
+ if k != "role" and v
2560
+ )
2561
+ if text:
2562
+ parts.append(f"{role.title()} system {idx}: {text[:400]}")
2563
+ for flow in list(parsed.iter_flow_entries())[:8]:
2564
+ cat = flow.get("category", "")
2565
+ dirn = flow.get("direction", "")
2566
+ desc = flow.get("description", "")[:100]
2567
+ parts.append(f"Flow [{cat}] {dirn}: {desc}")
2568
+
2569
+ analysis_summary = "\n\n".join(parts)
2570
+
2571
+ # Include web search snippets so the LLM can identify specific
2572
+ # countries even when the analysis uses generic labels like
2573
+ # "foreign importing countries."
2574
+ web_snippet_text = ""
2575
+ if self._last_web_results:
2576
+ lines: list[str] = []
2577
+ for idx, wr in enumerate(self._last_web_results[:10], 1):
2578
+ title = wr.get("title", "").strip()
2579
+ snippet = wr.get("snippet", "").strip()[:200]
2580
+ lines.append(f"[W{idx}] {title}: {snippet}")
2581
+ web_snippet_text = (
2582
+ "\n\nWeb search snippets (use these to identify specific "
2583
+ "countries when the analysis is vague):\n"
2584
+ + "\n".join(lines)
2585
+ )
2586
+
2587
+ # Build a reference list of valid ADM1 codes for countries
2588
+ # mentioned in the analysis. This prevents the LLM from
2589
+ # hallucinating codes like BRA014 (which doesn't exist) when
2590
+ # the correct code is BRA011 (Mato Grosso).
2591
+ adm1_reference_text = self._build_adm1_reference_for_prompt(
2592
+ parsed,
2593
+ get_adm1_codes_for_country,
2594
+ get_adm1_info,
2595
+ )
2596
+
2597
+ system_text = (
2598
+ "You extract structured map data from a metacoupling analysis "
2599
+ "and optional web search context. "
2600
+ "Return ONLY a JSON object, no commentary."
2601
+ )
2602
+ user_text = (
2603
+ "Extract map-ready data from this metacoupling analysis.\n\n"
2604
+ "Return a JSON object with these fields:\n"
2605
+ "- focal_country: ISO alpha-3 code of the primary sending "
2606
+ "country (e.g., \"BRA\", \"USA\")\n"
2607
+ "- adm1_region: subnational ADM1 code if the study focuses "
2608
+ "on a specific region. MUST be chosen from the reference "
2609
+ "list below — NEVER invent codes. Use null if the study is "
2610
+ "not focused on a specific subnational region.\n"
2611
+ "- mentioned_adm1_regions: list of ADM1 codes (from the "
2612
+ "reference list below) for subnational regions EXPLICITLY "
2613
+ "discussed in the analysis as sending, receiving, "
2614
+ "pericoupled, or spillover systems. Use an empty list [] "
2615
+ "if no specific subnational regions are discussed. This "
2616
+ "is used to filter the map and must only include regions "
2617
+ "substantively discussed, NOT regions that merely appear "
2618
+ "in a reference/validation section.\n"
2619
+ "- receiving_countries: list of ISO alpha-3 codes for "
2620
+ "receiving systems (countries that RECEIVE goods/services "
2621
+ "FROM the focal country)\n"
2622
+ "- spillover_countries: list of ISO alpha-3 codes for "
2623
+ "spillover systems (countries indirectly affected)\n"
2624
+ "- flows: list of objects with:\n"
2625
+ " - category: one of matter, capital, information, energy, "
2626
+ "people, organisms\n"
2627
+ " - source: ISO alpha-3 code of the exporter/sender\n"
2628
+ " - target: ISO alpha-3 code of the importer/receiver\n"
2629
+ " - bidirectional: true or false\n\n"
2630
+ "IMPORTANT RULES:\n"
2631
+ "1. Use ISO alpha-3 codes (USA, BRA, CHN, MEX, JPN, etc)\n"
2632
+ "2. receiving_countries = trade partners that BUY from or "
2633
+ "RECEIVE from the focal country. They are NOT competitors.\n"
2634
+ "3. spillover_countries = competitors, indirectly affected "
2635
+ "countries, environmental spillover. They are NOT trade "
2636
+ "partners.\n"
2637
+ "4. NEVER put competitor/competing exporter countries in "
2638
+ "receiving_countries. Example: if USA exports corn, Brazil "
2639
+ "and Argentina are COMPETITORS (spillover), not receivers.\n"
2640
+ "5. Most flows go FROM focal_country TO receiving_countries "
2641
+ "(matter, information, energy, people, organisms).\n"
2642
+ "6. Capital/payment flows go in REVERSE: FROM "
2643
+ "receiving_countries TO focal_country (e.g., "
2644
+ "'Japan \u2192 Australia' for beef payments). Create one "
2645
+ "capital flow per receiving country.\n"
2646
+ "7. No flows should involve spillover countries as source "
2647
+ "or target.\n"
2648
+ "8. When the analysis says 'importing countries' without "
2649
+ "naming them, use the web snippets to identify the most "
2650
+ "likely specific countries\n"
2651
+ "9. Do not invent countries with no supporting evidence\n"
2652
+ "10. For adm1_region: ONLY use codes from the reference "
2653
+ "list below. If the study's subnational focus does not "
2654
+ "appear in the list, return null for adm1_region.\n"
2655
+ "11. For mentioned_adm1_regions: include codes ONLY for "
2656
+ "regions where the analysis provides SUBSTANTIVE EVIDENCE "
2657
+ "of actual interaction with the focal region — specific "
2658
+ "flows, impacts, shared infrastructure, commodity "
2659
+ "transport, labor movement, land-use displacement, or "
2660
+ "other concrete linkages. Do NOT include a region just "
2661
+ "because it is named in a reference list, coupling "
2662
+ "classification, or pericoupled-neighbors enumeration. If "
2663
+ "a region is only mentioned as \"adjacent\" without any "
2664
+ "specific interaction described, do NOT include it. The "
2665
+ "focal_adm1 region should NOT be listed here — it is "
2666
+ "tracked separately via adm1_region.\n\n"
2667
+ f"{adm1_reference_text}"
2668
+ f"Analysis:\n{analysis_summary}"
2669
+ f"{web_snippet_text}"
2670
+ )
2671
+
2672
+ try:
2673
+ response = self._client.chat(
2674
+ messages=[
2675
+ Message(role="system", content=system_text),
2676
+ Message(role="user", content=user_text),
2677
+ ],
2678
+ temperature=0.0,
2679
+ max_tokens=8192,
2680
+ )
2681
+ except Exception as exc:
2682
+ print(
2683
+ "[MetacouplingAssistant] Map data extraction LLM call "
2684
+ f"failed: {exc}"
2685
+ )
2686
+ return None
2687
+
2688
+ try:
2689
+ raw_obj = _extract_json_object(response.content)
2690
+ except Exception as exc:
2691
+ print(
2692
+ f"[MetacouplingAssistant] JSON parsing error: {exc}. "
2693
+ f"Raw (first 300 chars): {response.content[:300]}"
2694
+ )
2695
+ return None
2696
+
2697
+ if raw_obj is None:
2698
+ print(
2699
+ "[MetacouplingAssistant] Map data extraction returned "
2700
+ "non-JSON response. Raw response (first 300 chars): "
2701
+ f"{response.content[:300]}"
2702
+ )
2703
+ return None
2704
+
2705
+ if not isinstance(raw_obj, dict):
2706
+ print(
2707
+ "[MetacouplingAssistant] Map data extraction returned "
2708
+ f"non-dict JSON: {type(raw_obj).__name__}"
2709
+ )
2710
+ return None
2711
+
2712
+ # --- Validate and normalise ---
2713
+ focal = resolve_country_code(str(raw_obj.get("focal_country", "")))
2714
+ if not focal:
2715
+ print(
2716
+ "[MetacouplingAssistant] Map data extraction: could not "
2717
+ "resolve focal country."
2718
+ )
2719
+ return None
2720
+
2721
+ adm1_region = raw_obj.get("adm1_region")
2722
+ if isinstance(adm1_region, str) and adm1_region.strip():
2723
+ adm1_region = adm1_region.strip()
2724
+ # Validate against the pericoupling database. The LLM
2725
+ # sometimes hallucinates codes (e.g., "BRA014" instead of
2726
+ # the correct "BRA011" for Mato Grosso).
2727
+ if get_adm1_info(adm1_region) is None:
2728
+ print(
2729
+ f"[MetacouplingAssistant] LLM returned invalid ADM1 "
2730
+ f"code '{adm1_region}' — falling back to regex "
2731
+ f"resolver."
2732
+ )
2733
+ adm1_region = None
2734
+ else:
2735
+ adm1_region = None
2736
+
2737
+ # Fallback: if the LLM didn't return a valid ADM1 code, try the
2738
+ # regex-based resolver which reads the analysis text directly
2739
+ # and resolves region names against the database. This is the
2740
+ # same resolver used by the pericoupling database validation.
2741
+ if adm1_region is None:
2742
+ fallback = self._resolve_adm1_from_analysis(parsed)
2743
+ if fallback:
2744
+ print(
2745
+ f"[MetacouplingAssistant] ADM1 code resolved from "
2746
+ f"analysis text: {fallback}."
2747
+ )
2748
+ adm1_region = fallback
2749
+
2750
+ def _resolve_code_list(items: object) -> list[str]:
2751
+ if not isinstance(items, list):
2752
+ return []
2753
+ codes: list[str] = []
2754
+ for item in items:
2755
+ code = resolve_country_code(str(item).strip())
2756
+ if code:
2757
+ codes.append(code)
2758
+ return codes
2759
+
2760
+ def _resolve_adm1_list(items: object) -> list[str]:
2761
+ """Validate a list of ADM1 codes against the database."""
2762
+ if not isinstance(items, list):
2763
+ return []
2764
+ codes: list[str] = []
2765
+ seen: set[str] = set()
2766
+ for item in items:
2767
+ code = str(item).strip()
2768
+ if not code or code in seen:
2769
+ continue
2770
+ if get_adm1_info(code) is not None:
2771
+ codes.append(code)
2772
+ seen.add(code)
2773
+ return codes
2774
+
2775
+ receiving = _resolve_code_list(raw_obj.get("receiving_countries"))
2776
+ spillover = _resolve_code_list(raw_obj.get("spillover_countries"))
2777
+ mentioned_adm1 = _resolve_adm1_list(
2778
+ raw_obj.get("mentioned_adm1_regions"),
2779
+ )
2780
+
2781
+ # Validate flows
2782
+ valid_categories = {
2783
+ "matter", "material", "capital", "financial",
2784
+ "information", "energy", "people", "organisms",
2785
+ }
2786
+ flows: list[dict[str, object]] = []
2787
+ raw_flows = raw_obj.get("flows", [])
2788
+ if isinstance(raw_flows, list):
2789
+ for item in raw_flows:
2790
+ if not isinstance(item, dict):
2791
+ continue
2792
+ cat = str(item.get("category", "")).strip().lower()
2793
+ if cat in ("material", "financial"):
2794
+ cat = {"material": "matter", "financial": "capital"}[cat]
2795
+ if cat not in valid_categories:
2796
+ continue
2797
+ src = resolve_country_code(str(item.get("source", "")))
2798
+ tgt = resolve_country_code(str(item.get("target", "")))
2799
+ if not src or not tgt or src == tgt:
2800
+ continue
2801
+ bidir = bool(item.get("bidirectional", False))
2802
+ src_name = get_country_name(src) or src
2803
+ tgt_name = get_country_name(tgt) or tgt
2804
+ if bidir:
2805
+ direction = f"Bidirectional ({src_name} \u2194 {tgt_name})"
2806
+ else:
2807
+ direction = f"{src_name} \u2192 {tgt_name}"
2808
+ flows.append({
2809
+ "category": cat,
2810
+ "source": src,
2811
+ "target": tgt,
2812
+ "direction": direction,
2813
+ "bidirectional": bidir,
2814
+ })
2815
+
2816
+ result = {
2817
+ "focal_country": focal,
2818
+ "adm1_region": adm1_region,
2819
+ "mentioned_adm1_regions": mentioned_adm1,
2820
+ "receiving_countries": receiving,
2821
+ "spillover_countries": spillover,
2822
+ "flows": flows,
2823
+ }
2824
+
2825
+ if self._verbose:
2826
+ print(
2827
+ f"[MetacouplingAssistant] Map extraction: focal={focal}, "
2828
+ f"receiving={receiving}, spillover={spillover}, "
2829
+ f"mentioned_adm1={mentioned_adm1}, "
2830
+ f"flows={len(flows)}."
2831
+ )
2832
+
2833
+ return result
2834
+
2835
+ # ------------------------------------------------------------------
2836
+ # Structured extraction supplement (rag_structured_extraction=True)
2837
+ # ------------------------------------------------------------------
2838
+
2839
+ def _structured_extract_supplement(
2840
+ self,
2841
+ parsed: ParsedAnalysis,
2842
+ ) -> dict[str, object] | None:
2843
+ """Second LLM call: extract systems / flows the draft may have missed.
2844
+
2845
+ Runs over ``self._last_rag_hits`` (the already-retrieved RAG
2846
+ passages) plus a concise summary of the draft analysis, and
2847
+ returns a dict with additional system mentions, empty-subfield
2848
+ fills, and supplementary flows — each carrying the passage IDs
2849
+ that support it.
2850
+
2851
+ The result is attached to :class:`AnalysisResult` as
2852
+ ``structured_supplement`` and rendered as a visible block in
2853
+ ``formatted``. The caller (``_build_result``) does NOT merge
2854
+ the supplement into ``parsed``; the main analysis body stays
2855
+ 100% LLM-authored so readers can tell the two apart.
2856
+
2857
+ Returns ``None`` when there are no retrieval hits to ground
2858
+ the extraction in, or when the LLM call / JSON parse fails.
2859
+ """
2860
+ import json
2861
+
2862
+ from metacouplingllm.knowledge.websearch import _extract_json_object
2863
+ from metacouplingllm.llm.client import Message
2864
+
2865
+ if not self._last_rag_hits:
2866
+ return None
2867
+
2868
+ # --- Build the passages block -----------------------------------
2869
+ passage_lines: list[str] = []
2870
+ for idx, r in enumerate(self._last_rag_hits, 1):
2871
+ chunk = r.chunk
2872
+ authors = chunk.authors or ""
2873
+ if len(authors) > 60:
2874
+ authors = authors.split(" and ")[0] + " et al."
2875
+ # Pass the FULL chunk text — the chunker already caps
2876
+ # chunks at ~250 words (~1600 chars median, <2000 chars at
2877
+ # p99, <5000 chars at max). The prior 600-char truncation
2878
+ # here was cutting off bilateral country data that lives
2879
+ # past the section-opening summary (e.g., Korea/Japan/
2880
+ # Russia MtCO2 values at position ~689 in long Results
2881
+ # chunks). Full passages let the extraction LLM find
2882
+ # specific partner countries with quantitative evidence.
2883
+ text = (chunk.text or "").strip().replace("\n", " ")
2884
+ passage_lines.append(
2885
+ f"[{idx}] {authors} ({chunk.year}) — "
2886
+ f"{chunk.paper_title[:80]}:\n {text}"
2887
+ )
2888
+ passages_block = "\n\n".join(passage_lines)
2889
+
2890
+ # --- Build a concise summary of the current draft ---------------
2891
+ draft_lines: list[str] = []
2892
+ if parsed.coupling_classification:
2893
+ draft_lines.append(
2894
+ "Coupling classification (truncated):\n"
2895
+ + parsed.coupling_classification[:500]
2896
+ )
2897
+ role_aliases = {
2898
+ "sending": ("sending", "focal"),
2899
+ "receiving": ("receiving",),
2900
+ "spillover": ("spillover", "adjacent"),
2901
+ }
2902
+ for label, candidate_roles in role_aliases.items():
2903
+ entry = None
2904
+ for role in candidate_roles:
2905
+ entry = parsed.get_first_system_entry(role)
2906
+ if entry:
2907
+ break
2908
+ if entry is not None:
2909
+ bits = [
2910
+ f"name={entry.get('name', '')!r}",
2911
+ f"human={entry.get('human_subsystem', '')[:120]!r}",
2912
+ f"natural={entry.get('natural_subsystem', '')[:120]!r}",
2913
+ f"geographic_scope={entry.get('geographic_scope', '')[:120]!r}",
2914
+ ]
2915
+ draft_lines.append(f"{label.title()} system: " + ", ".join(bits))
2916
+ else:
2917
+ draft_lines.append(f"{label.title()} system: (empty)")
2918
+ parsed_flows = list(parsed.iter_flow_entries())
2919
+ if parsed_flows:
2920
+ draft_lines.append("Existing flows:")
2921
+ for flow in parsed_flows[:12]:
2922
+ cat = flow.get("category", "?")
2923
+ direction = flow.get("direction", "?")
2924
+ desc = flow.get("description", "")[:100]
2925
+ draft_lines.append(f" - [{cat}] {direction}: {desc}")
2926
+ else:
2927
+ draft_lines.append("Existing flows: (none)")
2928
+ draft_summary = "\n".join(draft_lines)
2929
+
2930
+ # --- Schema shown to the LLM ------------------------------------
2931
+ schema = {
2932
+ "additional_sending_mentions": [
2933
+ {
2934
+ "name": "short descriptive name",
2935
+ "evidence_passage_ids": [1],
2936
+ }
2937
+ ],
2938
+ "additional_receiving_mentions": [
2939
+ {
2940
+ "name": "short descriptive name",
2941
+ "evidence_passage_ids": [1],
2942
+ }
2943
+ ],
2944
+ "additional_spillover_mentions": [
2945
+ {
2946
+ "name": "short descriptive name",
2947
+ "evidence_passage_ids": [1],
2948
+ }
2949
+ ],
2950
+ "sending_subsystem_fills": {
2951
+ "human_subsystem": "fill if draft field is empty, else null",
2952
+ "natural_subsystem": "fill if draft field is empty, else null",
2953
+ "geographic_scope": "fill if draft field is empty, else null",
2954
+ "evidence_passage_ids": [1],
2955
+ },
2956
+ "receiving_subsystem_fills": {
2957
+ "human_subsystem": None,
2958
+ "natural_subsystem": None,
2959
+ "geographic_scope": None,
2960
+ "evidence_passage_ids": [1],
2961
+ },
2962
+ "spillover_subsystem_fills": {
2963
+ "human_subsystem": None,
2964
+ "natural_subsystem": None,
2965
+ "geographic_scope": None,
2966
+ "evidence_passage_ids": [1],
2967
+ },
2968
+ "supplementary_flows": [
2969
+ {
2970
+ "category": "matter|capital|information|energy|people|organisms",
2971
+ "direction": "Source → Target",
2972
+ "description": "short flow description",
2973
+ "evidence_passage_ids": [1, 2],
2974
+ }
2975
+ ],
2976
+ }
2977
+
2978
+ system_text = (
2979
+ "You review a draft metacoupling analysis against its "
2980
+ "source research passages and identify additional systems "
2981
+ "or flows the draft may have missed, and empty subfields "
2982
+ "that the passages can fill. You NEVER invent information "
2983
+ "that is not supported by the passages, and you NEVER "
2984
+ "repeat items already in the draft. Return JSON only."
2985
+ )
2986
+
2987
+ query = (self._original_query or "").strip()
2988
+ user_text = (
2989
+ (f"## Research query\n{query}\n\n" if query else "")
2990
+ + "## Retrieved passages\n"
2991
+ + passages_block
2992
+ + "\n\n## Draft analysis (extracted fields)\n"
2993
+ + draft_summary
2994
+ + "\n\n## Task\n"
2995
+ + "Look at the retrieved passages and identify content that "
2996
+ "the draft analysis missed or under-specified. Do NOT repeat "
2997
+ "items already present in the draft.\n\n"
2998
+ + "1. For each role (sending, receiving, spillover), list any "
2999
+ "ADDITIONAL system mentions that appear in the passages but "
3000
+ "are NOT in the draft's system identification. Use short "
3001
+ "descriptive names (1-8 words) and cite the passage IDs that "
3002
+ "support each mention.\n\n"
3003
+ + "2. For each role, look at the draft's subsystem fields "
3004
+ "(human_subsystem, natural_subsystem, geographic_scope). If "
3005
+ "the draft field is EMPTY, fill it using content from the "
3006
+ "passages (1-2 short sentences). Set the value to null if "
3007
+ "the draft already has content or the passages do not "
3008
+ "address that subsystem. Include the evidence passage IDs.\n\n"
3009
+ + "3. List SUPPLEMENTARY flows — flows mentioned in the "
3010
+ "passages that are NOT already in the draft's existing "
3011
+ "flows. Each flow must have a category from "
3012
+ "(matter, capital, information, energy, people, organisms), "
3013
+ "a direction in 'Source → Target' form, a short "
3014
+ "description, and at least one evidence passage ID.\n\n"
3015
+ + "Rules:\n"
3016
+ + "- Use ONLY information supported by the retrieved "
3017
+ "passages (not your training knowledge).\n"
3018
+ + "- Every item MUST include evidence_passage_ids pointing "
3019
+ f"at passage numbers 1..{len(self._last_rag_hits)}.\n"
3020
+ + "- NEVER repeat items already in the draft.\n"
3021
+ + "- Use null or [] when there is nothing to add for a "
3022
+ "field.\n"
3023
+ + "- When a passage lists SPECIFIC COUNTRIES with numeric "
3024
+ "values (e.g., 'Korea (2.65 MtCO2), Japan (1.92 MtCO2), "
3025
+ "Russian Federation (1.46 MtCO2)', or 'Thailand (2.10 Mt), "
3026
+ "Japan (0.80 Mt) and United States (0.66 MtCO2) are the "
3027
+ "three largest contributors'), emit EACH named country as "
3028
+ "its own separate entry in the appropriate "
3029
+ "additional_{sending,receiving,spillover}_mentions list. "
3030
+ "Do NOT collapse them into a grouped abstraction like "
3031
+ "'Pacific Rim countries', 'major origin countries', or "
3032
+ "'top three countries' — that loses the country-level "
3033
+ "detail the map needs. When a numeric value is attached, "
3034
+ "include it parenthetically in the name field, e.g.:\n"
3035
+ " {\"name\": \"Korea (2.65 MtCO2 inbound tourism)\",\n"
3036
+ " \"evidence_passage_ids\": [6]}\n"
3037
+ " {\"name\": \"Japan (1.92 MtCO2 inbound tourism)\",\n"
3038
+ " \"evidence_passage_ids\": [6]}\n"
3039
+ " This rule applies ONLY to country / region names "
3040
+ "(sovereign states, territories, ADM1 regions). For "
3041
+ "non-geographic groupings (e.g., 'airlines', "
3042
+ "'hospitality sector') the grouped form is fine.\n"
3043
+ + "- Keep at most 12 items per additional_*_mentions list "
3044
+ "(raised from 6 so per-country entries have room) and "
3045
+ "at most 12 supplementary_flows.\n\n"
3046
+ + f"Return JSON matching this schema exactly:\n"
3047
+ + json.dumps(schema, ensure_ascii=True, indent=2)
3048
+ )
3049
+
3050
+ try:
3051
+ response = self._client.chat(
3052
+ messages=[
3053
+ Message(role="system", content=system_text),
3054
+ Message(role="user", content=user_text),
3055
+ ],
3056
+ temperature=0.0,
3057
+ max_tokens=self._max_tokens,
3058
+ )
3059
+ except Exception as exc:
3060
+ logger.warning(
3061
+ "Structured extraction LLM call failed: %s. "
3062
+ "Skipping supplement.", exc,
3063
+ )
3064
+ return None
3065
+
3066
+ raw_obj = _extract_json_object(response.content)
3067
+ if not isinstance(raw_obj, dict):
3068
+ logger.warning(
3069
+ "Structured extraction produced non-JSON response; "
3070
+ "skipping supplement."
3071
+ )
3072
+ return None
3073
+
3074
+ # --- Validate + normalise ---------------------------------------
3075
+ n_valid_ids = len(self._last_rag_hits)
3076
+ valid_categories = {
3077
+ "matter", "capital", "information",
3078
+ "energy", "people", "organisms",
3079
+ }
3080
+
3081
+ def _clean_ids(obj: object) -> list[int]:
3082
+ if not isinstance(obj, list):
3083
+ return []
3084
+ out: list[int] = []
3085
+ for item in obj:
3086
+ try:
3087
+ n = int(item)
3088
+ except (TypeError, ValueError):
3089
+ continue
3090
+ if 1 <= n <= n_valid_ids:
3091
+ out.append(n)
3092
+ # dedupe, preserve order
3093
+ seen: set[int] = set()
3094
+ dedup: list[int] = []
3095
+ for n in out:
3096
+ if n not in seen:
3097
+ seen.add(n)
3098
+ dedup.append(n)
3099
+ return dedup
3100
+
3101
+ def _clean_mentions(obj: object, cap: int) -> list[dict[str, object]]:
3102
+ if not isinstance(obj, list):
3103
+ return []
3104
+ out: list[dict[str, object]] = []
3105
+ for item in obj:
3106
+ if not isinstance(item, dict):
3107
+ continue
3108
+ name = str(item.get("name", "")).strip()
3109
+ if not name or len(name) > 200:
3110
+ continue
3111
+ ids = _clean_ids(item.get("evidence_passage_ids", []))
3112
+ if not ids:
3113
+ continue
3114
+ out.append({"name": name, "evidence_passage_ids": ids})
3115
+ if len(out) >= cap:
3116
+ break
3117
+ return out
3118
+
3119
+ def _clean_fills(obj: object) -> dict[str, object]:
3120
+ if not isinstance(obj, dict):
3121
+ return {
3122
+ "human_subsystem": None,
3123
+ "natural_subsystem": None,
3124
+ "geographic_scope": None,
3125
+ "evidence_passage_ids": [],
3126
+ }
3127
+ cleaned: dict[str, object] = {}
3128
+ for field in (
3129
+ "human_subsystem", "natural_subsystem", "geographic_scope",
3130
+ ):
3131
+ v = obj.get(field)
3132
+ if isinstance(v, str):
3133
+ v = v.strip()
3134
+ cleaned[field] = v if v else None
3135
+ else:
3136
+ cleaned[field] = None
3137
+ cleaned["evidence_passage_ids"] = _clean_ids(
3138
+ obj.get("evidence_passage_ids", []),
3139
+ )
3140
+ return cleaned
3141
+
3142
+ def _clean_flows(obj: object, cap: int) -> list[dict[str, object]]:
3143
+ if not isinstance(obj, list):
3144
+ return []
3145
+ out: list[dict[str, object]] = []
3146
+ for item in obj:
3147
+ if not isinstance(item, dict):
3148
+ continue
3149
+ cat = str(item.get("category", "")).strip().lower()
3150
+ if cat not in valid_categories:
3151
+ continue
3152
+ direction = str(item.get("direction", "")).strip()
3153
+ if not direction:
3154
+ continue
3155
+ description = str(item.get("description", "")).strip()
3156
+ if not description:
3157
+ continue
3158
+ ids = _clean_ids(item.get("evidence_passage_ids", []))
3159
+ if not ids:
3160
+ continue
3161
+ out.append({
3162
+ "category": cat,
3163
+ "direction": direction[:200],
3164
+ "description": description[:400],
3165
+ "evidence_passage_ids": ids,
3166
+ })
3167
+ if len(out) >= cap:
3168
+ break
3169
+ return out
3170
+
3171
+ # Caps match the prompt's announced limits (12 per list) so
3172
+ # per-country entries from long bilateral breakdowns aren't
3173
+ # silently trimmed.
3174
+ supplement: dict[str, object] = {
3175
+ "additional_sending_mentions": _clean_mentions(
3176
+ raw_obj.get("additional_sending_mentions", []), cap=12,
3177
+ ),
3178
+ "additional_receiving_mentions": _clean_mentions(
3179
+ raw_obj.get("additional_receiving_mentions", []), cap=12,
3180
+ ),
3181
+ "additional_spillover_mentions": _clean_mentions(
3182
+ raw_obj.get("additional_spillover_mentions", []), cap=12,
3183
+ ),
3184
+ "sending_subsystem_fills": _clean_fills(
3185
+ raw_obj.get("sending_subsystem_fills", {}),
3186
+ ),
3187
+ "receiving_subsystem_fills": _clean_fills(
3188
+ raw_obj.get("receiving_subsystem_fills", {}),
3189
+ ),
3190
+ "spillover_subsystem_fills": _clean_fills(
3191
+ raw_obj.get("spillover_subsystem_fills", {}),
3192
+ ),
3193
+ "supplementary_flows": _clean_flows(
3194
+ raw_obj.get("supplementary_flows", []), cap=12,
3195
+ ),
3196
+ }
3197
+
3198
+ if self._verbose:
3199
+ n_flows = len(supplement["supplementary_flows"]) # type: ignore[arg-type]
3200
+ n_mentions = sum(
3201
+ len(supplement[f"additional_{role}_mentions"]) # type: ignore[arg-type]
3202
+ for role in ("sending", "receiving", "spillover")
3203
+ )
3204
+ print(
3205
+ f"[MetacouplingAssistant] Structured supplement: "
3206
+ f"{n_flows} supplementary flows, "
3207
+ f"{n_mentions} additional mentions."
3208
+ )
3209
+
3210
+ return supplement
3211
+
3212
+ @staticmethod
3213
+ def _format_structured_supplement(
3214
+ supplement: dict[str, object],
3215
+ ) -> str:
3216
+ """Render a structured-supplement dict as a visible text block.
3217
+
3218
+ The block is appended to the formatted analysis output between
3219
+ the main body and the ``SUPPORTING EVIDENCE FROM LITERATURE``
3220
+ section, so the reader can tell at a glance which content came
3221
+ from the primary LLM pass and which came from the second
3222
+ structured-extraction pass.
3223
+ """
3224
+ lines: list[str] = []
3225
+ lines.append("=" * 72)
3226
+ lines.append(" SUPPLEMENTARY STRUCTURED EXTRACTION")
3227
+ lines.append(" (Additional systems / flows identified from retrieved literature)")
3228
+ lines.append("=" * 72)
3229
+ lines.append("")
3230
+
3231
+ def _cite(ids: list[int]) -> str:
3232
+ if not ids:
3233
+ return ""
3234
+ return " " + "".join(f"[{n}]" for n in ids)
3235
+
3236
+ wrote_anything = False
3237
+
3238
+ # --- Per-role additional mentions -------------------------------
3239
+ for role, heading in (
3240
+ ("sending", "Additional Sending System Mentions"),
3241
+ ("receiving", "Additional Receiving System Mentions"),
3242
+ ("spillover", "Additional Spillover System Mentions"),
3243
+ ):
3244
+ mentions = supplement.get(f"additional_{role}_mentions", [])
3245
+ if not isinstance(mentions, list) or not mentions:
3246
+ continue
3247
+ lines.append(f"{heading}:")
3248
+ for m in mentions:
3249
+ if not isinstance(m, dict):
3250
+ continue
3251
+ name = str(m.get("name", "")).strip()
3252
+ ids = m.get("evidence_passage_ids", []) or []
3253
+ if not isinstance(ids, list):
3254
+ ids = []
3255
+ if name:
3256
+ lines.append(f" - {name}{_cite(ids)}")
3257
+ wrote_anything = True
3258
+ lines.append("")
3259
+
3260
+ # --- Per-role subsystem fills -----------------------------------
3261
+ for role, heading in (
3262
+ ("sending", "Sending"),
3263
+ ("receiving", "Receiving"),
3264
+ ("spillover", "Spillover"),
3265
+ ):
3266
+ fills = supplement.get(f"{role}_subsystem_fills", {})
3267
+ if not isinstance(fills, dict):
3268
+ continue
3269
+ non_null = {
3270
+ k: v for k, v in fills.items()
3271
+ if k in ("human_subsystem", "natural_subsystem", "geographic_scope")
3272
+ and isinstance(v, str) and v
3273
+ }
3274
+ if not non_null:
3275
+ continue
3276
+ ids = fills.get("evidence_passage_ids", []) or []
3277
+ if not isinstance(ids, list):
3278
+ ids = []
3279
+ lines.append(f"{heading} System — Subsystem Fills (from literature):")
3280
+ for field, value in non_null.items():
3281
+ pretty = field.replace("_", " ").capitalize()
3282
+ lines.append(f" - {pretty}: {value}{_cite(ids)}")
3283
+ wrote_anything = True
3284
+ lines.append("")
3285
+
3286
+ # --- Supplementary flows ----------------------------------------
3287
+ supp_flows = supplement.get("supplementary_flows", [])
3288
+ if isinstance(supp_flows, list) and supp_flows:
3289
+ lines.append("Supplementary Flows:")
3290
+ for f in supp_flows:
3291
+ if not isinstance(f, dict):
3292
+ continue
3293
+ cat = str(f.get("category", "?"))
3294
+ direction = str(f.get("direction", ""))
3295
+ desc = str(f.get("description", ""))
3296
+ ids = f.get("evidence_passage_ids", []) or []
3297
+ if not isinstance(ids, list):
3298
+ ids = []
3299
+ lines.append(f" - [{cat}] {direction}: {desc}{_cite(ids)}")
3300
+ wrote_anything = True
3301
+ lines.append("")
3302
+
3303
+ if not wrote_anything:
3304
+ lines.append("(No additional systems or flows identified "
3305
+ "from the retrieved literature.)")
3306
+ lines.append("")
3307
+
3308
+ lines.append("=" * 72)
3309
+ return "\n".join(lines)
3310
+
3311
+ # ------------------------------------------------------------------
3312
+ # Structured supplement → map_data enrichment
3313
+ # ------------------------------------------------------------------
3314
+
3315
+ def _merge_supplement_into_map_data(
3316
+ self,
3317
+ parsed: ParsedAnalysis,
3318
+ supplement: dict[str, object],
3319
+ ) -> None:
3320
+ """Enrich ``parsed.map_data`` with countries/flows from the
3321
+ structured-extraction supplement.
3322
+
3323
+ ``_extract_map_data_from_analysis`` reads the finished analysis
3324
+ text, which the main LLM often writes with generic wording
3325
+ ("foreign destination countries"). The structured supplement,
3326
+ by contrast, reads the retrieved passages directly with a
3327
+ schema that requires per-country entries — so it can surface
3328
+ specific bilateral partners (Korea, Japan, Russia with MtCO2
3329
+ numbers) that the analysis abstracted away. This helper
3330
+ reconciles the two: the primary extraction stays authoritative,
3331
+ and the supplement fills in the gaps.
3332
+
3333
+ The merge is purely additive. We never DROP countries or flows
3334
+ that the primary extraction produced. We only ADD:
3335
+
3336
+ - Receiving / spillover countries the primary path missed, IF
3337
+ the supplement mention resolves to a valid ISO alpha-3 code
3338
+ (via ``resolve_country_code``).
3339
+ - Flows from supplementary_flows, parsed into source/target/
3340
+ category, IF both endpoints resolve and the pair is not
3341
+ already represented.
3342
+
3343
+ Never raises — all resolution failures are logged at DEBUG and
3344
+ that specific entry is skipped.
3345
+ """
3346
+ map_data = parsed.map_data
3347
+ if not isinstance(map_data, dict):
3348
+ return
3349
+ if not isinstance(supplement, dict):
3350
+ return
3351
+
3352
+ focal_code = map_data.get("focal_country")
3353
+ # Existing sets so we don't duplicate
3354
+ existing_recv: set[str] = set()
3355
+ for c in map_data.get("receiving_countries", []) or []:
3356
+ if isinstance(c, str):
3357
+ existing_recv.add(c.upper())
3358
+ existing_spill: set[str] = set()
3359
+ for c in map_data.get("spillover_countries", []) or []:
3360
+ if isinstance(c, str):
3361
+ existing_spill.add(c.upper())
3362
+
3363
+ # --- Receiving mentions ----------------------------------------
3364
+ for mention in supplement.get("additional_receiving_mentions", []) or []:
3365
+ if not isinstance(mention, dict):
3366
+ continue
3367
+ name = str(mention.get("name", "")).strip()
3368
+ if not name:
3369
+ continue
3370
+ code = resolve_country_code(name)
3371
+ if not code:
3372
+ logger.debug(
3373
+ "Supplement receiving mention %r did not resolve "
3374
+ "to an ISO code; skipping.", name,
3375
+ )
3376
+ continue
3377
+ if code == focal_code:
3378
+ continue # already the focal; not a separate receiving
3379
+ if code in existing_spill:
3380
+ continue # already classified as spillover; don't double-tag
3381
+ if code in existing_recv:
3382
+ continue
3383
+ map_data.setdefault("receiving_countries", []).append(code)
3384
+ existing_recv.add(code)
3385
+
3386
+ # --- Spillover mentions ----------------------------------------
3387
+ for mention in supplement.get("additional_spillover_mentions", []) or []:
3388
+ if not isinstance(mention, dict):
3389
+ continue
3390
+ name = str(mention.get("name", "")).strip()
3391
+ if not name:
3392
+ continue
3393
+ code = resolve_country_code(name)
3394
+ if not code:
3395
+ logger.debug(
3396
+ "Supplement spillover mention %r did not resolve "
3397
+ "to an ISO code; skipping.", name,
3398
+ )
3399
+ continue
3400
+ if code == focal_code:
3401
+ continue
3402
+ if code in existing_recv:
3403
+ continue
3404
+ if code in existing_spill:
3405
+ continue
3406
+ map_data.setdefault("spillover_countries", []).append(code)
3407
+ existing_spill.add(code)
3408
+
3409
+ # --- Sending mentions ------------------------------------------
3410
+ # For bidirectional phenomena (e.g., inbound tourism), a
3411
+ # "sending" country at the paper level IS effectively a partner
3412
+ # on the map. We surface them as receiving countries on the
3413
+ # map so they render as trade partners rather than being
3414
+ # dropped. The focal stays the focal.
3415
+ for mention in supplement.get("additional_sending_mentions", []) or []:
3416
+ if not isinstance(mention, dict):
3417
+ continue
3418
+ name = str(mention.get("name", "")).strip()
3419
+ if not name:
3420
+ continue
3421
+ code = resolve_country_code(name)
3422
+ if not code:
3423
+ logger.debug(
3424
+ "Supplement sending mention %r did not resolve "
3425
+ "to an ISO code; skipping.", name,
3426
+ )
3427
+ continue
3428
+ if code == focal_code:
3429
+ continue
3430
+ if code in existing_spill or code in existing_recv:
3431
+ continue
3432
+ map_data.setdefault("receiving_countries", []).append(code)
3433
+ existing_recv.add(code)
3434
+
3435
+ # --- Supplementary flows ---------------------------------------
3436
+ # Parse each flow's "Source → Target" direction into a pair of
3437
+ # ISO codes and append to map_data['flows']. Skip flows where
3438
+ # either endpoint can't be resolved, or where the same pair is
3439
+ # already present. We match the shape used by
3440
+ # ``_extract_map_data_from_analysis`` so the downstream
3441
+ # ``_resolve_flows_for_map`` pipeline can consume both
3442
+ # transparently.
3443
+ existing_flow_keys: set[tuple[str, str, str]] = set()
3444
+ for f in map_data.get("flows", []) or []:
3445
+ if not isinstance(f, dict):
3446
+ continue
3447
+ key = (
3448
+ str(f.get("source_country", "")).upper(),
3449
+ str(f.get("target_country", "")).upper(),
3450
+ str(f.get("category", "")).lower(),
3451
+ )
3452
+ existing_flow_keys.add(key)
3453
+
3454
+ for sup_flow in supplement.get("supplementary_flows", []) or []:
3455
+ if not isinstance(sup_flow, dict):
3456
+ continue
3457
+ direction = str(sup_flow.get("direction", "")).strip()
3458
+ if not direction:
3459
+ continue
3460
+ # Split on the canonical arrow regex used elsewhere
3461
+ parts = _FLOW_ARROW_RE.split(direction)
3462
+ parts = [p.strip() for p in parts if p.strip()]
3463
+ if len(parts) < 2:
3464
+ logger.debug(
3465
+ "Supplementary flow direction %r did not contain a "
3466
+ "resolvable arrow; skipping.", direction,
3467
+ )
3468
+ continue
3469
+ src_name, tgt_name = parts[0], parts[1]
3470
+ src_code = resolve_country_code(src_name)
3471
+ tgt_code = resolve_country_code(tgt_name)
3472
+ if not src_code or not tgt_code:
3473
+ logger.debug(
3474
+ "Supplementary flow %r → %r: endpoint(s) did not "
3475
+ "resolve (src=%r, tgt=%r); skipping.",
3476
+ src_name, tgt_name, src_code, tgt_code,
3477
+ )
3478
+ continue
3479
+ category = str(sup_flow.get("category", "")).strip().lower() or "matter"
3480
+ key = (src_code.upper(), tgt_code.upper(), category.lower())
3481
+ if key in existing_flow_keys:
3482
+ continue
3483
+ existing_flow_keys.add(key)
3484
+ map_data.setdefault("flows", []).append({
3485
+ "category": category,
3486
+ "source_country": src_code,
3487
+ "target_country": tgt_code,
3488
+ "direction": f"{src_code} \u2192 {tgt_code}",
3489
+ "description": str(
3490
+ sup_flow.get("description", "")
3491
+ )[:300],
3492
+ # Same confidence shape the primary extractor uses
3493
+ "kind": "proxy",
3494
+ "confidence": 0.6,
3495
+ "evidence": [],
3496
+ })
3497
+ # If the target isn't already a receiving country on the
3498
+ # map, add it — flows without visible endpoints produce
3499
+ # arrows pointing at grey countries, which is confusing.
3500
+ if (
3501
+ tgt_code != focal_code
3502
+ and tgt_code not in existing_spill
3503
+ and tgt_code not in existing_recv
3504
+ ):
3505
+ map_data.setdefault("receiving_countries", []).append(
3506
+ tgt_code,
3507
+ )
3508
+ existing_recv.add(tgt_code)
3509
+
3510
+ def _generate_map(self, parsed: ParsedAnalysis) -> Figure | None:
3511
+ """Generate the appropriate map based on analysis level.
3512
+
3513
+ **Primary path**: uses ``parsed.map_data`` (from a second LLM
3514
+ call) which contains validated ISO codes and flow endpoints.
3515
+
3516
+ **Fallback path**: regex extraction from analysis text (legacy).
3517
+
3518
+ Returns ``None`` if nothing can be resolved or deps are missing.
3519
+ """
3520
+ self._last_map_notice = None
3521
+ try:
3522
+ if self._has_unsupported_automap_scope(parsed):
3523
+ self._last_map_notice = self._format_map_unavailable_notice(
3524
+ "unsupported_local_geography"
3525
+ )
3526
+ if self._verbose:
3527
+ print(
3528
+ "[MetacouplingAssistant] Map generation skipped: "
3529
+ "the focal geography is below country/ADM1 scale."
3530
+ )
3531
+ return None
3532
+
3533
+ # ==============================================================
3534
+ # PRIMARY PATH: structured map data from second LLM call
3535
+ # ==============================================================
3536
+ md = parsed.map_data
3537
+ if md and isinstance(md, dict) and md.get("focal_country"):
3538
+ focal_code = md["focal_country"]
3539
+ adm1_region = md.get("adm1_region")
3540
+ receiving = md.get("receiving_countries", [])
3541
+ # Spillover countries are retained in map_data for
3542
+ # downstream use but intentionally NOT used here —
3543
+ # they render as grey (NA) to avoid conflating them
3544
+ # with actual trade partners.
3545
+ raw_flows = md.get("flows", [])
3546
+
3547
+ # ADM1 regions explicitly mentioned in the analysis.
3548
+ # The LLM provides this list (see extraction prompt);
3549
+ # when missing or empty, fall back to a regex scan.
3550
+ mentioned_adm1_list = md.get("mentioned_adm1_regions", [])
3551
+ if not mentioned_adm1_list:
3552
+ mentioned_adm1_list = sorted(
3553
+ self._extract_mentioned_adm1_from_text(parsed)
3554
+ )
3555
+ mentioned_adm1_set: set[str] = set(mentioned_adm1_list)
3556
+ # The focal ADM1 is handled separately by the classifier
3557
+ # (it gets "intracoupling"); drop it from the set to
3558
+ # keep the log output clean and the set minimal.
3559
+ if adm1_region:
3560
+ mentioned_adm1_set.discard(adm1_region)
3561
+
3562
+ # Build flow dicts for the renderer
3563
+ map_flows: list[dict[str, str]] = []
3564
+ for f in raw_flows:
3565
+ if isinstance(f, dict) and f.get("direction"):
3566
+ map_flows.append({
3567
+ "category": str(f.get("category", "")),
3568
+ "direction": str(f["direction"]),
3569
+ "description": str(f.get("description", "")),
3570
+ })
3571
+
3572
+ # Merge with web-structured flows if available
3573
+ map_flows = self._merge_map_flows(
3574
+ map_flows,
3575
+ self._structured_web_flow_dicts(),
3576
+ )
3577
+
3578
+ # Collect countries that should be highlighted as
3579
+ # telecoupling on the map. Only focal + receiving
3580
+ # belong here. Spillover countries (competitors,
3581
+ # indirectly affected) render as grey (NA) to avoid
3582
+ # visually conflating them with actual trade partners.
3583
+ mentioned = (
3584
+ {focal_code}
3585
+ | set(receiving)
3586
+ | self._structured_web_receiving_codes()
3587
+ )
3588
+
3589
+ # Drop flows whose endpoints don't belong on the map.
3590
+ # 1. Flow TARGETS must be in mentioned countries.
3591
+ # 2. Flow SOURCES must be the focal country or a
3592
+ # receiving country — never a spillover country.
3593
+ # Spillover countries are competitors/indirectly
3594
+ # affected; arrows FROM them (e.g. Brazil → Mexico)
3595
+ # don't belong on a map about the focal country's
3596
+ # trade.
3597
+ if map_flows:
3598
+ from metacouplingllm.knowledge.countries import (
3599
+ resolve_country_code as _rcc,
3600
+ )
3601
+
3602
+ # Valid flow sources: focal + receiving (not spillover)
3603
+ _valid_sources = {focal_code} | set(receiving)
3604
+
3605
+ def _flow_endpoints_valid(f: dict) -> bool:
3606
+ d = f.get("direction", "")
3607
+ parts = _FLOW_ARROW_RE.split(d)
3608
+ if len(parts) < 2:
3609
+ return True # can't validate → keep
3610
+
3611
+ # Validate target
3612
+ tgt_name = parts[-1].strip().rstrip(")")
3613
+ tgt_code = _rcc(tgt_name)
3614
+ if tgt_code and tgt_code not in mentioned:
3615
+ return False
3616
+
3617
+ # Validate source: must be focal or receiving
3618
+ src_name = parts[0].strip().lstrip("(")
3619
+ src_code = _rcc(src_name)
3620
+ if src_code and src_code not in _valid_sources:
3621
+ return False
3622
+
3623
+ return True
3624
+
3625
+ map_flows = [
3626
+ f for f in map_flows
3627
+ if _flow_endpoints_valid(f)
3628
+ ]
3629
+
3630
+ # Decide: ADM1 or country-level map?
3631
+ if adm1_region:
3632
+ try:
3633
+ from metacouplingllm.visualization.adm1_map import (
3634
+ plot_focal_adm1_map,
3635
+ )
3636
+
3637
+ print(
3638
+ f"[MetacouplingAssistant] Structured map → "
3639
+ f"ADM1 {adm1_region}, "
3640
+ f"{len(map_flows)} flows, "
3641
+ f"mentioned_adm1={sorted(mentioned_adm1_set)}."
3642
+ )
3643
+ return plot_focal_adm1_map(
3644
+ adm1_region,
3645
+ shapefile=self._adm1_shapefile,
3646
+ mentioned_countries=(
3647
+ mentioned if mentioned else None
3648
+ ),
3649
+ # Always pass the set, even when empty.
3650
+ # An empty set means "strict mode, no
3651
+ # regions qualify" — NOT "fall back to
3652
+ # legacy coloring of all DB neighbors".
3653
+ mentioned_adm1_codes=mentioned_adm1_set,
3654
+ flows=map_flows if map_flows else None,
3655
+ )
3656
+ except Exception as exc:
3657
+ if self._verbose:
3658
+ print(
3659
+ "[MetacouplingAssistant] Structured ADM1 "
3660
+ f"map failed: {exc}"
3661
+ )
3662
+ # Fall through to country-level
3663
+
3664
+ # Country-level map from structured data
3665
+ from metacouplingllm.visualization.worldmap import (
3666
+ plot_analysis_map,
3667
+ )
3668
+
3669
+ print(
3670
+ f"[MetacouplingAssistant] Structured map → "
3671
+ f"focal={focal_code}, "
3672
+ f"receiving={receiving}, "
3673
+ f"{len(map_flows)} flows."
3674
+ )
3675
+ return plot_analysis_map(
3676
+ parsed,
3677
+ adm0_shapefile=self._adm0_shapefile,
3678
+ focal_code_override=focal_code,
3679
+ mentioned_countries_override=mentioned,
3680
+ flows=map_flows if map_flows else None,
3681
+ )
3682
+
3683
+ # ==============================================================
3684
+ # FALLBACK PATH: regex extraction (legacy)
3685
+ # ==============================================================
3686
+ if md is not None:
3687
+ # map_data existed but was invalid — already warned
3688
+ pass
3689
+ else:
3690
+ print(
3691
+ "[MetacouplingAssistant] No structured map data — "
3692
+ "falling back to text extraction."
3693
+ )
3694
+
3695
+ # Legacy Attempt 1: Try subnational (ADM1) map
3696
+ adm1_code = self._resolve_adm1_from_analysis(parsed)
3697
+ if adm1_code:
3698
+ try:
3699
+ from metacouplingllm.visualization.adm1_map import (
3700
+ plot_focal_adm1_map,
3701
+ )
3702
+ from metacouplingllm.visualization.worldmap import (
3703
+ _extract_all_analysis_countries,
3704
+ )
3705
+
3706
+ all_role_codes = _extract_all_analysis_countries(parsed)
3707
+ mentioned_legacy: set[str] = set()
3708
+ mentioned_legacy.update(
3709
+ all_role_codes.get("sending", set())
3710
+ )
3711
+ mentioned_legacy.update(
3712
+ all_role_codes.get("receiving", set())
3713
+ )
3714
+ mentioned_legacy.update(
3715
+ all_role_codes.get("other", set())
3716
+ )
3717
+ # Skip spillover — receiving partners only.
3718
+ mentioned_legacy.update(
3719
+ self._structured_web_receiving_codes()
3720
+ )
3721
+
3722
+ from metacouplingllm.knowledge.adm1_pericoupling import (
3723
+ get_adm1_country,
3724
+ )
3725
+
3726
+ focal_country = get_adm1_country(adm1_code) or ""
3727
+ parsed_flows = list(parsed.iter_flow_entries())
3728
+ map_flows_legacy = (
3729
+ self._resolve_flows_for_adm1_map(
3730
+ parsed, adm1_code, focal_country,
3731
+ )
3732
+ if parsed_flows
3733
+ else []
3734
+ )
3735
+ map_flows_legacy = self._merge_map_flows(
3736
+ map_flows_legacy,
3737
+ self._structured_web_flow_dicts(),
3738
+ )
3739
+
3740
+ # Regex fallback for mentioned ADM1 regions.
3741
+ mentioned_adm1_legacy = (
3742
+ self._extract_mentioned_adm1_from_text(parsed)
3743
+ )
3744
+ # The focal ADM1 is handled separately by the
3745
+ # classifier via ``focal_code``; drop it here.
3746
+ mentioned_adm1_legacy.discard(adm1_code)
3747
+
3748
+ if self._verbose:
3749
+ print(
3750
+ f"[MetacouplingAssistant] Legacy ADM1 map for "
3751
+ f"{adm1_code}, flows={len(map_flows_legacy)}, "
3752
+ f"mentioned_adm1="
3753
+ f"{sorted(mentioned_adm1_legacy)}."
3754
+ )
3755
+ return plot_focal_adm1_map(
3756
+ adm1_code,
3757
+ shapefile=self._adm1_shapefile,
3758
+ mentioned_countries=(
3759
+ mentioned_legacy if mentioned_legacy else None
3760
+ ),
3761
+ # Always pass — empty set means strict mode
3762
+ # with no matches, NOT legacy fallback.
3763
+ mentioned_adm1_codes=mentioned_adm1_legacy,
3764
+ flows=(
3765
+ map_flows_legacy if map_flows_legacy else None
3766
+ ),
3767
+ )
3768
+ except Exception as exc:
3769
+ if self._verbose:
3770
+ print(
3771
+ f"[MetacouplingAssistant] Legacy ADM1 map "
3772
+ f"failed: {exc}"
3773
+ )
3774
+
3775
+ # Legacy Attempt 2: Country-level map
3776
+ from metacouplingllm.visualization.worldmap import (
3777
+ _extract_all_analysis_countries,
3778
+ _extract_countries_from_analysis,
3779
+ plot_analysis_map,
3780
+ )
3781
+
3782
+ if self._verbose:
3783
+ print(
3784
+ "[MetacouplingAssistant] Legacy country-level map..."
3785
+ )
3786
+ focal_code = ""
3787
+ all_role_codes = _extract_all_analysis_countries(parsed)
3788
+ sending_codes = all_role_codes.get("sending", set())
3789
+ focal_codes = all_role_codes.get("focal", set())
3790
+ if sending_codes:
3791
+ focal_code = sorted(sending_codes)[0]
3792
+ elif focal_codes:
3793
+ focal_code = sorted(focal_codes)[0]
3794
+ if not focal_code:
3795
+ focal_code = (
3796
+ _extract_countries_from_analysis(parsed).get(
3797
+ "sending"
3798
+ )
3799
+ or _extract_countries_from_analysis(parsed).get("focal")
3800
+ or ""
3801
+ )
3802
+ if not focal_code and self._last_web_map_signals:
3803
+ structured_focal = self._last_web_map_signals.get(
3804
+ "focal_country"
3805
+ )
3806
+ if isinstance(structured_focal, str):
3807
+ focal_code = structured_focal
3808
+ if not focal_code:
3809
+ self._last_map_notice = self._infer_unavailable_map_notice(
3810
+ parsed
3811
+ )
3812
+ if self._verbose:
3813
+ print(
3814
+ "[MetacouplingAssistant] Map generation skipped: "
3815
+ "no country or ADM1 focal geography could be "
3816
+ "resolved."
3817
+ )
3818
+ return None
3819
+ parsed_flows = list(parsed.iter_flow_entries())
3820
+ map_flows_legacy = (
3821
+ self._resolve_flows_for_map(parsed, focal_code)
3822
+ if parsed_flows and focal_code
3823
+ else []
3824
+ )
3825
+ map_flows_legacy = self._merge_map_flows(
3826
+ map_flows_legacy,
3827
+ self._structured_web_flow_dicts(),
3828
+ )
3829
+ if self._verbose:
3830
+ print(
3831
+ f"[MetacouplingAssistant] Legacy map focal="
3832
+ f"{focal_code or 'unknown'}, "
3833
+ f"flows={len(map_flows_legacy)}."
3834
+ )
3835
+ return plot_analysis_map(
3836
+ parsed,
3837
+ adm0_shapefile=self._adm0_shapefile,
3838
+ # Skip spillover — they render as grey (NA).
3839
+ extra_mentioned_countries=(
3840
+ self._structured_web_receiving_codes()
3841
+ ),
3842
+ flows=(
3843
+ map_flows_legacy if map_flows_legacy else None
3844
+ ),
3845
+ )
3846
+
3847
+ except ImportError:
3848
+ self._last_map_notice = self._format_map_unavailable_notice(
3849
+ "missing_dependencies"
3850
+ )
3851
+ if self._verbose:
3852
+ print(
3853
+ "[MetacouplingAssistant] Map generation skipped "
3854
+ "(geopandas/matplotlib not installed)."
3855
+ )
3856
+ return None
3857
+ except Exception as exc:
3858
+ self._last_map_notice = self._format_map_unavailable_notice(
3859
+ "generation_error"
3860
+ )
3861
+ if self._verbose:
3862
+ print(
3863
+ f"[MetacouplingAssistant] Map generation failed: {exc}"
3864
+ )
3865
+ return None
3866
+
3867
+ def _structured_web_country_codes(self) -> set[str]:
3868
+ """Return validated country codes extracted from structured web hints.
3869
+
3870
+ Includes focal, receiving, and spillover systems. Prefer
3871
+ :meth:`_structured_web_receiving_codes` and
3872
+ :meth:`_structured_web_spillover_codes` when callers need to
3873
+ distinguish the two roles (e.g., to colour them differently).
3874
+ """
3875
+ return (
3876
+ self._structured_web_receiving_codes()
3877
+ | self._structured_web_spillover_codes()
3878
+ )
3879
+
3880
+ def _structured_web_receiving_codes(self) -> set[str]:
3881
+ """Return validated receiving-system codes from web hints.
3882
+
3883
+ Also includes the focal country so that it can be used as a
3884
+ map-ready "mentioned countries" set.
3885
+ """
3886
+ signals = self._last_web_map_signals
3887
+ if not signals:
3888
+ return set()
3889
+
3890
+ codes: set[str] = set()
3891
+ focal = signals.get("focal_country")
3892
+ if isinstance(focal, str) and focal:
3893
+ codes.add(focal)
3894
+ items = signals.get("receiving_systems", [])
3895
+ if isinstance(items, list):
3896
+ for item in items:
3897
+ if not isinstance(item, dict):
3898
+ continue
3899
+ code = item.get("country")
3900
+ if isinstance(code, str) and code:
3901
+ codes.add(code)
3902
+ return codes
3903
+
3904
+ def _structured_web_spillover_codes(self) -> set[str]:
3905
+ """Return validated spillover-system codes from web hints."""
3906
+ signals = self._last_web_map_signals
3907
+ if not signals:
3908
+ return set()
3909
+
3910
+ codes: set[str] = set()
3911
+ items = signals.get("spillover_systems", [])
3912
+ if isinstance(items, list):
3913
+ for item in items:
3914
+ if not isinstance(item, dict):
3915
+ continue
3916
+ code = item.get("country")
3917
+ if isinstance(code, str) and code:
3918
+ codes.add(code)
3919
+ return codes
3920
+
3921
+ def _structured_web_flow_dicts(self) -> list[dict[str, str]]:
3922
+ """Return structured web-extracted flows as map-ready dicts."""
3923
+ signals = self._last_web_map_signals
3924
+ if not signals:
3925
+ return []
3926
+
3927
+ flows = signals.get("flows", [])
3928
+ if not isinstance(flows, list):
3929
+ return []
3930
+
3931
+ result: list[dict[str, str]] = []
3932
+ for item in flows:
3933
+ if not isinstance(item, dict):
3934
+ continue
3935
+ direction = item.get("direction")
3936
+ category = item.get("category")
3937
+ if not isinstance(direction, str) or not direction:
3938
+ continue
3939
+ if not isinstance(category, str) or not category:
3940
+ continue
3941
+ result.append(
3942
+ {
3943
+ "category": category,
3944
+ "direction": direction,
3945
+ "description": str(item.get("description", "")).strip(),
3946
+ }
3947
+ )
3948
+ return result
3949
+
3950
+ @staticmethod
3951
+ def _merge_map_flows(
3952
+ primary: list[dict[str, str]] | None,
3953
+ secondary: list[dict[str, str]] | None,
3954
+ ) -> list[dict[str, str]]:
3955
+ """Merge map flow lists without duplicating category-direction pairs."""
3956
+ merged: list[dict[str, str]] = []
3957
+ seen: set[tuple[str, str]] = set()
3958
+ for flow_group in (primary or [], secondary or []):
3959
+ for flow in flow_group:
3960
+ if not isinstance(flow, dict):
3961
+ continue
3962
+ category = str(flow.get("category", "")).strip().lower()
3963
+ direction = str(flow.get("direction", "")).strip()
3964
+ if not category or not direction:
3965
+ continue
3966
+ key = (category, direction)
3967
+ if key in seen:
3968
+ continue
3969
+ seen.add(key)
3970
+ merged.append(flow)
3971
+ return merged
3972
+
3973
+ @staticmethod
3974
+ def _format_map_notice(map_type: str) -> str:
3975
+ """Return a brief notice to append to formatted output."""
3976
+ if map_type == "adm1":
3977
+ return (
3978
+ "\n\n---\n"
3979
+ "📍 An ADM1 (subnational) metacoupling map has been "
3980
+ "generated. Access it via `result.map`."
3981
+ )
3982
+ return (
3983
+ "\n\n---\n"
3984
+ "🗺️ A country-level metacoupling map has been generated. "
3985
+ "Access it via `result.map`."
3986
+ )
3987
+
3988
+ @staticmethod
3989
+ def _format_map_unavailable_notice(reason: str) -> str:
3990
+ """Return a brief notice when auto-map could not produce a figure."""
3991
+ if reason == "unsupported_local_geography":
3992
+ return (
3993
+ "\n\n---\n"
3994
+ "Auto-map did not generate a figure. It currently supports "
3995
+ "only countries and ADM1 subnational regions, not city, "
3996
+ "watershed, protected-area, reserve, or park geometries. "
3997
+ "Specify the parent ADM1 region and country only if an "
3998
+ "ADM1-level proxy map is acceptable."
3999
+ )
4000
+ if reason == "missing_dependencies":
4001
+ return (
4002
+ "\n\n---\n"
4003
+ "Auto-map did not generate a figure because the map "
4004
+ "dependencies are not installed. Install geopandas and "
4005
+ "matplotlib to enable map output."
4006
+ )
4007
+ if reason == "generation_error":
4008
+ return (
4009
+ "\n\n---\n"
4010
+ "Auto-map did not generate a figure. It currently supports "
4011
+ "only countries and ADM1 subnational regions, and this "
4012
+ "analysis did not resolve cleanly to one of those map types."
4013
+ )
4014
+ return (
4015
+ "\n\n---\n"
4016
+ "Auto-map did not generate a figure. It currently supports only "
4017
+ "countries and ADM1 subnational regions. Specify a resolvable "
4018
+ "country or ADM1 region to get a map."
4019
+ )
4020
+
4021
+ def _infer_unavailable_map_notice(self, parsed: ParsedAnalysis) -> str:
4022
+ """Infer the most useful notice when auto-map cannot render."""
4023
+ scope_parts = [self._original_query or ""]
4024
+ for role in ("focal", "adjacent", "sending", "receiving", "spillover"):
4025
+ scope_parts.append(parsed.get_system_detail(role, "name"))
4026
+ scope_parts.append(parsed.get_system_detail(role, "geographic_scope"))
4027
+ combined = " ".join(part for part in scope_parts if part)
4028
+ if _UNSUPPORTED_AUTOMAP_SCOPE_RE.search(combined):
4029
+ return self._format_map_unavailable_notice(
4030
+ "unsupported_local_geography"
4031
+ )
4032
+ return self._format_map_unavailable_notice(
4033
+ "unresolved_supported_geography"
4034
+ )
4035
+
4036
+ def _has_unsupported_automap_scope(self, parsed: ParsedAnalysis) -> bool:
4037
+ """Return True when the topic is below country/ADM1 map support."""
4038
+ scope_parts = [self._original_query or ""]
4039
+ for role in ("focal", "adjacent", "sending", "receiving", "spillover"):
4040
+ scope_parts.append(parsed.get_system_detail(role, "name"))
4041
+ scope_parts.append(parsed.get_system_detail(role, "geographic_scope"))
4042
+ combined = " ".join(part for part in scope_parts if part)
4043
+ return bool(_UNSUPPORTED_AUTOMAP_SCOPE_RE.search(combined))
4044
+
4045
+ @staticmethod
4046
+ def _format_web_sources(
4047
+ results: list[dict[str, str]],
4048
+ ) -> str:
4049
+ """Format web search results as a references section."""
4050
+ if not results:
4051
+ return ""
4052
+ lines = [
4053
+ "\n\n======================================================================",
4054
+ " WEB SOURCES",
4055
+ "======================================================================\n",
4056
+ ]
4057
+ for i, r in enumerate(results, 1):
4058
+ title = r.get("title", "Untitled")
4059
+ url = r.get("url", "")
4060
+ snippet = r.get("snippet", "")
4061
+ lines.append(f" [W{i}] {title}")
4062
+ if url:
4063
+ lines.append(f" {url}")
4064
+ if snippet:
4065
+ lines.append(f" {snippet[:200]}")
4066
+ lines.append("")
4067
+ return "\n".join(lines)
4068
+
4069
+ # ------------------------------------------------------------------
4070
+ # Result builder
4071
+ # ------------------------------------------------------------------
4072
+
4073
+ def _build_result(self, response: LLMResponse) -> AnalysisResult:
4074
+ """Parse and format an LLM response into an AnalysisResult."""
4075
+ # Pre-retrieval citation sanitization. Run BEFORE parse_analysis()
4076
+ # so the parser never sees out-of-range tokens — the parser
4077
+ # preserves [N] tokens through structured field extraction, so
4078
+ # any [99] etc. would otherwise survive into the final output.
4079
+ # The sanitizer also runs an idempotent whitespace/punctuation
4080
+ # cleanup pass so the result reads naturally even when tokens
4081
+ # were stripped. We log here in addition to the citations module
4082
+ # so debug visibility lives close to the call site.
4083
+ if (
4084
+ self._rag_mode == "pre_retrieval"
4085
+ and self._last_rag_hits is not None
4086
+ ):
4087
+ n_valid = len(self._last_rag_hits)
4088
+ sanitized, invalid_ids = sanitize_citations(
4089
+ response.content, n_valid=n_valid
4090
+ )
4091
+ if invalid_ids:
4092
+ logger.warning(
4093
+ "MetacouplingAssistant sanitized %d invalid citation "
4094
+ "ID(s) from LLM response: %s (valid range: 1..%d)",
4095
+ len(invalid_ids),
4096
+ sorted(invalid_ids),
4097
+ n_valid,
4098
+ )
4099
+ response = LLMResponse(content=sanitized, usage=response.usage)
4100
+
4101
+ parsed = parse_analysis(response.content)
4102
+
4103
+ # Post-LLM pericoupling validation
4104
+ self._validate_pericoupling(parsed)
4105
+
4106
+ formatted = self._formatter.format_full(parsed)
4107
+
4108
+ # Optional: append literature recommendations
4109
+ if self._recommend_papers:
4110
+ from metacouplingllm.knowledge.literature import (
4111
+ format_recommendations,
4112
+ recommend_papers as _recommend,
4113
+ )
4114
+
4115
+ papers = _recommend(parsed, max_results=self._max_recommendations)
4116
+ if papers:
4117
+ formatted += "\n\n" + format_recommendations(papers)
4118
+
4119
+ # RAG evidence assembly. Two modes:
4120
+ #
4121
+ # - "post_hoc" (legacy): retrieve from corpus AFTER LLM response,
4122
+ # keyword-annotate citations onto the formatted text, then
4123
+ # append the evidence block. Code path unchanged from
4124
+ # pre-upgrade behavior.
4125
+ #
4126
+ # - "pre_retrieval": passages were already retrieved at
4127
+ # analyze() / refine() time and stored on
4128
+ # self._last_rag_hits. The LLM has already cited them inline
4129
+ # (subject to sanitization above). We just need to append
4130
+ # the human-readable evidence block — same format as post_hoc
4131
+ # mode so downstream consumers see no shape change.
4132
+ structured_supplement: dict[str, object] | None = None
4133
+ if self._rag_engine is not None:
4134
+ if self._rag_mode == "pre_retrieval":
4135
+ # Optional: second LLM call that scans the retrieved
4136
+ # passages for systems / flows the draft missed. The
4137
+ # supplement is rendered as its own visible block so
4138
+ # the main analysis body stays 100% LLM-authored.
4139
+ if (
4140
+ self._rag_structured_extraction
4141
+ and self._last_rag_hits
4142
+ ):
4143
+ try:
4144
+ structured_supplement = (
4145
+ self._structured_extract_supplement(parsed)
4146
+ )
4147
+ except Exception as exc:
4148
+ logger.warning(
4149
+ "Structured extraction supplement failed: "
4150
+ "%s. Skipping supplement.", exc,
4151
+ )
4152
+ structured_supplement = None
4153
+
4154
+ if structured_supplement is not None:
4155
+ try:
4156
+ formatted += "\n\n" + self._format_structured_supplement(
4157
+ structured_supplement
4158
+ )
4159
+ except Exception as exc:
4160
+ if self._verbose:
4161
+ print(
4162
+ f"[MetacouplingAssistant] Structured "
4163
+ f"supplement formatting failed: {exc}"
4164
+ )
4165
+
4166
+ try:
4167
+ from metacouplingllm.knowledge.rag import format_evidence
4168
+
4169
+ if self._last_rag_hits:
4170
+ rag_backend = (
4171
+ self._rag_engine.backend
4172
+ if self._rag_engine is not None
4173
+ else "tfidf"
4174
+ )
4175
+ formatted += "\n\n" + format_evidence(
4176
+ self._last_rag_hits,
4177
+ anchor_text=self._original_query or "",
4178
+ backend=rag_backend or "tfidf",
4179
+ )
4180
+ elif self._verbose:
4181
+ print(
4182
+ "[MetacouplingAssistant] Pre-retrieval RAG: "
4183
+ "no passages to append (empty hits)."
4184
+ )
4185
+ except Exception as exc:
4186
+ if self._verbose:
4187
+ print(
4188
+ f"[MetacouplingAssistant] Pre-retrieval RAG "
4189
+ f"evidence formatting failed: {exc}"
4190
+ )
4191
+ else: # post_hoc — existing behavior preserved exactly
4192
+ try:
4193
+ from metacouplingllm.knowledge.rag import (
4194
+ _build_query_from_analysis,
4195
+ annotate_citations,
4196
+ format_evidence,
4197
+ )
4198
+
4199
+ query = _build_query_from_analysis(parsed)
4200
+ if self._verbose:
4201
+ print(
4202
+ f"[MetacouplingAssistant] RAG query "
4203
+ f"({len(query)} chars): {query[:120]}..."
4204
+ )
4205
+
4206
+ if not query.strip():
4207
+ if self._verbose:
4208
+ print(
4209
+ "[MetacouplingAssistant] RAG query is empty — "
4210
+ "skipping evidence retrieval."
4211
+ )
4212
+ else:
4213
+ results = self._rag_engine.retrieve(
4214
+ query,
4215
+ top_k=self._rag_top_k,
4216
+ min_score=self._rag_min_score,
4217
+ max_chunks_per_paper=self._rag_max_chunks_per_paper,
4218
+ )
4219
+ if self._verbose:
4220
+ print(
4221
+ f"[MetacouplingAssistant] RAG returned "
4222
+ f"{len(results)} evidence passages."
4223
+ )
4224
+ if results:
4225
+ # Add inline [N] citations to analysis statements
4226
+ formatted = annotate_citations(formatted, results)
4227
+ # Append the evidence reference block, passing
4228
+ # the active backend so confidence thresholds
4229
+ # match the score range.
4230
+ rag_backend = (
4231
+ self._rag_engine.backend
4232
+ if self._rag_engine is not None
4233
+ else "tfidf"
4234
+ )
4235
+ formatted += "\n\n" + format_evidence(
4236
+ results,
4237
+ anchor_text=query,
4238
+ backend=rag_backend or "tfidf",
4239
+ )
4240
+ except Exception as exc:
4241
+ if self._verbose:
4242
+ print(f"[MetacouplingAssistant] RAG evidence failed: {exc}")
4243
+ # Non-fatal: continue without RAG evidence
4244
+
4245
+ # Optional: annotate and append web search sources
4246
+ if self._last_web_results:
4247
+ try:
4248
+ from metacouplingllm.knowledge.websearch import (
4249
+ annotate_web_citations,
4250
+ )
4251
+
4252
+ formatted = annotate_web_citations(
4253
+ formatted, self._last_web_results,
4254
+ )
4255
+ except Exception:
4256
+ pass # Non-fatal: skip web annotation
4257
+ formatted += self._format_web_sources(self._last_web_results)
4258
+
4259
+ # Optional: second LLM call to extract structured map data
4260
+ if self._auto_map and parsed.map_data is None:
4261
+ try:
4262
+ parsed.map_data = self._extract_map_data_from_analysis(
4263
+ parsed,
4264
+ )
4265
+ except Exception as exc:
4266
+ print(
4267
+ "[MetacouplingAssistant] Map data extraction "
4268
+ f"failed: {exc}"
4269
+ )
4270
+ parsed.map_data = None
4271
+
4272
+ # Enrich map data with specific countries / flows the
4273
+ # structured-extraction supplement found in the retrieved
4274
+ # passages. ``_extract_map_data_from_analysis`` reads the
4275
+ # finished analysis text, which often uses generic wording
4276
+ # ("foreign origin countries"); the supplement, by contrast,
4277
+ # reads the RAG passages directly with a schema that forces
4278
+ # per-country entries. Merging the two gives the map the
4279
+ # specific bilateral partners mentioned in the corpus.
4280
+ if (
4281
+ self._auto_map
4282
+ and parsed.map_data is not None
4283
+ and structured_supplement is not None
4284
+ ):
4285
+ try:
4286
+ self._merge_supplement_into_map_data(
4287
+ parsed, structured_supplement,
4288
+ )
4289
+ except Exception as exc:
4290
+ logger.warning(
4291
+ "Merging structured supplement into map_data "
4292
+ "failed: %s. Map will use only the direct "
4293
+ "extraction output.", exc,
4294
+ )
4295
+
4296
+ # Optional: auto-generate map
4297
+ fig: Figure | None = None
4298
+ map_notice: str | None = None
4299
+ if self._auto_map:
4300
+ fig = self._generate_map(parsed)
4301
+ if fig is not None:
4302
+ # Close the figure from pyplot's state manager so it does
4303
+ # NOT auto-display in Jupyter / Colab notebooks. The
4304
+ # Figure object itself remains fully usable — the user
4305
+ # can still call result.map.savefig(...) or
4306
+ # display(result.map).
4307
+ try:
4308
+ import matplotlib.pyplot as _plt
4309
+
4310
+ _plt.close(fig)
4311
+ except Exception:
4312
+ pass
4313
+
4314
+ # Determine map type for the notice
4315
+ md = parsed.map_data
4316
+ has_adm1 = (
4317
+ (md and md.get("adm1_region"))
4318
+ or self._resolve_adm1_from_analysis(parsed)
4319
+ )
4320
+ map_type = "adm1" if has_adm1 else "country"
4321
+ map_notice = self._format_map_notice(map_type)
4322
+ formatted += map_notice
4323
+ elif self._last_map_notice:
4324
+ map_notice = self._last_map_notice
4325
+ formatted += map_notice
4326
+
4327
+ return AnalysisResult(
4328
+ parsed=parsed,
4329
+ formatted=formatted,
4330
+ raw=response.content,
4331
+ turn_number=self._turn,
4332
+ usage=response.usage or None,
4333
+ map=fig,
4334
+ web_map_signals=self._last_web_map_signals,
4335
+ structured_supplement=structured_supplement,
4336
+ map_notice=map_notice,
4337
+ )
4338
+
4339
+ @staticmethod
4340
+ def _validate_adm1_pericoupling(parsed: ParsedAnalysis) -> bool:
4341
+ """Try to validate at the ADM1 (subnational) level.
4342
+
4343
+ If the analysis contains a resolvable subnational region,
4344
+ populates ``parsed.pericoupling_info`` with ADM1-level neighbor
4345
+ information and returns ``True``. Otherwise returns ``False``
4346
+ so the caller can fall through to country-level validation.
4347
+ """
4348
+ from metacouplingllm.knowledge.adm1_pericoupling import (
4349
+ get_adm1_info,
4350
+ get_adm1_neighbors,
4351
+ get_cross_border_neighbors,
4352
+ )
4353
+
4354
+ adm1_code = MetacouplingAssistant._resolve_adm1_from_analysis(parsed)
4355
+ if adm1_code is None:
4356
+ return False
4357
+
4358
+ focal_info = get_adm1_info(adm1_code)
4359
+ if not focal_info:
4360
+ return False
4361
+
4362
+ all_neighbors = get_adm1_neighbors(adm1_code)
4363
+ cross_border = get_cross_border_neighbors(adm1_code)
4364
+ domestic = all_neighbors - cross_border
4365
+
4366
+ def _names(codes: set[str]) -> str:
4367
+ items: list[str] = []
4368
+ for c in sorted(codes):
4369
+ info = get_adm1_info(c)
4370
+ if info:
4371
+ items.append(f"{info['name']} ({c})")
4372
+ return ", ".join(items)
4373
+
4374
+ peri_info: dict[str, str] = {
4375
+ "level": "adm1",
4376
+ "focal_region": (
4377
+ f"{focal_info['name']} ({adm1_code})"
4378
+ ),
4379
+ "focal_country": (
4380
+ f"{focal_info['country_name']} ({focal_info['iso_a3']})"
4381
+ ),
4382
+ }
4383
+
4384
+ if domestic:
4385
+ peri_info["domestic_neighbors"] = _names(domestic)
4386
+ if cross_border:
4387
+ peri_info["cross_border_neighbors"] = _names(cross_border)
4388
+
4389
+ # Consistency note
4390
+ peri_info["note"] = (
4391
+ "LLM classification is consistent with the "
4392
+ "ADM1 pericoupling database."
4393
+ )
4394
+
4395
+ parsed.pericoupling_info = peri_info
4396
+ return True
4397
+
4398
+ @staticmethod
4399
+ def _validate_pericoupling(parsed: ParsedAnalysis) -> None:
4400
+ """Extract country names from systems and validate against the DB.
4401
+
4402
+ Only validates pairs involving the **sending system** (focal
4403
+ country). This avoids spurious validation of pairs between
4404
+ receiving countries that the researcher did not intend to
4405
+ compare (e.g. USA ↔ Canada when the study is Mexico → USA/CAN).
4406
+
4407
+ Populates ``parsed.pericoupling_info`` with the validation result.
4408
+ """
4409
+ # Try ADM1 (subnational) validation first.
4410
+ if MetacouplingAssistant._validate_adm1_pericoupling(parsed):
4411
+ return
4412
+
4413
+ # --- Identify the focal (sending) country first ---
4414
+ focal_code: str | None = None
4415
+ focal_name: str | None = None
4416
+ sending_entry = (
4417
+ parsed.get_first_system_entry("sending")
4418
+ or parsed.get_first_system_entry("focal")
4419
+ )
4420
+ if sending_entry is not None:
4421
+ for key in ("name", "geographic_scope"):
4422
+ value = sending_entry.get(key, "")
4423
+ if value:
4424
+ code = resolve_country_code(value)
4425
+ if code:
4426
+ focal_code = code
4427
+ focal_name = value
4428
+ break
4429
+
4430
+ # Fall back to first detected country from all systems
4431
+ if focal_code is None:
4432
+ countries = MetacouplingAssistant._extract_country_names(parsed)
4433
+ if countries:
4434
+ code = resolve_country_code(countries[0])
4435
+ if code:
4436
+ focal_code = code
4437
+ focal_name = countries[0]
4438
+
4439
+ if focal_code is None:
4440
+ return # Cannot identify any country
4441
+
4442
+ # --- Collect other (non-focal) countries ---
4443
+ all_countries = MetacouplingAssistant._extract_country_names(parsed)
4444
+ other_codes: list[tuple[str, str]] = [] # (code, display_name)
4445
+ seen: set[str] = {focal_code}
4446
+ for name in all_countries:
4447
+ code = resolve_country_code(name)
4448
+ if code and code not in seen:
4449
+ seen.add(code)
4450
+ other_codes.append((code, name))
4451
+
4452
+ if not other_codes:
4453
+ return # Only one country detected
4454
+
4455
+ # --- Build validation info (focal ↔ each other only) ---
4456
+ info: dict[str, str] = {}
4457
+ info[f"focal_country"] = f"{get_country_name(focal_code)} ({focal_code})"
4458
+
4459
+ pair_lines: list[str] = []
4460
+ for code_b, _name_b in other_codes:
4461
+ result = lookup_pericoupling(focal_code, code_b)
4462
+ cn_a = get_country_name(focal_code)
4463
+ cn_b = get_country_name(code_b)
4464
+ if result.pair_type == PairCouplingType.PERICOUPLED:
4465
+ pair_lines.append(
4466
+ f"{cn_a} ({focal_code}) ↔ {cn_b} ({code_b}): PERICOUPLED"
4467
+ )
4468
+ elif result.pair_type == PairCouplingType.TELECOUPLED:
4469
+ pair_lines.append(
4470
+ f"{cn_a} ({focal_code}) ↔ {cn_b} ({code_b}): TELECOUPLED"
4471
+ )
4472
+
4473
+ if pair_lines:
4474
+ info["pair_results"] = "; ".join(pair_lines)
4475
+
4476
+ # Check agreement with LLM classification.
4477
+ if parsed.coupling_classification:
4478
+ llm_class = parsed.coupling_classification.lower()
4479
+ has_peri = any("PERICOUPLED" in p for p in pair_lines)
4480
+ has_tele = any("TELECOUPLED" in p for p in pair_lines)
4481
+
4482
+ if has_peri and "pericoupl" not in llm_class:
4483
+ info["note"] = (
4484
+ "The pericoupling database indicates at least one "
4485
+ "pericoupled country pair, but the LLM classified "
4486
+ "this study differently. Consider revising."
4487
+ )
4488
+ elif has_tele and not has_peri and "telecoupl" not in llm_class:
4489
+ info["note"] = (
4490
+ "The pericoupling database indicates all detected "
4491
+ "pairs are telecoupled, but the LLM classified "
4492
+ "this study differently. Consider revising."
4493
+ )
4494
+ else:
4495
+ info["note"] = (
4496
+ "LLM classification is consistent with the "
4497
+ "pericoupling database."
4498
+ )
4499
+
4500
+ parsed.pericoupling_info = info
4501
+
4502
+ @staticmethod
4503
+ def _extract_country_names(parsed: ParsedAnalysis) -> list[str]:
4504
+ """Extract country-like names from parsed systems data.
4505
+
4506
+ Looks at sending, receiving, and spillover system names/descriptions
4507
+ and tries to identify country references.
4508
+ """
4509
+ texts: list[str] = []
4510
+ for role in ("focal", "adjacent", "sending", "receiving", "spillover"):
4511
+ for entry in parsed.get_system_entries(role):
4512
+ if entry.get("name"):
4513
+ texts.append(entry["name"])
4514
+ if entry.get("geographic_scope"):
4515
+ texts.append(entry["geographic_scope"])
4516
+
4517
+ # Also scan the coupling_classification for country mentions
4518
+ if parsed.coupling_classification:
4519
+ texts.append(parsed.coupling_classification)
4520
+
4521
+ # Resolve unique countries from all collected text
4522
+ seen: set[str] = set()
4523
+ countries: list[str] = []
4524
+ for text in texts:
4525
+ # Try resolving the full text first (e.g. "Mexico")
4526
+ code = resolve_country_code(text)
4527
+ if code and code not in seen:
4528
+ seen.add(code)
4529
+ countries.append(text)
4530
+ continue
4531
+ # Otherwise scan for country names within the text
4532
+ # Split on common delimiters
4533
+ for chunk in re.split(r"[,;/()]+", text):
4534
+ chunk = chunk.strip()
4535
+ if not chunk:
4536
+ continue
4537
+ code = resolve_country_code(chunk)
4538
+ if code and code not in seen:
4539
+ seen.add(code)
4540
+ countries.append(chunk)
4541
+
4542
+ return countries