MetacouplingLLM 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. metacouplingllm/__init__.py +208 -0
  2. metacouplingllm/core.py +4542 -0
  3. metacouplingllm/data/Papers.zip +0 -0
  4. metacouplingllm/data/PeriTelecoupling_clean.csv +49953 -0
  5. metacouplingllm/data/PeriTelecoupling_subset.csv +11 -0
  6. metacouplingllm/data/chunk_embeddings.manifest.json +11 -0
  7. metacouplingllm/data/chunk_embeddings.npy +0 -0
  8. metacouplingllm/data/pericoupled_adm1_edge_list.csv +8291 -0
  9. metacouplingllm/data/telecoupling_literature.bib +3498 -0
  10. metacouplingllm/knowledge/__init__.py +1 -0
  11. metacouplingllm/knowledge/adm1_pericoupling.py +591 -0
  12. metacouplingllm/knowledge/citations.py +109 -0
  13. metacouplingllm/knowledge/countries.py +557 -0
  14. metacouplingllm/knowledge/examples.py +713 -0
  15. metacouplingllm/knowledge/framework.py +717 -0
  16. metacouplingllm/knowledge/literature.py +518 -0
  17. metacouplingllm/knowledge/pericoupling.py +241 -0
  18. metacouplingllm/knowledge/rag.py +1791 -0
  19. metacouplingllm/knowledge/references.py +112 -0
  20. metacouplingllm/knowledge/websearch.py +1842 -0
  21. metacouplingllm/llm/__init__.py +1 -0
  22. metacouplingllm/llm/client.py +636 -0
  23. metacouplingllm/llm/parser.py +868 -0
  24. metacouplingllm/output/__init__.py +1 -0
  25. metacouplingllm/output/formatter.py +276 -0
  26. metacouplingllm/prompts/__init__.py +1 -0
  27. metacouplingllm/prompts/builder.py +477 -0
  28. metacouplingllm/prompts/templates.py +556 -0
  29. metacouplingllm/visualization/__init__.py +114 -0
  30. metacouplingllm/visualization/adm1_map.py +884 -0
  31. metacouplingllm/visualization/worldmap.py +1372 -0
  32. metacouplingllm-0.1.0.dist-info/METADATA +440 -0
  33. metacouplingllm-0.1.0.dist-info/RECORD +35 -0
  34. metacouplingllm-0.1.0.dist-info/WHEEL +4 -0
  35. metacouplingllm-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,477 @@
1
+ """
2
+ Prompt assembly — combines the six template layers with domain knowledge.
3
+
4
+ The :class:`PromptBuilder` injects framework definitions, curated examples,
5
+ and references into the prompt templates to produce complete system prompts
6
+ and user messages.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import re
12
+ from typing import TYPE_CHECKING
13
+
14
+ from metacouplingllm.knowledge.adm1_pericoupling import (
15
+ get_adm1_info,
16
+ get_adm1_neighbors,
17
+ get_cross_border_neighbors,
18
+ resolve_adm1_code,
19
+ )
20
+ from metacouplingllm.knowledge.countries import (
21
+ ISO_ALPHA3_NAMES,
22
+ get_country_name,
23
+ resolve_country_code,
24
+ )
25
+ from metacouplingllm.knowledge.examples import (
26
+ format_example,
27
+ get_relevant_examples,
28
+ )
29
+ from metacouplingllm.knowledge.framework import get_framework_knowledge
30
+ from metacouplingllm.knowledge.pericoupling import PairCouplingType, lookup_pericoupling
31
+ from metacouplingllm.knowledge.references import format_references
32
+ from metacouplingllm.prompts.templates import (
33
+ CITATION_RULES_LAYER,
34
+ EXAMPLES_LAYER,
35
+ INITIAL_USER_TEMPLATE,
36
+ INTERACTION_LAYER,
37
+ KNOWLEDGE_LAYER,
38
+ METHODOLOGY_LAYER,
39
+ OUTPUT_FORMAT_LAYER,
40
+ REFINEMENT_USER_TEMPLATE,
41
+ REFINEMENT_WITH_FOCUS_TEMPLATE,
42
+ ROLE_LAYER,
43
+ )
44
+
45
+ if TYPE_CHECKING: # pragma: no cover - types only
46
+ from metacouplingllm.knowledge.rag import RetrievalResult
47
+
48
+
49
+ # Maximum characters of passage text to include in the prompt for any
50
+ # single retrieved chunk. Longer chunks are truncated with a "..." suffix.
51
+ #
52
+ # The chunker in ``knowledge.rag._chunk_markdown`` caps each chunk at
53
+ # ~250 words (~1600 chars median, p99 under 2000 chars, absolute max
54
+ # observed ~4700 chars). Setting this cap at 5000 is effectively a
55
+ # no-op for legitimate chunks — it only kicks in if the chunker
56
+ # produces a pathological chunk. The prior 800-char cap was cutting
57
+ # ~50% of every chunk's content, causing the main analysis LLM to
58
+ # miss bilateral country data that lives past the section-opening
59
+ # paragraph (e.g., "Korea (2.65 MtCO2)" at char 689 in a long Results
60
+ # chunk). A research tool should not silently lose half the evidence.
61
+ _PASSAGE_MAX_CHARS = 5000
62
+
63
+
64
+ class PromptBuilder:
65
+ """Assembles multi-layer system prompts and user messages.
66
+
67
+ Parameters
68
+ ----------
69
+ max_examples:
70
+ Maximum number of curated examples to include in the prompt.
71
+ """
72
+
73
+ def __init__(self, max_examples: int = 2) -> None:
74
+ self._max_examples = max_examples
75
+
76
+ def build_system_prompt(
77
+ self,
78
+ research_context: str | None = None,
79
+ web_context: str | None = None,
80
+ include_citation_rules: bool = False,
81
+ ) -> str:
82
+ """Build the full system prompt by combining all template layers.
83
+
84
+ Parameters
85
+ ----------
86
+ research_context:
87
+ Optional research description used to select the most relevant
88
+ examples. If ``None``, default examples are used.
89
+ web_context:
90
+ Optional pre-formatted web search context to inject into the
91
+ prompt. Typically produced by
92
+ :func:`~metacouplingllm.knowledge.websearch.format_web_context`.
93
+ include_citation_rules:
94
+ When ``True``, inject :data:`CITATION_RULES_LAYER` between the
95
+ output-format layer and the interaction layer. Used by the
96
+ pre-retrieval RAG pipeline so the LLM knows how to cite the
97
+ ``<retrieved_literature>`` block that will appear in user
98
+ messages. Defaults to ``False`` for backward compatibility
99
+ with the post-hoc RAG pipeline.
100
+
101
+ Returns
102
+ -------
103
+ A complete system prompt string ready to be sent as the system
104
+ message to an LLM.
105
+ """
106
+ # Layer 1 — Role
107
+ parts: list[str] = [ROLE_LAYER]
108
+
109
+ # Layer 2 — Knowledge (framework definitions)
110
+ framework_text = get_framework_knowledge()
111
+ parts.append(KNOWLEDGE_LAYER.format(framework_definitions=framework_text))
112
+
113
+ # Layer 3 — Methodology
114
+ parts.append(METHODOLOGY_LAYER)
115
+
116
+ # Layer 4 — Examples
117
+ examples = get_relevant_examples(
118
+ research_context or "",
119
+ max_examples=self._max_examples,
120
+ )
121
+ if examples:
122
+ formatted = "\n\n".join(format_example(ex) for ex in examples)
123
+ parts.append(EXAMPLES_LAYER.format(formatted_examples=formatted))
124
+
125
+ # Layer 5 — Output format
126
+ parts.append(OUTPUT_FORMAT_LAYER)
127
+
128
+ # Layer 5b — Citation rules (only in pre-retrieval RAG mode).
129
+ # Placed after OUTPUT_FORMAT_LAYER so the LLM reads citation
130
+ # mechanics adjacent to the format spec, and immediately before
131
+ # INTERACTION_LAYER so recency bias keeps the strict rules
132
+ # fresh in the model's attention.
133
+ if include_citation_rules:
134
+ parts.append(CITATION_RULES_LAYER)
135
+
136
+ # Layer 6 — Interaction guidelines
137
+ parts.append(INTERACTION_LAYER)
138
+
139
+ # Pericoupling database hint (pre-LLM injection)
140
+ # Try subnational (ADM1) hint first; fall back to country-level.
141
+ if research_context:
142
+ hint = self._build_adm1_pericoupling_hint(research_context)
143
+ if hint is None:
144
+ hint = self._build_pericoupling_hint(research_context)
145
+ if hint:
146
+ parts.append(hint)
147
+
148
+ # Web search context (pre-LLM injection)
149
+ if web_context:
150
+ parts.append(web_context)
151
+
152
+ # Append references
153
+ parts.append(format_references())
154
+
155
+ return "\n\n".join(parts)
156
+
157
+ @staticmethod
158
+ def build_initial_message(
159
+ research_description: str,
160
+ literature_passages: "list[RetrievalResult] | None" = None,
161
+ ) -> str:
162
+ """Format the user's research description for the first turn.
163
+
164
+ Parameters
165
+ ----------
166
+ research_description:
167
+ The researcher's description of their study.
168
+ literature_passages:
169
+ Optional list of pre-retrieved RAG passages to inject as a
170
+ ``<retrieved_literature>`` XML block. The block is placed
171
+ BEFORE the research description (scope-setting) so the
172
+ user's actual question remains the last thing the LLM
173
+ reads. ``None`` (default) and ``[]`` are both handled —
174
+ see :meth:`_format_literature_block`. When ``None`` no block
175
+ is injected at all (post-hoc / no-RAG behavior).
176
+
177
+ Returns
178
+ -------
179
+ A formatted user message string.
180
+ """
181
+ body = INITIAL_USER_TEMPLATE.format(
182
+ research_description=research_description.strip(),
183
+ )
184
+ if literature_passages is None:
185
+ return body
186
+ # Pre-retrieval mode: scope-setting block goes BEFORE the
187
+ # research description so the user's ask is the last thing
188
+ # the LLM reads (recency bias).
189
+ block = PromptBuilder._format_literature_block(literature_passages)
190
+ return f"{block}\n\n{body}"
191
+
192
+ @staticmethod
193
+ def build_refinement_message(
194
+ additional_info: str,
195
+ focus_component: str | None = None,
196
+ literature_passages: "list[RetrievalResult] | None" = None,
197
+ ) -> str:
198
+ """Format a follow-up message for refining the analysis.
199
+
200
+ Parameters
201
+ ----------
202
+ additional_info:
203
+ Additional context or questions from the researcher.
204
+ focus_component:
205
+ Optional framework component to focus on (e.g., ``"systems"``,
206
+ ``"flows"``). When provided, the prompt asks the LLM to
207
+ elaborate on that specific component.
208
+ literature_passages:
209
+ Optional list of pre-retrieved RAG passages to inject as a
210
+ ``<retrieved_literature>`` XML block. Unlike the initial
211
+ message, the block is placed AFTER the refinement text so
212
+ it reads as supporting evidence for the user's follow-up
213
+ question rather than scope-setting context.
214
+
215
+ Returns
216
+ -------
217
+ A formatted refinement message string.
218
+ """
219
+ info = additional_info.strip()
220
+
221
+ if focus_component:
222
+ body = REFINEMENT_WITH_FOCUS_TEMPLATE.format(
223
+ focus_component=focus_component,
224
+ additional_info=info,
225
+ )
226
+ else:
227
+ body = REFINEMENT_USER_TEMPLATE.format(additional_info=info)
228
+
229
+ if literature_passages is None:
230
+ return body
231
+ block = PromptBuilder._format_literature_block(literature_passages)
232
+ return f"{body}\n\n{block}"
233
+
234
+ # ------------------------------------------------------------------
235
+ # Literature block formatting
236
+ # ------------------------------------------------------------------
237
+
238
+ @staticmethod
239
+ def _format_literature_block(
240
+ results: "list[RetrievalResult] | None",
241
+ ) -> str:
242
+ """Render retrieved passages as an XML ``<retrieved_literature>`` block.
243
+
244
+ Parameters
245
+ ----------
246
+ results:
247
+ List of :class:`~metacouplingllm.knowledge.rag.RetrievalResult`
248
+ objects in the order they should be cited (1-indexed). May
249
+ be ``None`` or empty — in either case the block is emitted
250
+ as the self-closing tag ``<retrieved_literature/>`` so the
251
+ LLM knows that retrieval ran but found no relevant passages
252
+ (preventing it from inventing citations).
253
+
254
+ Returns
255
+ -------
256
+ A string containing one ``<retrieved_literature>`` block. Each
257
+ passage is rendered as
258
+ ``<passage id="N" paper_key="..." authors="..." year="..." section="..." score="0.87">text</passage>``.
259
+ Passage text is truncated to roughly :data:`_PASSAGE_MAX_CHARS`
260
+ characters with a ``...`` suffix.
261
+ """
262
+ if not results:
263
+ return "<retrieved_literature/>"
264
+
265
+ lines: list[str] = ["<retrieved_literature>"]
266
+ for idx, r in enumerate(results, 1):
267
+ chunk = r.chunk
268
+ authors = _xml_attr_escape(chunk.authors)
269
+ paper_key = _xml_attr_escape(chunk.paper_key)
270
+ section = _xml_attr_escape(chunk.section)
271
+ text = chunk.text.strip()
272
+ if len(text) > _PASSAGE_MAX_CHARS:
273
+ text = text[: _PASSAGE_MAX_CHARS - 3].rstrip() + "..."
274
+ text = _xml_text_escape(text)
275
+ lines.append(
276
+ f'<passage id="{idx}" paper_key="{paper_key}" '
277
+ f'authors="{authors}" year="{chunk.year}" '
278
+ f'section="{section}" score="{r.score:.2f}">'
279
+ )
280
+ lines.append(text)
281
+ lines.append("</passage>")
282
+ lines.append("</retrieved_literature>")
283
+ return "\n".join(lines)
284
+
285
+ # ------------------------------------------------------------------
286
+ # Pericoupling hint
287
+ # ------------------------------------------------------------------
288
+
289
+ @staticmethod
290
+ def _build_pericoupling_hint(research_context: str) -> str | None:
291
+ """Scan research text for country names and build a pericoupling hint.
292
+
293
+ Only generates pairs that involve the **first detected country**
294
+ (assumed to be the focal/sending country). This avoids spurious
295
+ pairs between receiving countries that the researcher did not
296
+ intend to compare (e.g. USA ↔ Canada when the study is about
297
+ Mexico exporting to both).
298
+
299
+ Returns ``None`` if fewer than two countries are found.
300
+ """
301
+ # Detect country codes mentioned in the text.
302
+ found_codes: list[str] = []
303
+ seen: set[str] = set()
304
+
305
+ # Try to resolve each word-sequence. We scan overlapping n-grams
306
+ # (up to 4 words) to catch multi-word names like "United States".
307
+ words = re.findall(r"[A-Za-z'']+(?:\s+[A-Za-z'']+){0,3}", research_context)
308
+ for phrase in words:
309
+ code = resolve_country_code(phrase)
310
+ if code and code not in seen:
311
+ found_codes.append(code)
312
+ seen.add(code)
313
+
314
+ if len(found_codes) < 2:
315
+ return None
316
+
317
+ # Use the first detected country as the focal country and only
318
+ # look up pairs between the focal country and the others.
319
+ focal_code = found_codes[0]
320
+ other_codes = found_codes[1:]
321
+
322
+ lines: list[str] = []
323
+ for code_b in other_codes:
324
+ result = lookup_pericoupling(focal_code, code_b)
325
+ name_a = get_country_name(focal_code)
326
+ name_b = get_country_name(code_b)
327
+ if result.pair_type == PairCouplingType.PERICOUPLED:
328
+ lines.append(
329
+ f"- {name_a} ({focal_code}) and {name_b} ({code_b}) are "
330
+ f"**pericoupled** (geographically adjacent) according "
331
+ f"to the pericoupling database."
332
+ )
333
+ elif result.pair_type == PairCouplingType.TELECOUPLED:
334
+ lines.append(
335
+ f"- {name_a} ({focal_code}) and {name_b} ({code_b}) are "
336
+ f"**telecoupled** (geographically distant) according "
337
+ f"to the pericoupling database."
338
+ )
339
+
340
+ if not lines:
341
+ return None
342
+
343
+ focal_name = get_country_name(focal_code)
344
+ header = (
345
+ "## PERICOUPLING DATABASE INFORMATION\n\n"
346
+ f"Focal country detected: {focal_name} ({focal_code}).\n"
347
+ "The following country-pair classifications come from a curated "
348
+ "pericoupling database (ISO 3166-1 alpha-3). Use these ground-truth "
349
+ "adjacency relationships when classifying coupling types in your "
350
+ "analysis:\n"
351
+ )
352
+ return header + "\n".join(lines)
353
+
354
+ @staticmethod
355
+ def _build_adm1_pericoupling_hint(research_context: str) -> str | None:
356
+ """Scan research text for ADM1 region names and build a subnational hint.
357
+
358
+ Uses the same n-gram scanning approach as
359
+ :meth:`_build_pericoupling_hint`, but resolves phrases against the
360
+ ADM1 pericoupling database. On the first successful match (the
361
+ *focal region*), retrieves its domestic and cross-border neighbors
362
+ and formats them into a prompt section.
363
+
364
+ Returns ``None`` if no ADM1 region is found in the text.
365
+ """
366
+ # Scan for ADM1 region names via overlapping n-grams (up to 4 words).
367
+ focal_code: str | None = None
368
+ words = re.findall(
369
+ r"[A-Za-z'']+(?:\s+[A-Za-z'']+){0,3}", research_context,
370
+ )
371
+ for phrase in words:
372
+ code = resolve_adm1_code(phrase)
373
+ if code:
374
+ focal_code = code
375
+ break
376
+
377
+ if focal_code is None:
378
+ return None
379
+
380
+ focal_info = get_adm1_info(focal_code)
381
+ if not focal_info:
382
+ return None
383
+
384
+ all_neighbors = get_adm1_neighbors(focal_code)
385
+ cross_border = get_cross_border_neighbors(focal_code)
386
+ domestic = all_neighbors - cross_border
387
+
388
+ # Build readable neighbor lists.
389
+ def _format_neighbors(codes: set[str]) -> list[str]:
390
+ items: list[str] = []
391
+ for c in sorted(codes):
392
+ info = get_adm1_info(c)
393
+ if info:
394
+ items.append(
395
+ f"{info['name']} ({c}, {info['country_name']})"
396
+ )
397
+ return items
398
+
399
+ lines: list[str] = []
400
+
401
+ if domestic:
402
+ domestic_names = _format_neighbors(domestic)
403
+ lines.append(
404
+ "**Same-country (pericoupled) neighbors:**\n"
405
+ + "\n".join(f"- {n}" for n in domestic_names)
406
+ )
407
+
408
+ if cross_border:
409
+ cross_names = _format_neighbors(cross_border)
410
+ lines.append(
411
+ "**Cross-border (pericoupled) neighbors:**\n"
412
+ + "\n".join(f"- {n}" for n in cross_names)
413
+ )
414
+
415
+ if not lines:
416
+ return None
417
+
418
+ focal_name = focal_info["name"]
419
+ country_name = focal_info["country_name"]
420
+ header = (
421
+ "## ADM1 ADJACENCY DATABASE (REFERENCE ONLY)\n\n"
422
+ f"Focal subnational region detected: **{focal_name}** "
423
+ f"({focal_code}, {country_name}).\n\n"
424
+ "The following regions share a geographic border with the "
425
+ "focal region according to a curated ADM1 adjacency database. "
426
+ "**Adjacency alone is NOT evidence of pericoupling.** Only "
427
+ "classify a region as pericoupled if you have independent "
428
+ "evidence (from the research context, web snippets, or your "
429
+ "training knowledge) of actual interactions, flows, impacts, "
430
+ "or exchanges between that region and the focal region — "
431
+ "e.g., labor movement, transport corridors, commodity flows, "
432
+ "land-use displacement, or shared watersheds. If you have no "
433
+ "specific evidence for a listed region, do NOT name it in "
434
+ "your analysis and do NOT classify it as pericoupled.\n\n"
435
+ "These adjacency relationships are provided for reference only.\n\n"
436
+ )
437
+ return header + "\n\n".join(lines)
438
+
439
+
440
+ # ----------------------------------------------------------------------
441
+ # XML escaping helpers for the literature block
442
+ # ----------------------------------------------------------------------
443
+ #
444
+ # Kept module-private and minimal — we are not building a full XML
445
+ # document, just emitting tags the LLM can read. Five characters
446
+ # (& < > " ') need to be escaped to keep the attributes/content from
447
+ # breaking the surrounding tag structure.
448
+
449
+ _XML_ATTR_ESCAPES = {
450
+ "&": "&amp;",
451
+ "<": "&lt;",
452
+ ">": "&gt;",
453
+ '"': "&quot;",
454
+ "'": "&apos;",
455
+ }
456
+
457
+ _XML_TEXT_ESCAPES = {
458
+ "&": "&amp;",
459
+ "<": "&lt;",
460
+ ">": "&gt;",
461
+ }
462
+
463
+
464
+ def _xml_attr_escape(value: str) -> str:
465
+ """Escape ``&``, ``<``, ``>``, ``"`` and ``'`` for use in an XML attribute."""
466
+ out = value
467
+ for ch, esc in _XML_ATTR_ESCAPES.items():
468
+ out = out.replace(ch, esc)
469
+ return out
470
+
471
+
472
+ def _xml_text_escape(value: str) -> str:
473
+ """Escape ``&``, ``<``, ``>`` for use as XML text content."""
474
+ out = value
475
+ for ch, esc in _XML_TEXT_ESCAPES.items():
476
+ out = out.replace(ch, esc)
477
+ return out