MetacouplingLLM 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. metacouplingllm/__init__.py +208 -0
  2. metacouplingllm/core.py +4542 -0
  3. metacouplingllm/data/Papers.zip +0 -0
  4. metacouplingllm/data/PeriTelecoupling_clean.csv +49953 -0
  5. metacouplingllm/data/PeriTelecoupling_subset.csv +11 -0
  6. metacouplingllm/data/chunk_embeddings.manifest.json +11 -0
  7. metacouplingllm/data/chunk_embeddings.npy +0 -0
  8. metacouplingllm/data/pericoupled_adm1_edge_list.csv +8291 -0
  9. metacouplingllm/data/telecoupling_literature.bib +3498 -0
  10. metacouplingllm/knowledge/__init__.py +1 -0
  11. metacouplingllm/knowledge/adm1_pericoupling.py +591 -0
  12. metacouplingllm/knowledge/citations.py +109 -0
  13. metacouplingllm/knowledge/countries.py +557 -0
  14. metacouplingllm/knowledge/examples.py +713 -0
  15. metacouplingllm/knowledge/framework.py +717 -0
  16. metacouplingllm/knowledge/literature.py +518 -0
  17. metacouplingllm/knowledge/pericoupling.py +241 -0
  18. metacouplingllm/knowledge/rag.py +1791 -0
  19. metacouplingllm/knowledge/references.py +112 -0
  20. metacouplingllm/knowledge/websearch.py +1842 -0
  21. metacouplingllm/llm/__init__.py +1 -0
  22. metacouplingllm/llm/client.py +636 -0
  23. metacouplingllm/llm/parser.py +868 -0
  24. metacouplingllm/output/__init__.py +1 -0
  25. metacouplingllm/output/formatter.py +276 -0
  26. metacouplingllm/prompts/__init__.py +1 -0
  27. metacouplingllm/prompts/builder.py +477 -0
  28. metacouplingllm/prompts/templates.py +556 -0
  29. metacouplingllm/visualization/__init__.py +114 -0
  30. metacouplingllm/visualization/adm1_map.py +884 -0
  31. metacouplingllm/visualization/worldmap.py +1372 -0
  32. metacouplingllm-0.1.0.dist-info/METADATA +440 -0
  33. metacouplingllm-0.1.0.dist-info/RECORD +35 -0
  34. metacouplingllm-0.1.0.dist-info/WHEEL +4 -0
  35. metacouplingllm-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,1842 @@
1
+ """
2
+ Web search integration for grounding metacoupling analyses.
3
+
4
+ Provides a lightweight wrapper around DuckDuckGo search (free, no API
5
+ key) to inject real-world context into the LLM prompt before analysis.
6
+ This is especially useful for niche topics where the LLM's pre-training
7
+ knowledge may be limited (e.g., "Michigan's pork exports").
8
+
9
+ Three backends are tried in order:
10
+
11
+ 1. ``ddgs`` (pip install ddgs) — fast, full-featured
12
+ 2. ``duckduckgo_search`` — older package, same author
13
+ 3. **stdlib fallback** — ``urllib`` + ``html.parser``, zero external
14
+ dependencies. Works on Google Colab and other restricted
15
+ environments where the Rust-compiled ``primp`` dependency cannot be
16
+ installed.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import html.parser
22
+ import json
23
+ import re
24
+ import socket
25
+ import urllib.error
26
+ import urllib.parse
27
+ import urllib.request
28
+ from dataclasses import dataclass
29
+ from typing import Any, Protocol
30
+
31
+ _QUERY_FILLER = frozenset(
32
+ (
33
+ "a an analyze analysed analyzed analyzes analyzing analysis and apply "
34
+ "applying assess assessed assessing assesses case consider considering "
35
+ "examines examining explain explains focus focused focusing for from "
36
+ "how impact impacts in investigate investigating looking my of on or "
37
+ "our paper project question questions research researcher researchers "
38
+ "review reviews study studying the to topic topics what will with"
39
+ ).split()
40
+ )
41
+
42
+ _STRUCTURAL_TERMS = frozenset(
43
+ (
44
+ "chain chains commodity commodities coupling couplings effect effects "
45
+ "environment environmental export exported exporter exporters exports "
46
+ "flow flows food framework frameworks import imported importer "
47
+ "importers imports industry industries market markets metacoupling "
48
+ "partner partners production productions sector sectors supply system "
49
+ "systems telecoupling trade traded trader traders trading transfer "
50
+ "transfers"
51
+ ).split()
52
+ )
53
+
54
+ _TRADE_HINTS = frozenset(
55
+ (
56
+ "destination destinations export exported exporter exporters exports "
57
+ "import imported importer importers imports market markets partner "
58
+ "partners trade traded trader traders trading"
59
+ ).split()
60
+ )
61
+
62
+
63
+ class WebSearchBackend(Protocol):
64
+ """Protocol for pluggable web-search backends."""
65
+
66
+ def search(
67
+ self,
68
+ query: str,
69
+ max_results: int = 5,
70
+ ) -> list[dict[str, str]]:
71
+ """Return search results as ``title``/``snippet``/``url`` dicts."""
72
+ ...
73
+
74
+
75
+ @dataclass
76
+ class OpenAIWebSearchBackend:
77
+ """Web-search backend powered by OpenAI Responses API ``web_search``.
78
+
79
+ Results are normalized to the same ``title``/``snippet``/``url`` shape
80
+ used by the existing DuckDuckGo backends so downstream consumers do not
81
+ need to change.
82
+ """
83
+
84
+ client: Any
85
+ model: str = "gpt-5"
86
+ reasoning: str = "default"
87
+ external_web_access: bool = True
88
+ allowed_domains: list[str] | None = None
89
+ user_location: dict[str, object] | None = None
90
+
91
+ def search(
92
+ self,
93
+ query: str,
94
+ max_results: int = 5,
95
+ ) -> list[dict[str, str]]:
96
+ if not query.strip() or max_results <= 0:
97
+ return []
98
+
99
+ tool: dict[str, object] = {
100
+ "type": "web_search",
101
+ "external_web_access": self.external_web_access,
102
+ }
103
+ if self.allowed_domains:
104
+ tool["filters"] = {"allowed_domains": self.allowed_domains}
105
+ if self.user_location:
106
+ tool["user_location"] = self.user_location
107
+
108
+ prompt = (
109
+ f"Search the web for: {query.strip()}\n"
110
+ f"Return up to {max_results} highly relevant results as JSON only.\n"
111
+ "Schema: "
112
+ '{"results":[{"title":"...","url":"...","snippet":"..."}]}\n'
113
+ "Rules:\n"
114
+ "- Use only information grounded in the web search results.\n"
115
+ "- Keep snippets concise and factual.\n"
116
+ "- Do not invent URLs.\n"
117
+ "- Return JSON only.\n"
118
+ )
119
+
120
+ kwargs: dict[str, object] = {
121
+ "model": self.model,
122
+ "tools": [tool],
123
+ "tool_choice": "auto",
124
+ "include": ["web_search_call.action.sources"],
125
+ "input": prompt,
126
+ }
127
+ if self.reasoning and self.reasoning != "default":
128
+ kwargs["reasoning"] = {"effort": self.reasoning}
129
+
130
+ response = self.client.responses.create(**kwargs)
131
+ output_text = getattr(response, "output_text", "") or ""
132
+ raw_obj = _extract_json_object(output_text)
133
+ parsed_results = []
134
+ if isinstance(raw_obj, dict):
135
+ parsed_results = _normalise_backend_results(
136
+ raw_obj.get("results", []),
137
+ max_results=max_results,
138
+ )
139
+
140
+ # Fallback: if the model didn't emit JSON results cleanly, recover
141
+ # from the included source list so the search still yields usable URLs.
142
+ source_results = _extract_openai_source_results(response, max_results)
143
+ if parsed_results:
144
+ merged = []
145
+ seen: set[tuple[str, str]] = set()
146
+ _merge_unique_results(
147
+ merged,
148
+ parsed_results,
149
+ seen=seen,
150
+ max_results=max_results,
151
+ )
152
+ _merge_unique_results(
153
+ merged,
154
+ source_results,
155
+ seen=seen,
156
+ max_results=max_results,
157
+ )
158
+ return merged[:max_results]
159
+ return source_results[:max_results]
160
+
161
+
162
+ # Model → web_search tool version compatibility. Used by
163
+ # AnthropicWebSearchBackend when ``tool_version`` is left as None.
164
+ # Per Anthropic docs, web_search_20260209 (with dynamic filtering) is
165
+ # supported on Opus 4.7, Opus 4.6, and Sonnet 4.6. Older Claude 4 models
166
+ # must use web_search_20250305 (no dynamic filtering).
167
+ _WEB_SEARCH_MODEL_VERSIONS: dict[str, str] = {
168
+ # Supports web_search_20260209 (dynamic filtering)
169
+ "claude-opus-4-7": "web_search_20260209",
170
+ "claude-opus-4-6": "web_search_20260209",
171
+ "claude-sonnet-4-6": "web_search_20260209",
172
+ # Older Claude 4 models — use web_search_20250305 (no dynamic filtering)
173
+ "claude-opus-4-5": "web_search_20250305",
174
+ "claude-opus-4-1": "web_search_20250305",
175
+ "claude-opus-4-0": "web_search_20250305",
176
+ "claude-sonnet-4-5": "web_search_20250305",
177
+ "claude-sonnet-4-0": "web_search_20250305",
178
+ "claude-haiku-4-5": "web_search_20250305", # conservative default
179
+ }
180
+
181
+
182
+ def _infer_web_search_tool_version(model: str) -> str:
183
+ """Pick the ``web_search`` tool version best supported by *model*.
184
+
185
+ Returns ``web_search_20260209`` (dynamic filtering) for Opus 4.7, Opus
186
+ 4.6, and Sonnet 4.6 — the models Anthropic explicitly lists as
187
+ supporting the newer version. Returns ``web_search_20250305`` (no
188
+ dynamic filtering) for older Claude 4 models. Unknown models default
189
+ to the newer version; the caller can pass ``tool_version`` explicitly
190
+ to override.
191
+ """
192
+ known = _WEB_SEARCH_MODEL_VERSIONS.get(model)
193
+ if known:
194
+ return known
195
+ return "web_search_20260209"
196
+
197
+
198
+ @dataclass
199
+ class AnthropicWebSearchBackend:
200
+ """Web-search backend powered by the Anthropic Messages API ``web_search``.
201
+
202
+ By default the tool version is **auto-selected from ``model``** via
203
+ :func:`_infer_web_search_tool_version`:
204
+
205
+ ========================== ==========================
206
+ Model tool_version
207
+ ========================== ==========================
208
+ claude-opus-4-7 (default) web_search_20260209
209
+ claude-opus-4-6 web_search_20260209
210
+ claude-sonnet-4-6 web_search_20260209
211
+ claude-opus-4-5 / 4-1 / 4-0 web_search_20250305
212
+ claude-sonnet-4-5 / 4-0 web_search_20250305
213
+ claude-haiku-4-5 web_search_20250305
214
+ unknown web_search_20260209
215
+ ========================== ==========================
216
+
217
+ Pass an explicit ``tool_version`` to override (useful when a new model
218
+ ships before the lookup table is updated).
219
+
220
+ Snippets are pulled from citations on Claude's text blocks (each citation
221
+ carries ``url``, ``title``, and up to 150 chars of ``cited_text``). If
222
+ Claude returns no citations, the backend falls back to the raw
223
+ ``web_search_tool_result`` blocks for ``url`` + ``title`` + ``page_age``
224
+ (the underlying ``encrypted_content`` is opaque to callers by design).
225
+
226
+ Prompt caching is intentionally not used: a web-search call's cacheable
227
+ prefix (tool definition plus short user prompt) is well under Opus
228
+ 4.7's 4096-token minimum, so caching would silently no-op.
229
+ """
230
+
231
+ client: Any
232
+ model: str = "claude-opus-4-7"
233
+ tool_version: str | None = None # None = auto-select from model
234
+ max_uses: int = 5
235
+ allowed_domains: list[str] | None = None
236
+ blocked_domains: list[str] | None = None
237
+ user_location: dict[str, object] | None = None
238
+ max_tokens: int = 8192
239
+
240
+ def search(
241
+ self,
242
+ query: str,
243
+ max_results: int = 5,
244
+ ) -> list[dict[str, str]]:
245
+ if not query.strip() or max_results <= 0:
246
+ return []
247
+
248
+ tool_version = (
249
+ self.tool_version
250
+ or _infer_web_search_tool_version(self.model)
251
+ )
252
+ tool: dict[str, object] = {
253
+ "type": tool_version,
254
+ "name": "web_search",
255
+ "max_uses": self.max_uses,
256
+ }
257
+ if self.allowed_domains:
258
+ tool["allowed_domains"] = self.allowed_domains
259
+ if self.blocked_domains:
260
+ tool["blocked_domains"] = self.blocked_domains
261
+ if self.user_location:
262
+ tool["user_location"] = self.user_location
263
+
264
+ prompt = (
265
+ f"Search the web for: {query.strip()}\n"
266
+ f"Return up to {max_results} highly relevant sources. "
267
+ "Cite every factual statement you make inline so that every "
268
+ "source you used appears in a citation."
269
+ )
270
+
271
+ response = self.client.messages.create(
272
+ model=self.model,
273
+ max_tokens=self.max_tokens,
274
+ tools=[tool],
275
+ messages=[{"role": "user", "content": prompt}],
276
+ )
277
+
278
+ return _extract_anthropic_web_results(response, max_results)
279
+
280
+
281
+ @dataclass
282
+ class GeminiWebSearchBackend:
283
+ """Web-search backend powered by Google Gemini's grounding tool.
284
+
285
+ Built against the new ``google.genai`` SDK. Calls
286
+ ``client.models.generate_content`` with the ``google_search`` tool
287
+ enabled (Gemini 2.x grounding). The model performs the search and
288
+ synthesises results; we parse the response for a JSON list of
289
+ ``title``/``snippet``/``url`` items, falling back to
290
+ ``grounding_metadata.grounding_chunks`` for URL+title recovery if
291
+ JSON parsing fails.
292
+
293
+ Parameters
294
+ ----------
295
+ client:
296
+ A ``google.genai.Client`` instance.
297
+ model:
298
+ Gemini model identifier supporting grounding (e.g.
299
+ ``"gemini-2.5-flash"`` or ``"gemini-2.5-pro"``).
300
+ max_tokens:
301
+ Output budget for the search-and-summarise call.
302
+ """
303
+
304
+ client: Any
305
+ model: str = "gemini-2.5-flash"
306
+ max_tokens: int = 8192
307
+
308
+ def search(
309
+ self,
310
+ query: str,
311
+ max_results: int = 5,
312
+ ) -> list[dict[str, str]]:
313
+ if not query.strip() or max_results <= 0:
314
+ return []
315
+
316
+ prompt = (
317
+ f"Search the web for: {query.strip()}\n"
318
+ f"Return up to {max_results} highly relevant results as JSON only.\n"
319
+ "Schema: "
320
+ '{"results":[{"title":"...","url":"...","snippet":"..."}]}\n'
321
+ "Rules:\n"
322
+ "- Use only information grounded in the web search results.\n"
323
+ "- Keep snippets concise and factual.\n"
324
+ "- Do not invent URLs.\n"
325
+ "- Return JSON only.\n"
326
+ )
327
+
328
+ try:
329
+ # google.genai 1.x: tools accept a list of dicts; the
330
+ # google_search tool is enabled with an empty config.
331
+ response = self.client.models.generate_content(
332
+ model=self.model,
333
+ contents=prompt,
334
+ config={
335
+ "max_output_tokens": self.max_tokens,
336
+ "tools": [{"google_search": {}}],
337
+ },
338
+ )
339
+ except Exception:
340
+ return []
341
+
342
+ # Pull the model's text (may raise if blocked).
343
+ try:
344
+ output_text = response.text or ""
345
+ except Exception:
346
+ output_text = ""
347
+
348
+ raw_obj = _extract_json_object(output_text)
349
+ parsed_results: list[dict[str, str]] = []
350
+ if isinstance(raw_obj, dict):
351
+ parsed_results = _normalise_backend_results(
352
+ raw_obj.get("results", []),
353
+ max_results=max_results,
354
+ )
355
+
356
+ # Fallback: walk grounding_metadata.grounding_chunks for url + title.
357
+ grounding_results = _extract_gemini_grounding_results(
358
+ response, max_results
359
+ )
360
+ if parsed_results:
361
+ merged: list[dict[str, str]] = []
362
+ seen: set[tuple[str, str]] = set()
363
+ _merge_unique_results(
364
+ merged,
365
+ parsed_results,
366
+ seen=seen,
367
+ max_results=max_results,
368
+ )
369
+ _merge_unique_results(
370
+ merged,
371
+ grounding_results,
372
+ seen=seen,
373
+ max_results=max_results,
374
+ )
375
+ return merged[:max_results]
376
+ return grounding_results[:max_results]
377
+
378
+
379
+ @dataclass
380
+ class GrokWebSearchBackend:
381
+ """Web-search backend powered by xAI Grok's Live Search tool.
382
+
383
+ Live Search is invoked by passing ``extra_body={"search_parameters":
384
+ {...}}`` to a chat-completions call. The response includes
385
+ ``citations`` (a list of URLs) plus the assistant message with
386
+ inline citations. We ask the model to return JSON; if that fails,
387
+ we fall back to ``response.citations`` for URL recovery.
388
+
389
+ Parameters
390
+ ----------
391
+ client:
392
+ An ``openai.OpenAI`` instance configured with
393
+ ``base_url="https://api.x.ai/v1"``.
394
+ model:
395
+ Grok model identifier (e.g., ``"grok-3"``, ``"grok-2"``).
396
+ search_mode:
397
+ xAI's ``mode`` for Live Search: ``"auto"`` (default — model
398
+ decides), ``"on"`` (always search), or ``"off"`` (no search).
399
+ max_search_results:
400
+ Upper bound on internal search calls per request (xAI default
401
+ is 15).
402
+ sources:
403
+ Override the default Live Search sources. Default is
404
+ ``[{"type": "web"}, {"type": "x"}]`` — web + Twitter/X.
405
+ max_tokens:
406
+ Output budget for the search-and-summarise call.
407
+ """
408
+
409
+ client: Any
410
+ model: str = "grok-3"
411
+ search_mode: str = "auto"
412
+ max_search_results: int = 15
413
+ sources: list[dict[str, object]] | None = None
414
+ max_tokens: int = 8192
415
+
416
+ def search(
417
+ self,
418
+ query: str,
419
+ max_results: int = 5,
420
+ ) -> list[dict[str, str]]:
421
+ if not query.strip() or max_results <= 0:
422
+ return []
423
+
424
+ prompt = (
425
+ f"Search the web for: {query.strip()}\n"
426
+ f"Return up to {max_results} highly relevant results as JSON only.\n"
427
+ "Schema: "
428
+ '{"results":[{"title":"...","url":"...","snippet":"..."}]}\n'
429
+ "Rules:\n"
430
+ "- Use only information grounded in your live search results.\n"
431
+ "- Keep snippets concise and factual.\n"
432
+ "- Do not invent URLs.\n"
433
+ "- Return JSON only.\n"
434
+ )
435
+
436
+ sources = self.sources or [{"type": "web"}, {"type": "x"}]
437
+ try:
438
+ response = self.client.chat.completions.create(
439
+ model=self.model,
440
+ messages=[{"role": "user", "content": prompt}],
441
+ max_tokens=self.max_tokens,
442
+ extra_body={
443
+ "search_parameters": {
444
+ "mode": self.search_mode,
445
+ "max_search_results": self.max_search_results,
446
+ "sources": sources,
447
+ },
448
+ },
449
+ )
450
+ except Exception:
451
+ return []
452
+
453
+ message_text = ""
454
+ if getattr(response, "choices", None):
455
+ choice = response.choices[0]
456
+ message = getattr(choice, "message", None)
457
+ if message is not None:
458
+ message_text = getattr(message, "content", "") or ""
459
+
460
+ raw_obj = _extract_json_object(message_text)
461
+ parsed_results: list[dict[str, str]] = []
462
+ if isinstance(raw_obj, dict):
463
+ parsed_results = _normalise_backend_results(
464
+ raw_obj.get("results", []),
465
+ max_results=max_results,
466
+ )
467
+
468
+ # Fallback: harvest citations (URLs) when JSON parsing fails.
469
+ citation_results = _extract_grok_citation_results(
470
+ response, max_results
471
+ )
472
+ if parsed_results:
473
+ merged: list[dict[str, str]] = []
474
+ seen: set[tuple[str, str]] = set()
475
+ _merge_unique_results(
476
+ merged,
477
+ parsed_results,
478
+ seen=seen,
479
+ max_results=max_results,
480
+ )
481
+ _merge_unique_results(
482
+ merged,
483
+ citation_results,
484
+ seen=seen,
485
+ max_results=max_results,
486
+ )
487
+ return merged[:max_results]
488
+ return citation_results[:max_results]
489
+
490
+
491
+ def _extract_gemini_grounding_results(
492
+ response: object,
493
+ max_results: int,
494
+ ) -> list[dict[str, str]]:
495
+ """Walk a Gemini response for ``grounding_chunks[].web`` URL+title pairs."""
496
+ candidates = getattr(response, "candidates", None) or []
497
+ out: list[dict[str, str]] = []
498
+ seen_urls: set[str] = set()
499
+ for candidate in candidates:
500
+ meta = getattr(candidate, "grounding_metadata", None)
501
+ if meta is None:
502
+ continue
503
+ chunks = getattr(meta, "grounding_chunks", None) or []
504
+ for chunk in chunks:
505
+ web = getattr(chunk, "web", None)
506
+ if web is None:
507
+ continue
508
+ url = (getattr(web, "uri", "") or "").strip()
509
+ title = (getattr(web, "title", "") or "").strip() or url
510
+ if not url or url in seen_urls:
511
+ continue
512
+ seen_urls.add(url)
513
+ out.append({"title": title, "snippet": "", "url": url})
514
+ if len(out) >= max_results:
515
+ return out
516
+ return out
517
+
518
+
519
+ def _extract_grok_citation_results(
520
+ response: object,
521
+ max_results: int,
522
+ ) -> list[dict[str, str]]:
523
+ """Harvest ``response.citations`` (URLs) as fallback web results."""
524
+ citations = getattr(response, "citations", None) or []
525
+ out: list[dict[str, str]] = []
526
+ seen_urls: set[str] = set()
527
+ for cite in citations:
528
+ # `citations` may be a list of strings (URLs) or dict-like objects
529
+ # depending on xAI SDK version. Handle both shapes.
530
+ if isinstance(cite, str):
531
+ url = cite.strip()
532
+ title = url
533
+ snippet = ""
534
+ elif isinstance(cite, dict):
535
+ url = str(cite.get("url") or "").strip()
536
+ title = str(cite.get("title") or url).strip()
537
+ snippet = str(cite.get("snippet") or cite.get("excerpt") or "").strip()
538
+ else:
539
+ url = str(getattr(cite, "url", "") or "").strip()
540
+ title = str(getattr(cite, "title", url) or url).strip()
541
+ snippet = str(getattr(cite, "snippet", "") or "").strip()
542
+ if not url or url in seen_urls:
543
+ continue
544
+ seen_urls.add(url)
545
+ out.append({"title": title, "snippet": snippet, "url": url})
546
+ if len(out) >= max_results:
547
+ break
548
+ return out
549
+
550
+
551
+ # -------------------------------------------------------------------
552
+ # Stdlib fallback — zero external dependencies
553
+ # -------------------------------------------------------------------
554
+
555
+ class _DuckDuckGoLiteParser(html.parser.HTMLParser):
556
+ """Minimal parser for DuckDuckGo Lite HTML results page."""
557
+
558
+ def __init__(self) -> None:
559
+ super().__init__()
560
+ self.results: list[dict[str, str]] = []
561
+ self._in_link = False
562
+ self._in_snippet = False
563
+ self._current: dict[str, str] = {}
564
+ self._text_buf: list[str] = []
565
+
566
+ # DuckDuckGo Lite wraps each result link in
567
+ # <a rel="nofollow" class="result-link" href="…">
568
+ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
569
+ attr = dict(attrs)
570
+ if tag == "a" and attr.get("class") == "result-link":
571
+ self._in_link = True
572
+ self._current = {"url": attr.get("href", ""), "title": "", "snippet": ""}
573
+ self._text_buf = []
574
+ elif tag == "td" and attr.get("class") == "result-snippet":
575
+ self._in_snippet = True
576
+ self._text_buf = []
577
+
578
+ def handle_endtag(self, tag: str) -> None:
579
+ if tag == "a" and self._in_link:
580
+ self._in_link = False
581
+ self._current["title"] = " ".join(self._text_buf).strip()
582
+ elif tag == "td" and self._in_snippet:
583
+ self._in_snippet = False
584
+ self._current["snippet"] = " ".join(self._text_buf).strip()
585
+ if self._current.get("url"):
586
+ self.results.append(self._current)
587
+ self._current = {}
588
+
589
+ def handle_data(self, data: str) -> None:
590
+ if self._in_link or self._in_snippet:
591
+ self._text_buf.append(data)
592
+
593
+
594
+ def _resolve_ddg_url(raw: str) -> str:
595
+ """Extract the real destination URL from a DuckDuckGo redirect link.
596
+
597
+ DuckDuckGo Lite wraps result links as
598
+ ``//duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com&rut=…``.
599
+ This function extracts the ``uddg`` parameter value, falling back
600
+ to the original URL if it doesn't match the redirect pattern.
601
+ """
602
+ if "duckduckgo.com/l/" in raw and "uddg=" in raw:
603
+ parsed = urllib.parse.urlparse(raw if "://" in raw else "https:" + raw)
604
+ params = urllib.parse.parse_qs(parsed.query)
605
+ uddg = params.get("uddg", [None])[0]
606
+ if uddg:
607
+ return uddg
608
+ return raw
609
+
610
+
611
+ def _search_stdlib(query: str, max_results: int) -> list[dict[str, str]]:
612
+ """Search DuckDuckGo using only the standard library.
613
+
614
+ Handles network errors, timeouts, and malformed HTML gracefully
615
+ by returning an empty list on failure.
616
+ """
617
+ # Guard against excessively long queries that could exceed URL limits
618
+ if len(query) > 2000:
619
+ query = query[:2000]
620
+
621
+ url = "https://lite.duckduckgo.com/lite/?" + urllib.parse.urlencode({"q": query})
622
+ req = urllib.request.Request(
623
+ url,
624
+ headers={
625
+ "User-Agent": (
626
+ "Mozilla/5.0 (compatible; metacoupling-python/0.1; "
627
+ "+https://github.com/metacoupling/metacoupling)"
628
+ ),
629
+ },
630
+ )
631
+ try:
632
+ with urllib.request.urlopen(req, timeout=15) as resp:
633
+ body = resp.read().decode("utf-8", errors="replace")
634
+ except (urllib.error.URLError, urllib.error.HTTPError, socket.timeout,
635
+ OSError) as exc:
636
+ print(f"[WebSearch] stdlib network error: {exc}")
637
+ return []
638
+
639
+ try:
640
+ parser = _DuckDuckGoLiteParser()
641
+ parser.feed(body)
642
+ except Exception as exc:
643
+ print(f"[WebSearch] HTML parsing error: {exc}")
644
+ return []
645
+
646
+ # Resolve redirect URLs to actual destinations
647
+ for r in parser.results:
648
+ r["url"] = _resolve_ddg_url(r["url"])
649
+
650
+ return parser.results[:max_results]
651
+
652
+
653
+ def _search_ddgs(query: str, max_results: int) -> list[dict[str, str]]:
654
+ """Search using the modern ``ddgs`` backend."""
655
+ from ddgs import DDGS # type: ignore[import-untyped]
656
+
657
+ results: list[dict[str, str]] = []
658
+ for r in DDGS().text(query, max_results=max_results):
659
+ results.append({
660
+ "title": r.get("title", ""),
661
+ "snippet": r.get("body", ""),
662
+ "url": _resolve_ddg_url(r.get("href", "")),
663
+ })
664
+ return results
665
+
666
+
667
+ def _search_duckduckgo_search(
668
+ query: str,
669
+ max_results: int,
670
+ ) -> list[dict[str, str]]:
671
+ """Search using the legacy ``duckduckgo_search`` backend."""
672
+ import warnings
673
+
674
+ with warnings.catch_warnings():
675
+ warnings.filterwarnings(
676
+ "ignore",
677
+ category=RuntimeWarning,
678
+ module=r"duckduckgo_search(\..*)?",
679
+ )
680
+ from duckduckgo_search import DDGS as DDGS2 # type: ignore[import-untyped]
681
+
682
+ results: list[dict[str, str]] = []
683
+ for r in DDGS2().text(query, max_results=max_results):
684
+ results.append({
685
+ "title": r.get("title", ""),
686
+ "snippet": r.get("body", ""),
687
+ "url": _resolve_ddg_url(r.get("href", "")),
688
+ })
689
+ return results
690
+
691
+
692
+ def _result_key(result: dict[str, str]) -> tuple[str, str]:
693
+ """Build a stable dedupe key for a search result."""
694
+ url = result.get("url", "").strip().lower().rstrip("/")
695
+ if url:
696
+ return ("url", url)
697
+ title = " ".join(result.get("title", "").lower().split())
698
+ snippet = " ".join(result.get("snippet", "").lower().split())
699
+ return ("text", f"{title}|{snippet}")
700
+
701
+
702
+ def _normalise_backend_results(
703
+ results: object,
704
+ *,
705
+ max_results: int,
706
+ ) -> list[dict[str, str]]:
707
+ """Normalise backend output to ``title``/``snippet``/``url`` dicts."""
708
+ if not isinstance(results, list):
709
+ return []
710
+
711
+ normalised: list[dict[str, str]] = []
712
+ for item in results:
713
+ if not isinstance(item, dict):
714
+ continue
715
+ title = str(item.get("title", "")).strip()
716
+ snippet = str(item.get("snippet", "")).strip()
717
+ url = str(item.get("url", "")).strip()
718
+ if not url:
719
+ continue
720
+ normalised.append({
721
+ "title": title,
722
+ "snippet": snippet,
723
+ "url": url,
724
+ })
725
+ if len(normalised) >= max_results:
726
+ break
727
+ return normalised
728
+
729
+
730
+ def _extract_openai_source_results(
731
+ response: object,
732
+ max_results: int,
733
+ ) -> list[dict[str, str]]:
734
+ """Best-effort extraction of ``url``/``title`` source rows from a response."""
735
+ raw: object
736
+ if hasattr(response, "model_dump"):
737
+ try:
738
+ raw = response.model_dump()
739
+ except Exception:
740
+ raw = None
741
+ elif isinstance(response, dict):
742
+ raw = response
743
+ else:
744
+ raw = None
745
+
746
+ if not isinstance(raw, (dict, list)):
747
+ return []
748
+
749
+ seen_urls: set[str] = set()
750
+ found: list[dict[str, str]] = []
751
+
752
+ def _walk(node: object) -> None:
753
+ if len(found) >= max_results:
754
+ return
755
+ if isinstance(node, dict):
756
+ url = node.get("url")
757
+ title = node.get("title")
758
+ if isinstance(url, str) and url.strip():
759
+ clean_url = url.strip()
760
+ if clean_url not in seen_urls:
761
+ seen_urls.add(clean_url)
762
+ found.append({
763
+ "title": str(title).strip() if title else clean_url,
764
+ "snippet": str(
765
+ node.get("snippet")
766
+ or node.get("excerpt")
767
+ or node.get("description")
768
+ or ""
769
+ ).strip(),
770
+ "url": clean_url,
771
+ })
772
+ if len(found) >= max_results:
773
+ return
774
+ for value in node.values():
775
+ _walk(value)
776
+ elif isinstance(node, list):
777
+ for item in node:
778
+ _walk(item)
779
+
780
+ _walk(raw)
781
+ return found[:max_results]
782
+
783
+
784
+ def _extract_anthropic_web_results(
785
+ response: object,
786
+ max_results: int,
787
+ ) -> list[dict[str, str]]:
788
+ """Extract ``title``/``snippet``/``url`` rows from an Anthropic response.
789
+
790
+ Primary path: walk the ``citations`` lists on each ``text`` content block
791
+ — every citation carries the source URL, title, and up to 150 chars of
792
+ ``cited_text`` (the snippet). Dedupe by URL, take the first citation's
793
+ ``cited_text`` as the snippet when a URL is cited multiple times.
794
+
795
+ Fallback path: when Claude returned search results but no citations,
796
+ walk the ``web_search_tool_result`` blocks for ``url`` + ``title``.
797
+ ``encrypted_content`` is opaque by design, so snippets in this path
798
+ degrade to just the ``page_age`` timestamp.
799
+ """
800
+ raw: object
801
+ if hasattr(response, "model_dump"):
802
+ try:
803
+ raw = response.model_dump()
804
+ except Exception:
805
+ raw = None
806
+ elif isinstance(response, dict):
807
+ raw = response
808
+ else:
809
+ raw = None
810
+
811
+ if not isinstance(raw, dict):
812
+ return []
813
+
814
+ content = raw.get("content")
815
+ if not isinstance(content, list):
816
+ return []
817
+
818
+ seen_urls: set[str] = set()
819
+ found: list[dict[str, str]] = []
820
+
821
+ # Primary: citations on text blocks
822
+ for block in content:
823
+ if not isinstance(block, dict) or block.get("type") != "text":
824
+ continue
825
+ citations = block.get("citations")
826
+ if not isinstance(citations, list):
827
+ continue
828
+ for cit in citations:
829
+ if not isinstance(cit, dict):
830
+ continue
831
+ if cit.get("type") != "web_search_result_location":
832
+ continue
833
+ url = str(cit.get("url", "") or "").strip()
834
+ if not url or url in seen_urls:
835
+ continue
836
+ seen_urls.add(url)
837
+ title = str(cit.get("title", "") or "").strip() or url
838
+ snippet = str(cit.get("cited_text", "") or "").strip()
839
+ found.append({"title": title, "snippet": snippet, "url": url})
840
+ if len(found) >= max_results:
841
+ return found
842
+
843
+ # Fallback: web_search_tool_result blocks when no citations were emitted
844
+ if not found:
845
+ for block in content:
846
+ if not isinstance(block, dict):
847
+ continue
848
+ if block.get("type") != "web_search_tool_result":
849
+ continue
850
+ tr_content = block.get("content")
851
+ if not isinstance(tr_content, list):
852
+ continue
853
+ for res in tr_content:
854
+ if not isinstance(res, dict):
855
+ continue
856
+ if res.get("type") != "web_search_result":
857
+ continue
858
+ url = str(res.get("url", "") or "").strip()
859
+ if not url or url in seen_urls:
860
+ continue
861
+ seen_urls.add(url)
862
+ title = str(res.get("title", "") or "").strip() or url
863
+ page_age = str(res.get("page_age", "") or "").strip()
864
+ snippet = f"(page age: {page_age})" if page_age else ""
865
+ found.append({"title": title, "snippet": snippet, "url": url})
866
+ if len(found) >= max_results:
867
+ return found
868
+
869
+ return found[:max_results]
870
+
871
+
872
+ def _merge_unique_results(
873
+ merged: list[dict[str, str]],
874
+ new_results: list[dict[str, str]],
875
+ *,
876
+ seen: set[tuple[str, str]],
877
+ max_results: int,
878
+ ) -> None:
879
+ """Append unique results to *merged* until *max_results* is reached."""
880
+ for result in new_results:
881
+ key = _result_key(result)
882
+ if key in seen:
883
+ continue
884
+ seen.add(key)
885
+ merged.append(result)
886
+ if len(merged) >= max_results:
887
+ return
888
+
889
+
890
+ def _extract_query_phrases(query: str, max_words: int = 4) -> list[str]:
891
+ """Return overlapping word phrases from longest to shortest."""
892
+ words = re.findall(r"[A-Za-z][A-Za-z'’-]*", query)
893
+ phrases: list[str] = []
894
+ seen: set[str] = set()
895
+ for size in range(max_words, 0, -1):
896
+ for idx in range(0, len(words) - size + 1):
897
+ phrase = " ".join(words[idx: idx + size]).strip()
898
+ if not phrase:
899
+ continue
900
+ key = phrase.lower()
901
+ if key in seen:
902
+ continue
903
+ seen.add(key)
904
+ phrases.append(phrase)
905
+ return phrases
906
+
907
+
908
+ def _strip_possessive(text: str) -> str:
909
+ """Strip a trailing possessive marker from *text*."""
910
+ return re.sub(r"(?:'s|’s)$", "", text.strip(), flags=re.IGNORECASE)
911
+
912
+
913
+ def _detect_subnational_focus(query: str) -> tuple[str | None, str | None]:
914
+ """Return ADM1 place and parent country when the query is subnational."""
915
+ try:
916
+ from metacouplingllm.knowledge.adm1_pericoupling import (
917
+ get_adm1_info,
918
+ resolve_adm1_code,
919
+ )
920
+ from metacouplingllm.knowledge.countries import (
921
+ get_country_name,
922
+ resolve_country_code,
923
+ )
924
+ except Exception:
925
+ return None, None
926
+
927
+ for phrase in _extract_query_phrases(query):
928
+ cleaned = _strip_possessive(phrase)
929
+ if not cleaned:
930
+ continue
931
+ code = resolve_adm1_code(cleaned)
932
+ if not code:
933
+ continue
934
+ info = get_adm1_info(code)
935
+ if not info:
936
+ return cleaned, None
937
+ country_name = info.get("country_name")
938
+ country_code = resolve_country_code(country_name or "")
939
+ if country_code:
940
+ country_name = get_country_name(country_code)
941
+ return info.get("name", cleaned), country_name
942
+
943
+ return None, None
944
+
945
+
946
+ def _extract_topic_terms(
947
+ query: str,
948
+ *,
949
+ focal_place: str | None,
950
+ focal_country: str | None,
951
+ max_terms: int = 3,
952
+ ) -> list[str]:
953
+ """Extract likely commodity/topic terms from a research query."""
954
+ lowered = query.replace("’", "'").lower()
955
+
956
+ # Strip focal place/country names AND their ISO codes so they
957
+ # don't appear as topic terms (e.g., "usa" from "USA").
958
+ try:
959
+ from metacouplingllm.knowledge.countries import resolve_country_code
960
+
961
+ _extra_strips: list[str] = []
962
+ for phrase in (focal_place, focal_country):
963
+ if phrase:
964
+ code = resolve_country_code(phrase)
965
+ if code:
966
+ _extra_strips.append(code.lower())
967
+ except Exception:
968
+ _extra_strips = []
969
+
970
+ for phrase in (focal_place, focal_country):
971
+ if not phrase:
972
+ continue
973
+ lowered = re.sub(
974
+ rf"\b{re.escape(phrase.lower())}(?:'s)?\b",
975
+ " ",
976
+ lowered,
977
+ )
978
+ for code_lower in _extra_strips:
979
+ lowered = re.sub(rf"\b{re.escape(code_lower)}\b", " ", lowered)
980
+
981
+ topic_terms: list[str] = []
982
+ seen: set[str] = set()
983
+ for token in re.findall(r"[a-z][a-z'-]{2,}", lowered):
984
+ token = _strip_possessive(token.lower())
985
+ if (
986
+ token in _QUERY_FILLER
987
+ or token in _STRUCTURAL_TERMS
988
+ or token in _TRADE_HINTS
989
+ ):
990
+ continue
991
+ if token in seen:
992
+ continue
993
+ seen.add(token)
994
+ topic_terms.append(token)
995
+ if len(topic_terms) >= max_terms:
996
+ break
997
+
998
+ return topic_terms
999
+
1000
+
1001
+ def _has_trade_intent(query: str) -> bool:
1002
+ """Return True when the research query is about trade/export partners."""
1003
+ tokens = {
1004
+ _strip_possessive(token.lower())
1005
+ for token in re.findall(r"[A-Za-z][A-Za-z’’-]*", query)
1006
+ }
1007
+ return any(token in _TRADE_HINTS for token in tokens)
1008
+
1009
+
1010
+ def _detect_national_focus(query: str) -> str | None:
1011
+ """Extract a country name from the query when no subnational region found.
1012
+
1013
+ E.g., ``"corn production and exports in USA"`` → ``"United States"``.
1014
+ """
1015
+ try:
1016
+ from metacouplingllm.knowledge.countries import (
1017
+ get_country_name,
1018
+ resolve_country_code,
1019
+ )
1020
+ except Exception:
1021
+ return None
1022
+
1023
+ for phrase in _extract_query_phrases(query):
1024
+ cleaned = _strip_possessive(phrase)
1025
+ if not cleaned:
1026
+ continue
1027
+ code = resolve_country_code(cleaned)
1028
+ if code:
1029
+ return get_country_name(code) or cleaned
1030
+
1031
+ return None
1032
+
1033
+
1034
+ def _build_search_queries(query: str) -> list[str]:
1035
+ """Expand a raw research query into higher-signal search variants."""
1036
+ base_query = " ".join(query.split())
1037
+ if not base_query:
1038
+ return []
1039
+
1040
+ if not _has_trade_intent(base_query):
1041
+ return [base_query]
1042
+
1043
+ # Try subnational focus first (e.g., "Michigan pork exports")
1044
+ focal_place, focal_country = _detect_subnational_focus(base_query)
1045
+
1046
+ # If no subnational focus, try national (e.g., "USA corn exports")
1047
+ if not focal_place:
1048
+ focal_country = _detect_national_focus(base_query)
1049
+ if not focal_country:
1050
+ return [base_query]
1051
+ focal_place = focal_country
1052
+
1053
+ topic_terms = _extract_topic_terms(
1054
+ base_query,
1055
+ focal_place=focal_place,
1056
+ focal_country=focal_country,
1057
+ )
1058
+
1059
+ queries: list[str] = []
1060
+ seen: set[str] = set()
1061
+
1062
+ def add(candidate: str) -> None:
1063
+ cleaned = " ".join(candidate.split()).strip()
1064
+ if not cleaned:
1065
+ return
1066
+ key = cleaned.lower()
1067
+ if key in seen:
1068
+ return
1069
+ seen.add(key)
1070
+ queries.append(cleaned)
1071
+
1072
+ if topic_terms:
1073
+ for term in topic_terms:
1074
+ add(f"{focal_place} {term} export destinations")
1075
+ add(f"{focal_place} {term} trade partners")
1076
+ add(f"{focal_place} {term} export markets")
1077
+ if focal_country:
1078
+ add(f"{focal_country} {term} exports top destinations")
1079
+ add(f"{focal_country} {term} export markets")
1080
+ else:
1081
+ add(f"{focal_place} export destinations")
1082
+ add(f"{focal_place} trade partners")
1083
+ add(f"{focal_place} export markets")
1084
+ if focal_country:
1085
+ add(f"{focal_country} exports top destinations")
1086
+ add(f"{focal_country} export markets")
1087
+
1088
+ add(base_query)
1089
+ return queries
1090
+
1091
+
1092
+ def _search_single_query(
1093
+ query: str,
1094
+ max_results: int,
1095
+ ) -> list[dict[str, str]]:
1096
+ """Search one query string across all backends and merge results."""
1097
+ merged: list[dict[str, str]] = []
1098
+ seen: set[tuple[str, str]] = set()
1099
+ backends = (
1100
+ ("ddgs", _search_ddgs),
1101
+ ("duckduckgo_search", _search_duckduckgo_search),
1102
+ ("stdlib fallback", _search_stdlib),
1103
+ )
1104
+
1105
+ for backend_name, backend in backends:
1106
+ try:
1107
+ results = backend(query, max_results)
1108
+ except ImportError:
1109
+ continue
1110
+ except Exception as exc:
1111
+ print(f"[WebSearch] {backend_name} backend failed: {exc}")
1112
+ continue
1113
+
1114
+ if not results:
1115
+ continue
1116
+
1117
+ _merge_unique_results(
1118
+ merged, results, seen=seen, max_results=max_results,
1119
+ )
1120
+ if len(merged) >= max_results:
1121
+ break
1122
+
1123
+ return merged[:max_results]
1124
+
1125
+
1126
+ # -------------------------------------------------------------------
1127
+ # Public API
1128
+ # -------------------------------------------------------------------
1129
+
1130
+ _DUCKDUCKGO_LABEL = "DuckDuckGo (fan-out + top-up)"
1131
+
1132
+
1133
+ def _backend_display_name(backend: object | None) -> str:
1134
+ """Return a human label describing *backend* for logs/console output."""
1135
+ if backend is None:
1136
+ return _DUCKDUCKGO_LABEL
1137
+ if isinstance(backend, AnthropicWebSearchBackend):
1138
+ return f"Claude web_search ({backend.model})"
1139
+ if isinstance(backend, OpenAIWebSearchBackend):
1140
+ return f"OpenAI web_search ({backend.model})"
1141
+ return type(backend).__name__
1142
+
1143
+
1144
+ def search_web(
1145
+ query: str,
1146
+ max_results: int = 5,
1147
+ *,
1148
+ backend: WebSearchBackend | None = None,
1149
+ metadata: dict[str, object] | None = None,
1150
+ ) -> list[dict[str, str]]:
1151
+ """Search the web and return top results.
1152
+
1153
+ Parameters
1154
+ ----------
1155
+ query:
1156
+ Search query string.
1157
+ max_results:
1158
+ Maximum number of results to return.
1159
+ backend:
1160
+ Optional pluggable backend (e.g., ``OpenAIWebSearchBackend``,
1161
+ ``AnthropicWebSearchBackend``). Tried first; falls back to the
1162
+ DuckDuckGo cascade on exception or empty result.
1163
+ metadata:
1164
+ Optional out-dict the caller can pass to learn which backend
1165
+ actually produced the returned results. When provided, it is
1166
+ populated with:
1167
+
1168
+ - ``"backend_used"``: human label of the backend that produced
1169
+ the results (e.g. ``"Claude web_search (claude-opus-4-7)"`` or
1170
+ ``"DuckDuckGo (fan-out + top-up)"``).
1171
+ - ``"fallback_from"`` *(only present when fallback occurred)*: a
1172
+ short reason string like ``"Claude web_search raised: ..."``
1173
+ or ``"OpenAI web_search returned 0 results"``.
1174
+
1175
+ Returns
1176
+ -------
1177
+ A list of dicts, each with ``title``, ``snippet``, and ``url`` keys.
1178
+ Returns an empty list if the search fails or the library is not
1179
+ installed.
1180
+
1181
+ Notes
1182
+ -----
1183
+ When *backend* is provided, it is tried first. If it yields no usable
1184
+ results or raises, the built-in DuckDuckGo-family backends are used.
1185
+
1186
+ The built-in search path attempts three backends in order, and unique
1187
+ results are merged until *max_results* is reached:
1188
+
1189
+ 1. ``ddgs`` (pip install ddgs)
1190
+ 2. ``duckduckgo_search`` (pip install duckduckgo-search)
1191
+ 3. stdlib-only fallback using ``urllib`` + ``html.parser`` — works
1192
+ everywhere including Google Colab without extra packages.
1193
+ """
1194
+ if max_results <= 0:
1195
+ return []
1196
+
1197
+ fallback_reason: str | None = None
1198
+ if backend is not None:
1199
+ try:
1200
+ results = _normalise_backend_results(
1201
+ backend.search(query, max_results=max_results),
1202
+ max_results=max_results,
1203
+ )
1204
+ except Exception as exc:
1205
+ print(f"[WebSearch] custom backend failed: {exc}")
1206
+ fallback_reason = (
1207
+ f"{_backend_display_name(backend)} raised: {exc}"
1208
+ )
1209
+ else:
1210
+ if results:
1211
+ if metadata is not None:
1212
+ metadata["backend_used"] = _backend_display_name(backend)
1213
+ return results[:max_results]
1214
+ fallback_reason = (
1215
+ f"{_backend_display_name(backend)} returned 0 results"
1216
+ )
1217
+
1218
+ queries = _build_search_queries(query)
1219
+ if not queries:
1220
+ if metadata is not None:
1221
+ metadata["backend_used"] = _DUCKDUCKGO_LABEL
1222
+ if fallback_reason is not None:
1223
+ metadata["fallback_from"] = fallback_reason
1224
+ return []
1225
+
1226
+ merged: list[dict[str, str]] = []
1227
+ seen: set[tuple[str, str]] = set()
1228
+ per_query_max = (
1229
+ max_results
1230
+ if len(queries) == 1
1231
+ else max(4, min(max_results, 6))
1232
+ )
1233
+
1234
+ for search_query in queries:
1235
+ results = _search_single_query(search_query, per_query_max)
1236
+ if not results:
1237
+ continue
1238
+
1239
+ _merge_unique_results(
1240
+ merged,
1241
+ results,
1242
+ seen=seen,
1243
+ max_results=max_results,
1244
+ )
1245
+ if len(merged) >= max_results:
1246
+ break
1247
+
1248
+ # Top-up: when fan-out is used (multiple variants) AND the per-variant
1249
+ # cap left us short of max_results, re-issue the raw query at the full
1250
+ # budget. The dedup keys ensure variant URLs are not double-counted.
1251
+ # This recovers cases where a single query has plenty of unique hits
1252
+ # available (e.g., "Michigan pork exports" returns ~47 unique at max=50)
1253
+ # that the per-variant cap of ~6 would otherwise throttle away.
1254
+ if len(queries) > 1 and len(merged) < max_results:
1255
+ topup = _search_single_query(queries[-1], max_results)
1256
+ _merge_unique_results(
1257
+ merged,
1258
+ topup,
1259
+ seen=seen,
1260
+ max_results=max_results,
1261
+ )
1262
+
1263
+ if metadata is not None:
1264
+ metadata["backend_used"] = _DUCKDUCKGO_LABEL
1265
+ if fallback_reason is not None:
1266
+ metadata["fallback_from"] = fallback_reason
1267
+
1268
+ return merged[:max_results]
1269
+
1270
+
1271
+ def format_web_context(results: list[dict[str, str]]) -> str:
1272
+ """Format web search results into a prompt-ready context section.
1273
+
1274
+ Parameters
1275
+ ----------
1276
+ results:
1277
+ List of search result dicts from :func:`search_web`.
1278
+
1279
+ Returns
1280
+ -------
1281
+ A formatted string starting with ``## WEB SEARCH CONTEXT``, or an
1282
+ empty string if no results are provided.
1283
+ """
1284
+ if not results:
1285
+ return ""
1286
+
1287
+ lines: list[str] = [
1288
+ "## WEB SEARCH CONTEXT",
1289
+ "",
1290
+ "The following web search snippets provide real-world context "
1291
+ "for the research topic. Use this information to ground your "
1292
+ "analysis in current facts (e.g., trade volumes, destinations, "
1293
+ "policies), but always apply the metacoupling framework "
1294
+ "rigorously.",
1295
+ "",
1296
+ "For subnational trade topics, some snippets may come from "
1297
+ "broader country-level searches when place-specific trade-partner "
1298
+ "data are sparse. Treat those as proxy context and label them "
1299
+ "clearly rather than presenting them as direct subnational facts.",
1300
+ "",
1301
+ "When you use a fact from a web snippet, cite it inline as "
1302
+ "[W1], [W2], etc. These are distinct from the literature "
1303
+ "citations [1], [2], … that may be added separately.",
1304
+ "",
1305
+ ]
1306
+
1307
+ for i, r in enumerate(results, 1):
1308
+ title = r.get("title", "Untitled")
1309
+ snippet = r.get("snippet", "")
1310
+ url = r.get("url", "")
1311
+ lines.append(f"[W{i}] **{title}**")
1312
+ if snippet:
1313
+ lines.append(f" {snippet}")
1314
+ if url:
1315
+ lines.append(f" Source: {url}")
1316
+ lines.append("")
1317
+
1318
+ return "\n".join(lines)
1319
+
1320
+
1321
+ def _extract_json_object(text: str) -> dict[str, object] | None:
1322
+ """Best-effort extraction of a JSON object from model output.
1323
+
1324
+ Handles common LLM output patterns including:
1325
+ - Bare JSON
1326
+ - JSON inside markdown fenced code blocks
1327
+ - JSON surrounded by explanation text
1328
+ - Truncated JSON (attempts repair by closing brackets)
1329
+ """
1330
+ if not text or not text.strip():
1331
+ return None
1332
+
1333
+ stripped = text.strip()
1334
+ candidates: list[str] = [stripped]
1335
+
1336
+ # Try fenced code blocks first (most reliable)
1337
+ fenced = re.findall(r"```(?:json)?\s*(\{.*?\})\s*```", stripped, re.DOTALL)
1338
+ candidates.extend(fenced)
1339
+
1340
+ # Try extracting between first { and last }
1341
+ first = stripped.find("{")
1342
+ last = stripped.rfind("}")
1343
+ if first != -1 and last != -1 and last > first:
1344
+ candidates.append(stripped[first:last + 1])
1345
+
1346
+ for candidate in candidates:
1347
+ try:
1348
+ obj = json.loads(candidate)
1349
+ except (json.JSONDecodeError, ValueError):
1350
+ continue
1351
+ if isinstance(obj, dict):
1352
+ return obj
1353
+
1354
+ # Last resort: try to repair truncated JSON by closing open brackets.
1355
+ # This handles the case where max_tokens cut the response mid-JSON.
1356
+ if first != -1:
1357
+ raw = stripped[first:]
1358
+ # Count unclosed braces/brackets and close them
1359
+ open_braces = raw.count("{") - raw.count("}")
1360
+ open_brackets = raw.count("[") - raw.count("]")
1361
+ if open_braces > 0 or open_brackets > 0:
1362
+ # Strip any trailing incomplete key-value pair
1363
+ repaired = re.sub(r',\s*"[^"]*"\s*:\s*$', "", raw)
1364
+ repaired = re.sub(r',\s*$', "", repaired)
1365
+ repaired += "]" * max(0, open_brackets) + "}" * max(0, open_braces)
1366
+ try:
1367
+ obj = json.loads(repaired)
1368
+ except (json.JSONDecodeError, ValueError):
1369
+ pass
1370
+ else:
1371
+ if isinstance(obj, dict):
1372
+ return obj
1373
+
1374
+ return None
1375
+
1376
+
1377
+ def _normalise_web_evidence_ids(
1378
+ evidence: object,
1379
+ available_ids: set[str],
1380
+ ) -> list[str]:
1381
+ """Return valid evidence ids like ``['W1', 'W2']``."""
1382
+ if not isinstance(evidence, list):
1383
+ return []
1384
+ cleaned: list[str] = []
1385
+ for item in evidence:
1386
+ if not isinstance(item, str):
1387
+ continue
1388
+ match = re.search(r"W\d+", item.upper())
1389
+ if not match:
1390
+ continue
1391
+ code = match.group(0)
1392
+ if code in available_ids and code not in cleaned:
1393
+ cleaned.append(code)
1394
+ return cleaned
1395
+
1396
+
1397
+ def _coerce_confidence(value: object) -> float:
1398
+ """Convert *value* to a bounded confidence score."""
1399
+ try:
1400
+ score = float(value)
1401
+ except Exception:
1402
+ return 0.0
1403
+ return max(0.0, min(1.0, score))
1404
+
1405
+
1406
+ def extract_web_map_signals(
1407
+ query: str,
1408
+ results: list[dict[str, str]],
1409
+ llm_client: object,
1410
+ *,
1411
+ min_confidence: float = 0.7,
1412
+ max_targets: int = 6,
1413
+ temperature: float = 0.0,
1414
+ max_tokens: int = 8192,
1415
+ ) -> dict[str, object] | None:
1416
+ """Use an LLM to extract validated map-ready signals from web snippets.
1417
+
1418
+ Returns a dict containing ``focal_country``, ``receiving_systems``,
1419
+ ``spillover_systems``, and ``flows`` when extraction succeeds.
1420
+ """
1421
+ if not results:
1422
+ return None
1423
+
1424
+ from metacouplingllm.knowledge.countries import (
1425
+ get_country_name,
1426
+ resolve_country_code,
1427
+ )
1428
+ from metacouplingllm.llm.client import Message
1429
+
1430
+ available_ids = {f"W{i}" for i in range(1, len(results) + 1)}
1431
+ snippet_lines: list[str] = []
1432
+ for idx, result in enumerate(results, 1):
1433
+ snippet_lines.append(f"[W{idx}] {result.get('title', '').strip()}")
1434
+ snippet_lines.append(f"Snippet: {result.get('snippet', '').strip()}")
1435
+ snippet_lines.append(f"URL: {result.get('url', '').strip()}")
1436
+ snippet_lines.append("")
1437
+
1438
+ schema = {
1439
+ "focal_country": "country name, ISO code, or null",
1440
+ "receiving_systems": [
1441
+ {
1442
+ "country": "country name or ISO code",
1443
+ "kind": "direct or proxy",
1444
+ "confidence": 0.0,
1445
+ "evidence": ["W1"],
1446
+ "reason": "short explanation",
1447
+ }
1448
+ ],
1449
+ "spillover_systems": [
1450
+ {
1451
+ "country": "country name or ISO code",
1452
+ "kind": "direct or proxy",
1453
+ "confidence": 0.0,
1454
+ "evidence": ["W2"],
1455
+ "reason": "short explanation",
1456
+ }
1457
+ ],
1458
+ "flows": [
1459
+ {
1460
+ "category": "matter/capital/information/energy/people/organisms",
1461
+ "source_country": "country name or ISO code",
1462
+ "target_country": "country name or ISO code",
1463
+ "kind": "direct or proxy",
1464
+ "confidence": 0.0,
1465
+ "evidence": ["W1"],
1466
+ "description": "short flow description",
1467
+ }
1468
+ ],
1469
+ }
1470
+ system_text = (
1471
+ "You extract conservative, map-ready metacoupling signals from web "
1472
+ "search snippets. Use only the provided snippets. Do not invent "
1473
+ "countries, destinations, or flows. Return JSON only."
1474
+ )
1475
+ user_text = (
1476
+ f"Research query:\n{query.strip()}\n\n"
1477
+ "Extract map-ready countries and flows for a metacoupling map.\n"
1478
+ "Rules:\n"
1479
+ "- Use null or empty lists when uncertain.\n"
1480
+ "- Label items as 'proxy' when the snippet is broader than the focal "
1481
+ "study but still useful context.\n"
1482
+ "- Only include countries that are explicitly supported by the "
1483
+ "snippets.\n"
1484
+ "- ONLY extract countries and flows that are DIRECTLY RELEVANT to "
1485
+ "the research query above. Ignore trade data about unrelated "
1486
+ "products, sectors, or commodities, even if they involve the focal "
1487
+ "country. For example, if the research is about feed barley, do NOT "
1488
+ "extract sheep offal exports or unrelated commodity flows.\n"
1489
+ "- receiving_systems = countries that BUY from or RECEIVE "
1490
+ "goods/services/capital from the focal country (trade partners, "
1491
+ "importers)\n"
1492
+ "- spillover_systems = competitors, indirectly affected countries, "
1493
+ "or countries that experience environmental/economic spillover "
1494
+ "effects\n"
1495
+ "- NEVER put competing exporter countries in receiving_systems. "
1496
+ "Competitors belong in spillover_systems.\n"
1497
+ "- Most flows go FROM the focal country TO receiving countries "
1498
+ "(matter, information, energy, people, organisms).\n"
1499
+ "- Capital/payment flows go in REVERSE: FROM receiving countries "
1500
+ "TO the focal country.\n"
1501
+ "- Do NOT create flows from spillover/competitor countries to "
1502
+ "anyone. For example, if USA exports corn to Mexico "
1503
+ "and Brazil is a competitor, do NOT add a Brazil \u2192 Mexico flow.\n"
1504
+ "- Keep at most 6 receiving systems, 6 spillover systems, and 8 flows.\n"
1505
+ "- Every item must include evidence ids like W1.\n\n"
1506
+ f"JSON schema example:\n{json.dumps(schema, ensure_ascii=True)}\n\n"
1507
+ "Web snippets:\n"
1508
+ f"{chr(10).join(snippet_lines)}"
1509
+ )
1510
+
1511
+ response = llm_client.chat(
1512
+ messages=[
1513
+ Message(role="system", content=system_text),
1514
+ Message(role="user", content=user_text),
1515
+ ],
1516
+ temperature=temperature,
1517
+ max_tokens=max_tokens,
1518
+ )
1519
+ raw_obj = _extract_json_object(response.content)
1520
+ if raw_obj is None:
1521
+ return None
1522
+
1523
+ def _normalise_country_entry(item: object) -> dict[str, object] | None:
1524
+ if not isinstance(item, dict):
1525
+ return None
1526
+ code = resolve_country_code(str(item.get("country", "")).strip())
1527
+ if not code:
1528
+ return None
1529
+ confidence = _coerce_confidence(item.get("confidence", 0.0))
1530
+ evidence = _normalise_web_evidence_ids(
1531
+ item.get("evidence", []),
1532
+ available_ids,
1533
+ )
1534
+ if confidence < min_confidence or not evidence:
1535
+ return None
1536
+ kind = str(item.get("kind", "direct")).strip().lower()
1537
+ if kind not in {"direct", "proxy"}:
1538
+ kind = "direct"
1539
+ reason = str(item.get("reason", "")).strip()
1540
+ return {
1541
+ "country": code,
1542
+ "kind": kind,
1543
+ "confidence": confidence,
1544
+ "evidence": evidence,
1545
+ "reason": reason,
1546
+ }
1547
+
1548
+ def _normalise_flow_entry(item: object) -> dict[str, object] | None:
1549
+ if not isinstance(item, dict):
1550
+ return None
1551
+ src_code = resolve_country_code(str(item.get("source_country", "")).strip())
1552
+ tgt_code = resolve_country_code(str(item.get("target_country", "")).strip())
1553
+ if not src_code or not tgt_code or src_code == tgt_code:
1554
+ return None
1555
+ category = str(item.get("category", "")).strip().lower()
1556
+ if category not in {
1557
+ "matter", "material", "capital", "financial", "information",
1558
+ "energy", "people", "organisms",
1559
+ }:
1560
+ return None
1561
+ confidence = _coerce_confidence(item.get("confidence", 0.0))
1562
+ evidence = _normalise_web_evidence_ids(
1563
+ item.get("evidence", []),
1564
+ available_ids,
1565
+ )
1566
+ if confidence < min_confidence or not evidence:
1567
+ return None
1568
+ kind = str(item.get("kind", "direct")).strip().lower()
1569
+ if kind not in {"direct", "proxy"}:
1570
+ kind = "direct"
1571
+ description = str(item.get("description", "")).strip()
1572
+ src_name = get_country_name(src_code) or src_code
1573
+ tgt_name = get_country_name(tgt_code) or tgt_code
1574
+ return {
1575
+ "category": category,
1576
+ "source_country": src_code,
1577
+ "target_country": tgt_code,
1578
+ "direction": f"{src_name} \u2192 {tgt_name}",
1579
+ "description": description,
1580
+ "kind": kind,
1581
+ "confidence": confidence,
1582
+ "evidence": evidence,
1583
+ }
1584
+
1585
+ focal_country = resolve_country_code(str(raw_obj.get("focal_country", "")).strip())
1586
+ receiving: list[dict[str, object]] = []
1587
+ seen_receiving: set[str] = set()
1588
+ for item in raw_obj.get("receiving_systems", []) if isinstance(raw_obj.get("receiving_systems", []), list) else []:
1589
+ normalized = _normalise_country_entry(item)
1590
+ if not normalized:
1591
+ continue
1592
+ code = str(normalized["country"])
1593
+ if code in seen_receiving:
1594
+ continue
1595
+ seen_receiving.add(code)
1596
+ receiving.append(normalized)
1597
+ if len(receiving) >= max_targets:
1598
+ break
1599
+
1600
+ spillover: list[dict[str, object]] = []
1601
+ seen_spillover: set[str] = set()
1602
+ for item in raw_obj.get("spillover_systems", []) if isinstance(raw_obj.get("spillover_systems", []), list) else []:
1603
+ normalized = _normalise_country_entry(item)
1604
+ if not normalized:
1605
+ continue
1606
+ code = str(normalized["country"])
1607
+ if code in seen_spillover:
1608
+ continue
1609
+ seen_spillover.add(code)
1610
+ spillover.append(normalized)
1611
+ if len(spillover) >= max_targets:
1612
+ break
1613
+
1614
+ flows: list[dict[str, object]] = []
1615
+ seen_flows: set[tuple[str, str, str]] = set()
1616
+ raw_flows = raw_obj.get("flows", [])
1617
+ if isinstance(raw_flows, list):
1618
+ for item in raw_flows:
1619
+ normalized = _normalise_flow_entry(item)
1620
+ if not normalized:
1621
+ continue
1622
+ key = (
1623
+ str(normalized["category"]),
1624
+ str(normalized["source_country"]),
1625
+ str(normalized["target_country"]),
1626
+ )
1627
+ if key in seen_flows:
1628
+ continue
1629
+ seen_flows.add(key)
1630
+ flows.append(normalized)
1631
+ if len(flows) >= max_targets + 2:
1632
+ break
1633
+
1634
+ if not focal_country and receiving and flows:
1635
+ focal_country = str(flows[0]["source_country"])
1636
+
1637
+ if not any((focal_country, receiving, spillover, flows)):
1638
+ return None
1639
+
1640
+ return {
1641
+ "focal_country": focal_country,
1642
+ "receiving_systems": receiving,
1643
+ "spillover_systems": spillover,
1644
+ "flows": flows,
1645
+ }
1646
+
1647
+
1648
+ def format_web_map_signals_context(signals: dict[str, object] | None) -> str:
1649
+ """Format structured web map signals into a prompt-ready section."""
1650
+ if not signals:
1651
+ return ""
1652
+
1653
+ lines = [
1654
+ "## STRUCTURED WEB MAP SIGNALS",
1655
+ "",
1656
+ "The following countries and flows were conservatively extracted "
1657
+ "from the web snippets and validated for mapping. Treat these as "
1658
+ "high-priority map hints, while still reasoning carefully about "
1659
+ "direct versus proxy evidence.",
1660
+ "",
1661
+ ]
1662
+
1663
+ focal = signals.get("focal_country")
1664
+ if isinstance(focal, str) and focal:
1665
+ lines.append(f"- Focal country: {focal}")
1666
+
1667
+ receiving = signals.get("receiving_systems", [])
1668
+ if isinstance(receiving, list) and receiving:
1669
+ lines.append("- Receiving systems:")
1670
+ for item in receiving:
1671
+ if not isinstance(item, dict):
1672
+ continue
1673
+ lines.append(
1674
+ " - "
1675
+ f"{item.get('country')} ({item.get('kind', 'direct')}, "
1676
+ f"confidence={item.get('confidence', 0):.2f}, "
1677
+ f"evidence={','.join(item.get('evidence', []))})"
1678
+ )
1679
+
1680
+ spillover = signals.get("spillover_systems", [])
1681
+ if isinstance(spillover, list) and spillover:
1682
+ lines.append("- Spillover systems:")
1683
+ for item in spillover:
1684
+ if not isinstance(item, dict):
1685
+ continue
1686
+ lines.append(
1687
+ " - "
1688
+ f"{item.get('country')} ({item.get('kind', 'direct')}, "
1689
+ f"confidence={item.get('confidence', 0):.2f}, "
1690
+ f"evidence={','.join(item.get('evidence', []))})"
1691
+ )
1692
+
1693
+ flows = signals.get("flows", [])
1694
+ if isinstance(flows, list) and flows:
1695
+ lines.append("- Map-ready flows:")
1696
+ for item in flows:
1697
+ if not isinstance(item, dict):
1698
+ continue
1699
+ lines.append(
1700
+ " - "
1701
+ f"{item.get('category')}: {item.get('direction')} "
1702
+ f"({item.get('kind', 'direct')}, "
1703
+ f"confidence={item.get('confidence', 0):.2f}, "
1704
+ f"evidence={','.join(item.get('evidence', []))})"
1705
+ )
1706
+
1707
+ return "\n".join(lines)
1708
+
1709
+
1710
+ # -------------------------------------------------------------------
1711
+ # Post-processing: add inline [WN] citations
1712
+ # -------------------------------------------------------------------
1713
+
1714
+ # Stopwords excluded from keyword matching
1715
+ _STOP = frozenset(
1716
+ "a an the and or but in on of to for is are was were be been by "
1717
+ "at as it its with from this that these those not no do does did "
1718
+ "has have had will would shall should can could may might must "
1719
+ "so if than too very also about more most such each every all "
1720
+ "any both few many much some their them they which what when "
1721
+ "where who how because between into through during before after "
1722
+ "above below up down out off over under again further then once "
1723
+ "here there just only own same other another new old well still "
1724
+ "even us our we you your he she him her his".split()
1725
+ )
1726
+
1727
+
1728
+ def _web_tokenise(text: str) -> set[str]:
1729
+ """Extract distinctive lowercase keywords from *text*."""
1730
+ tokens = re.findall(r"[a-z0-9]{3,}", text.lower())
1731
+ return {t for t in tokens if t not in _STOP}
1732
+
1733
+
1734
+ def annotate_web_citations(
1735
+ formatted: str,
1736
+ web_results: list[dict[str, str]],
1737
+ min_keyword_overlap: int = 3,
1738
+ min_overlap_ratio: float = 0.25,
1739
+ ) -> str:
1740
+ """Add inline ``[WN]`` citations to formatted analysis lines.
1741
+
1742
+ Works the same way as the literature ``annotate_citations`` but uses
1743
+ web-search snippets and the ``[W1]``, ``[W2]``, … prefix so that
1744
+ web and literature references are visually distinct.
1745
+
1746
+ Parameters
1747
+ ----------
1748
+ formatted:
1749
+ The formatted analysis text.
1750
+ web_results:
1751
+ Web search result dicts (each with ``title``, ``snippet``, ``url``).
1752
+ min_keyword_overlap:
1753
+ Minimum shared keywords for a citation to be added.
1754
+ min_overlap_ratio:
1755
+ Minimum fraction of the line's keywords that must appear in the
1756
+ web snippet.
1757
+
1758
+ Returns
1759
+ -------
1760
+ The text with ``[WN]`` citations appended to matching lines.
1761
+ """
1762
+ if not web_results:
1763
+ return formatted
1764
+
1765
+ # Build keyword sets for each web snippet
1766
+ snippet_keywords: list[set[str]] = []
1767
+ for r in web_results:
1768
+ combined = (r.get("title", "") + " " + r.get("snippet", "")).strip()
1769
+ snippet_keywords.append(_web_tokenise(combined))
1770
+
1771
+ _skip_prefixes = (
1772
+ "COUPLING CLASSIFICATION", "SYSTEMS IDENTIFICATION",
1773
+ "FLOWS ANALYSIS", "AGENTS", "CAUSES", "EFFECTS",
1774
+ "RESEARCH GAPS", "PERICOUPLING DATABASE",
1775
+ "METACOUPLING FRAMEWORK", "SUPPORTING EVIDENCE",
1776
+ "RECOMMENDED LITERATURE", "WEB SOURCES", "Note:",
1777
+ )
1778
+
1779
+ lines = formatted.split("\n")
1780
+ annotated: list[str] = []
1781
+
1782
+ in_ref_block = False
1783
+
1784
+ for line in lines:
1785
+ stripped = line.strip()
1786
+
1787
+ # Don't annotate inside reference blocks (evidence or web sources)
1788
+ if any(h in stripped for h in (
1789
+ "SUPPORTING EVIDENCE FROM LITERATURE",
1790
+ "WEB SOURCES",
1791
+ )):
1792
+ in_ref_block = True
1793
+ if in_ref_block:
1794
+ annotated.append(line)
1795
+ continue
1796
+
1797
+ # Skip empty lines, separators, and section headers
1798
+ if (
1799
+ not stripped
1800
+ or stripped.startswith(tuple(_skip_prefixes))
1801
+ or set(stripped) <= {"=", "-", " "}
1802
+ ):
1803
+ annotated.append(line)
1804
+ continue
1805
+
1806
+ # Skip sub-headings
1807
+ if stripped.startswith("[") and stripped.endswith("]"):
1808
+ annotated.append(line)
1809
+ continue
1810
+ if stripped.endswith(":") and len(stripped.split()) <= 3:
1811
+ annotated.append(line)
1812
+ continue
1813
+
1814
+ line_tokens = _web_tokenise(stripped)
1815
+ if len(line_tokens) < 3:
1816
+ annotated.append(line)
1817
+ continue
1818
+
1819
+ # Find matching web snippets
1820
+ citations: list[int] = []
1821
+ for idx, sk in enumerate(snippet_keywords):
1822
+ overlap = line_tokens & sk
1823
+ n_overlap = len(overlap)
1824
+ ratio = n_overlap / len(line_tokens) if line_tokens else 0
1825
+ if n_overlap >= min_keyword_overlap and ratio >= min_overlap_ratio:
1826
+ citations.append(idx + 1)
1827
+
1828
+ if citations:
1829
+ # Avoid duplicates if LLM already cited [WN] in the line
1830
+ existing = set()
1831
+ for m in re.finditer(r"\[W(\d+)\]", line):
1832
+ existing.add(int(m.group(1)))
1833
+ new_cites = [c for c in citations if c not in existing]
1834
+ if new_cites:
1835
+ cite_str = " " + " ".join(f"[W{n}]" for n in new_cites)
1836
+ annotated.append(f"{line}{cite_str}")
1837
+ else:
1838
+ annotated.append(line)
1839
+ else:
1840
+ annotated.append(line)
1841
+
1842
+ return "\n".join(annotated)