MetacouplingLLM 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metacouplingllm/__init__.py +208 -0
- metacouplingllm/core.py +4542 -0
- metacouplingllm/data/Papers.zip +0 -0
- metacouplingllm/data/PeriTelecoupling_clean.csv +49953 -0
- metacouplingllm/data/PeriTelecoupling_subset.csv +11 -0
- metacouplingllm/data/chunk_embeddings.manifest.json +11 -0
- metacouplingllm/data/chunk_embeddings.npy +0 -0
- metacouplingllm/data/pericoupled_adm1_edge_list.csv +8291 -0
- metacouplingllm/data/telecoupling_literature.bib +3498 -0
- metacouplingllm/knowledge/__init__.py +1 -0
- metacouplingllm/knowledge/adm1_pericoupling.py +591 -0
- metacouplingllm/knowledge/citations.py +109 -0
- metacouplingllm/knowledge/countries.py +557 -0
- metacouplingllm/knowledge/examples.py +713 -0
- metacouplingllm/knowledge/framework.py +717 -0
- metacouplingllm/knowledge/literature.py +518 -0
- metacouplingllm/knowledge/pericoupling.py +241 -0
- metacouplingllm/knowledge/rag.py +1791 -0
- metacouplingllm/knowledge/references.py +112 -0
- metacouplingllm/knowledge/websearch.py +1842 -0
- metacouplingllm/llm/__init__.py +1 -0
- metacouplingllm/llm/client.py +636 -0
- metacouplingllm/llm/parser.py +868 -0
- metacouplingllm/output/__init__.py +1 -0
- metacouplingllm/output/formatter.py +276 -0
- metacouplingllm/prompts/__init__.py +1 -0
- metacouplingllm/prompts/builder.py +477 -0
- metacouplingllm/prompts/templates.py +556 -0
- metacouplingllm/visualization/__init__.py +114 -0
- metacouplingllm/visualization/adm1_map.py +884 -0
- metacouplingllm/visualization/worldmap.py +1372 -0
- metacouplingllm-0.1.0.dist-info/METADATA +440 -0
- metacouplingllm-0.1.0.dist-info/RECORD +35 -0
- metacouplingllm-0.1.0.dist-info/WHEEL +4 -0
- metacouplingllm-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,1842 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Web search integration for grounding metacoupling analyses.
|
|
3
|
+
|
|
4
|
+
Provides a lightweight wrapper around DuckDuckGo search (free, no API
|
|
5
|
+
key) to inject real-world context into the LLM prompt before analysis.
|
|
6
|
+
This is especially useful for niche topics where the LLM's pre-training
|
|
7
|
+
knowledge may be limited (e.g., "Michigan's pork exports").
|
|
8
|
+
|
|
9
|
+
Three backends are tried in order:
|
|
10
|
+
|
|
11
|
+
1. ``ddgs`` (pip install ddgs) — fast, full-featured
|
|
12
|
+
2. ``duckduckgo_search`` — older package, same author
|
|
13
|
+
3. **stdlib fallback** — ``urllib`` + ``html.parser``, zero external
|
|
14
|
+
dependencies. Works on Google Colab and other restricted
|
|
15
|
+
environments where the Rust-compiled ``primp`` dependency cannot be
|
|
16
|
+
installed.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import html.parser
|
|
22
|
+
import json
|
|
23
|
+
import re
|
|
24
|
+
import socket
|
|
25
|
+
import urllib.error
|
|
26
|
+
import urllib.parse
|
|
27
|
+
import urllib.request
|
|
28
|
+
from dataclasses import dataclass
|
|
29
|
+
from typing import Any, Protocol
|
|
30
|
+
|
|
31
|
+
_QUERY_FILLER = frozenset(
|
|
32
|
+
(
|
|
33
|
+
"a an analyze analysed analyzed analyzes analyzing analysis and apply "
|
|
34
|
+
"applying assess assessed assessing assesses case consider considering "
|
|
35
|
+
"examines examining explain explains focus focused focusing for from "
|
|
36
|
+
"how impact impacts in investigate investigating looking my of on or "
|
|
37
|
+
"our paper project question questions research researcher researchers "
|
|
38
|
+
"review reviews study studying the to topic topics what will with"
|
|
39
|
+
).split()
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
_STRUCTURAL_TERMS = frozenset(
|
|
43
|
+
(
|
|
44
|
+
"chain chains commodity commodities coupling couplings effect effects "
|
|
45
|
+
"environment environmental export exported exporter exporters exports "
|
|
46
|
+
"flow flows food framework frameworks import imported importer "
|
|
47
|
+
"importers imports industry industries market markets metacoupling "
|
|
48
|
+
"partner partners production productions sector sectors supply system "
|
|
49
|
+
"systems telecoupling trade traded trader traders trading transfer "
|
|
50
|
+
"transfers"
|
|
51
|
+
).split()
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
_TRADE_HINTS = frozenset(
|
|
55
|
+
(
|
|
56
|
+
"destination destinations export exported exporter exporters exports "
|
|
57
|
+
"import imported importer importers imports market markets partner "
|
|
58
|
+
"partners trade traded trader traders trading"
|
|
59
|
+
).split()
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class WebSearchBackend(Protocol):
|
|
64
|
+
"""Protocol for pluggable web-search backends."""
|
|
65
|
+
|
|
66
|
+
def search(
|
|
67
|
+
self,
|
|
68
|
+
query: str,
|
|
69
|
+
max_results: int = 5,
|
|
70
|
+
) -> list[dict[str, str]]:
|
|
71
|
+
"""Return search results as ``title``/``snippet``/``url`` dicts."""
|
|
72
|
+
...
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@dataclass
|
|
76
|
+
class OpenAIWebSearchBackend:
|
|
77
|
+
"""Web-search backend powered by OpenAI Responses API ``web_search``.
|
|
78
|
+
|
|
79
|
+
Results are normalized to the same ``title``/``snippet``/``url`` shape
|
|
80
|
+
used by the existing DuckDuckGo backends so downstream consumers do not
|
|
81
|
+
need to change.
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
client: Any
|
|
85
|
+
model: str = "gpt-5"
|
|
86
|
+
reasoning: str = "default"
|
|
87
|
+
external_web_access: bool = True
|
|
88
|
+
allowed_domains: list[str] | None = None
|
|
89
|
+
user_location: dict[str, object] | None = None
|
|
90
|
+
|
|
91
|
+
def search(
|
|
92
|
+
self,
|
|
93
|
+
query: str,
|
|
94
|
+
max_results: int = 5,
|
|
95
|
+
) -> list[dict[str, str]]:
|
|
96
|
+
if not query.strip() or max_results <= 0:
|
|
97
|
+
return []
|
|
98
|
+
|
|
99
|
+
tool: dict[str, object] = {
|
|
100
|
+
"type": "web_search",
|
|
101
|
+
"external_web_access": self.external_web_access,
|
|
102
|
+
}
|
|
103
|
+
if self.allowed_domains:
|
|
104
|
+
tool["filters"] = {"allowed_domains": self.allowed_domains}
|
|
105
|
+
if self.user_location:
|
|
106
|
+
tool["user_location"] = self.user_location
|
|
107
|
+
|
|
108
|
+
prompt = (
|
|
109
|
+
f"Search the web for: {query.strip()}\n"
|
|
110
|
+
f"Return up to {max_results} highly relevant results as JSON only.\n"
|
|
111
|
+
"Schema: "
|
|
112
|
+
'{"results":[{"title":"...","url":"...","snippet":"..."}]}\n'
|
|
113
|
+
"Rules:\n"
|
|
114
|
+
"- Use only information grounded in the web search results.\n"
|
|
115
|
+
"- Keep snippets concise and factual.\n"
|
|
116
|
+
"- Do not invent URLs.\n"
|
|
117
|
+
"- Return JSON only.\n"
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
kwargs: dict[str, object] = {
|
|
121
|
+
"model": self.model,
|
|
122
|
+
"tools": [tool],
|
|
123
|
+
"tool_choice": "auto",
|
|
124
|
+
"include": ["web_search_call.action.sources"],
|
|
125
|
+
"input": prompt,
|
|
126
|
+
}
|
|
127
|
+
if self.reasoning and self.reasoning != "default":
|
|
128
|
+
kwargs["reasoning"] = {"effort": self.reasoning}
|
|
129
|
+
|
|
130
|
+
response = self.client.responses.create(**kwargs)
|
|
131
|
+
output_text = getattr(response, "output_text", "") or ""
|
|
132
|
+
raw_obj = _extract_json_object(output_text)
|
|
133
|
+
parsed_results = []
|
|
134
|
+
if isinstance(raw_obj, dict):
|
|
135
|
+
parsed_results = _normalise_backend_results(
|
|
136
|
+
raw_obj.get("results", []),
|
|
137
|
+
max_results=max_results,
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
# Fallback: if the model didn't emit JSON results cleanly, recover
|
|
141
|
+
# from the included source list so the search still yields usable URLs.
|
|
142
|
+
source_results = _extract_openai_source_results(response, max_results)
|
|
143
|
+
if parsed_results:
|
|
144
|
+
merged = []
|
|
145
|
+
seen: set[tuple[str, str]] = set()
|
|
146
|
+
_merge_unique_results(
|
|
147
|
+
merged,
|
|
148
|
+
parsed_results,
|
|
149
|
+
seen=seen,
|
|
150
|
+
max_results=max_results,
|
|
151
|
+
)
|
|
152
|
+
_merge_unique_results(
|
|
153
|
+
merged,
|
|
154
|
+
source_results,
|
|
155
|
+
seen=seen,
|
|
156
|
+
max_results=max_results,
|
|
157
|
+
)
|
|
158
|
+
return merged[:max_results]
|
|
159
|
+
return source_results[:max_results]
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
# Model → web_search tool version compatibility. Used by
|
|
163
|
+
# AnthropicWebSearchBackend when ``tool_version`` is left as None.
|
|
164
|
+
# Per Anthropic docs, web_search_20260209 (with dynamic filtering) is
|
|
165
|
+
# supported on Opus 4.7, Opus 4.6, and Sonnet 4.6. Older Claude 4 models
|
|
166
|
+
# must use web_search_20250305 (no dynamic filtering).
|
|
167
|
+
_WEB_SEARCH_MODEL_VERSIONS: dict[str, str] = {
|
|
168
|
+
# Supports web_search_20260209 (dynamic filtering)
|
|
169
|
+
"claude-opus-4-7": "web_search_20260209",
|
|
170
|
+
"claude-opus-4-6": "web_search_20260209",
|
|
171
|
+
"claude-sonnet-4-6": "web_search_20260209",
|
|
172
|
+
# Older Claude 4 models — use web_search_20250305 (no dynamic filtering)
|
|
173
|
+
"claude-opus-4-5": "web_search_20250305",
|
|
174
|
+
"claude-opus-4-1": "web_search_20250305",
|
|
175
|
+
"claude-opus-4-0": "web_search_20250305",
|
|
176
|
+
"claude-sonnet-4-5": "web_search_20250305",
|
|
177
|
+
"claude-sonnet-4-0": "web_search_20250305",
|
|
178
|
+
"claude-haiku-4-5": "web_search_20250305", # conservative default
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _infer_web_search_tool_version(model: str) -> str:
|
|
183
|
+
"""Pick the ``web_search`` tool version best supported by *model*.
|
|
184
|
+
|
|
185
|
+
Returns ``web_search_20260209`` (dynamic filtering) for Opus 4.7, Opus
|
|
186
|
+
4.6, and Sonnet 4.6 — the models Anthropic explicitly lists as
|
|
187
|
+
supporting the newer version. Returns ``web_search_20250305`` (no
|
|
188
|
+
dynamic filtering) for older Claude 4 models. Unknown models default
|
|
189
|
+
to the newer version; the caller can pass ``tool_version`` explicitly
|
|
190
|
+
to override.
|
|
191
|
+
"""
|
|
192
|
+
known = _WEB_SEARCH_MODEL_VERSIONS.get(model)
|
|
193
|
+
if known:
|
|
194
|
+
return known
|
|
195
|
+
return "web_search_20260209"
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
@dataclass
|
|
199
|
+
class AnthropicWebSearchBackend:
|
|
200
|
+
"""Web-search backend powered by the Anthropic Messages API ``web_search``.
|
|
201
|
+
|
|
202
|
+
By default the tool version is **auto-selected from ``model``** via
|
|
203
|
+
:func:`_infer_web_search_tool_version`:
|
|
204
|
+
|
|
205
|
+
========================== ==========================
|
|
206
|
+
Model tool_version
|
|
207
|
+
========================== ==========================
|
|
208
|
+
claude-opus-4-7 (default) web_search_20260209
|
|
209
|
+
claude-opus-4-6 web_search_20260209
|
|
210
|
+
claude-sonnet-4-6 web_search_20260209
|
|
211
|
+
claude-opus-4-5 / 4-1 / 4-0 web_search_20250305
|
|
212
|
+
claude-sonnet-4-5 / 4-0 web_search_20250305
|
|
213
|
+
claude-haiku-4-5 web_search_20250305
|
|
214
|
+
unknown web_search_20260209
|
|
215
|
+
========================== ==========================
|
|
216
|
+
|
|
217
|
+
Pass an explicit ``tool_version`` to override (useful when a new model
|
|
218
|
+
ships before the lookup table is updated).
|
|
219
|
+
|
|
220
|
+
Snippets are pulled from citations on Claude's text blocks (each citation
|
|
221
|
+
carries ``url``, ``title``, and up to 150 chars of ``cited_text``). If
|
|
222
|
+
Claude returns no citations, the backend falls back to the raw
|
|
223
|
+
``web_search_tool_result`` blocks for ``url`` + ``title`` + ``page_age``
|
|
224
|
+
(the underlying ``encrypted_content`` is opaque to callers by design).
|
|
225
|
+
|
|
226
|
+
Prompt caching is intentionally not used: a web-search call's cacheable
|
|
227
|
+
prefix (tool definition plus short user prompt) is well under Opus
|
|
228
|
+
4.7's 4096-token minimum, so caching would silently no-op.
|
|
229
|
+
"""
|
|
230
|
+
|
|
231
|
+
client: Any
|
|
232
|
+
model: str = "claude-opus-4-7"
|
|
233
|
+
tool_version: str | None = None # None = auto-select from model
|
|
234
|
+
max_uses: int = 5
|
|
235
|
+
allowed_domains: list[str] | None = None
|
|
236
|
+
blocked_domains: list[str] | None = None
|
|
237
|
+
user_location: dict[str, object] | None = None
|
|
238
|
+
max_tokens: int = 8192
|
|
239
|
+
|
|
240
|
+
def search(
|
|
241
|
+
self,
|
|
242
|
+
query: str,
|
|
243
|
+
max_results: int = 5,
|
|
244
|
+
) -> list[dict[str, str]]:
|
|
245
|
+
if not query.strip() or max_results <= 0:
|
|
246
|
+
return []
|
|
247
|
+
|
|
248
|
+
tool_version = (
|
|
249
|
+
self.tool_version
|
|
250
|
+
or _infer_web_search_tool_version(self.model)
|
|
251
|
+
)
|
|
252
|
+
tool: dict[str, object] = {
|
|
253
|
+
"type": tool_version,
|
|
254
|
+
"name": "web_search",
|
|
255
|
+
"max_uses": self.max_uses,
|
|
256
|
+
}
|
|
257
|
+
if self.allowed_domains:
|
|
258
|
+
tool["allowed_domains"] = self.allowed_domains
|
|
259
|
+
if self.blocked_domains:
|
|
260
|
+
tool["blocked_domains"] = self.blocked_domains
|
|
261
|
+
if self.user_location:
|
|
262
|
+
tool["user_location"] = self.user_location
|
|
263
|
+
|
|
264
|
+
prompt = (
|
|
265
|
+
f"Search the web for: {query.strip()}\n"
|
|
266
|
+
f"Return up to {max_results} highly relevant sources. "
|
|
267
|
+
"Cite every factual statement you make inline so that every "
|
|
268
|
+
"source you used appears in a citation."
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
response = self.client.messages.create(
|
|
272
|
+
model=self.model,
|
|
273
|
+
max_tokens=self.max_tokens,
|
|
274
|
+
tools=[tool],
|
|
275
|
+
messages=[{"role": "user", "content": prompt}],
|
|
276
|
+
)
|
|
277
|
+
|
|
278
|
+
return _extract_anthropic_web_results(response, max_results)
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
@dataclass
|
|
282
|
+
class GeminiWebSearchBackend:
|
|
283
|
+
"""Web-search backend powered by Google Gemini's grounding tool.
|
|
284
|
+
|
|
285
|
+
Built against the new ``google.genai`` SDK. Calls
|
|
286
|
+
``client.models.generate_content`` with the ``google_search`` tool
|
|
287
|
+
enabled (Gemini 2.x grounding). The model performs the search and
|
|
288
|
+
synthesises results; we parse the response for a JSON list of
|
|
289
|
+
``title``/``snippet``/``url`` items, falling back to
|
|
290
|
+
``grounding_metadata.grounding_chunks`` for URL+title recovery if
|
|
291
|
+
JSON parsing fails.
|
|
292
|
+
|
|
293
|
+
Parameters
|
|
294
|
+
----------
|
|
295
|
+
client:
|
|
296
|
+
A ``google.genai.Client`` instance.
|
|
297
|
+
model:
|
|
298
|
+
Gemini model identifier supporting grounding (e.g.
|
|
299
|
+
``"gemini-2.5-flash"`` or ``"gemini-2.5-pro"``).
|
|
300
|
+
max_tokens:
|
|
301
|
+
Output budget for the search-and-summarise call.
|
|
302
|
+
"""
|
|
303
|
+
|
|
304
|
+
client: Any
|
|
305
|
+
model: str = "gemini-2.5-flash"
|
|
306
|
+
max_tokens: int = 8192
|
|
307
|
+
|
|
308
|
+
def search(
|
|
309
|
+
self,
|
|
310
|
+
query: str,
|
|
311
|
+
max_results: int = 5,
|
|
312
|
+
) -> list[dict[str, str]]:
|
|
313
|
+
if not query.strip() or max_results <= 0:
|
|
314
|
+
return []
|
|
315
|
+
|
|
316
|
+
prompt = (
|
|
317
|
+
f"Search the web for: {query.strip()}\n"
|
|
318
|
+
f"Return up to {max_results} highly relevant results as JSON only.\n"
|
|
319
|
+
"Schema: "
|
|
320
|
+
'{"results":[{"title":"...","url":"...","snippet":"..."}]}\n'
|
|
321
|
+
"Rules:\n"
|
|
322
|
+
"- Use only information grounded in the web search results.\n"
|
|
323
|
+
"- Keep snippets concise and factual.\n"
|
|
324
|
+
"- Do not invent URLs.\n"
|
|
325
|
+
"- Return JSON only.\n"
|
|
326
|
+
)
|
|
327
|
+
|
|
328
|
+
try:
|
|
329
|
+
# google.genai 1.x: tools accept a list of dicts; the
|
|
330
|
+
# google_search tool is enabled with an empty config.
|
|
331
|
+
response = self.client.models.generate_content(
|
|
332
|
+
model=self.model,
|
|
333
|
+
contents=prompt,
|
|
334
|
+
config={
|
|
335
|
+
"max_output_tokens": self.max_tokens,
|
|
336
|
+
"tools": [{"google_search": {}}],
|
|
337
|
+
},
|
|
338
|
+
)
|
|
339
|
+
except Exception:
|
|
340
|
+
return []
|
|
341
|
+
|
|
342
|
+
# Pull the model's text (may raise if blocked).
|
|
343
|
+
try:
|
|
344
|
+
output_text = response.text or ""
|
|
345
|
+
except Exception:
|
|
346
|
+
output_text = ""
|
|
347
|
+
|
|
348
|
+
raw_obj = _extract_json_object(output_text)
|
|
349
|
+
parsed_results: list[dict[str, str]] = []
|
|
350
|
+
if isinstance(raw_obj, dict):
|
|
351
|
+
parsed_results = _normalise_backend_results(
|
|
352
|
+
raw_obj.get("results", []),
|
|
353
|
+
max_results=max_results,
|
|
354
|
+
)
|
|
355
|
+
|
|
356
|
+
# Fallback: walk grounding_metadata.grounding_chunks for url + title.
|
|
357
|
+
grounding_results = _extract_gemini_grounding_results(
|
|
358
|
+
response, max_results
|
|
359
|
+
)
|
|
360
|
+
if parsed_results:
|
|
361
|
+
merged: list[dict[str, str]] = []
|
|
362
|
+
seen: set[tuple[str, str]] = set()
|
|
363
|
+
_merge_unique_results(
|
|
364
|
+
merged,
|
|
365
|
+
parsed_results,
|
|
366
|
+
seen=seen,
|
|
367
|
+
max_results=max_results,
|
|
368
|
+
)
|
|
369
|
+
_merge_unique_results(
|
|
370
|
+
merged,
|
|
371
|
+
grounding_results,
|
|
372
|
+
seen=seen,
|
|
373
|
+
max_results=max_results,
|
|
374
|
+
)
|
|
375
|
+
return merged[:max_results]
|
|
376
|
+
return grounding_results[:max_results]
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
@dataclass
|
|
380
|
+
class GrokWebSearchBackend:
|
|
381
|
+
"""Web-search backend powered by xAI Grok's Live Search tool.
|
|
382
|
+
|
|
383
|
+
Live Search is invoked by passing ``extra_body={"search_parameters":
|
|
384
|
+
{...}}`` to a chat-completions call. The response includes
|
|
385
|
+
``citations`` (a list of URLs) plus the assistant message with
|
|
386
|
+
inline citations. We ask the model to return JSON; if that fails,
|
|
387
|
+
we fall back to ``response.citations`` for URL recovery.
|
|
388
|
+
|
|
389
|
+
Parameters
|
|
390
|
+
----------
|
|
391
|
+
client:
|
|
392
|
+
An ``openai.OpenAI`` instance configured with
|
|
393
|
+
``base_url="https://api.x.ai/v1"``.
|
|
394
|
+
model:
|
|
395
|
+
Grok model identifier (e.g., ``"grok-3"``, ``"grok-2"``).
|
|
396
|
+
search_mode:
|
|
397
|
+
xAI's ``mode`` for Live Search: ``"auto"`` (default — model
|
|
398
|
+
decides), ``"on"`` (always search), or ``"off"`` (no search).
|
|
399
|
+
max_search_results:
|
|
400
|
+
Upper bound on internal search calls per request (xAI default
|
|
401
|
+
is 15).
|
|
402
|
+
sources:
|
|
403
|
+
Override the default Live Search sources. Default is
|
|
404
|
+
``[{"type": "web"}, {"type": "x"}]`` — web + Twitter/X.
|
|
405
|
+
max_tokens:
|
|
406
|
+
Output budget for the search-and-summarise call.
|
|
407
|
+
"""
|
|
408
|
+
|
|
409
|
+
client: Any
|
|
410
|
+
model: str = "grok-3"
|
|
411
|
+
search_mode: str = "auto"
|
|
412
|
+
max_search_results: int = 15
|
|
413
|
+
sources: list[dict[str, object]] | None = None
|
|
414
|
+
max_tokens: int = 8192
|
|
415
|
+
|
|
416
|
+
def search(
|
|
417
|
+
self,
|
|
418
|
+
query: str,
|
|
419
|
+
max_results: int = 5,
|
|
420
|
+
) -> list[dict[str, str]]:
|
|
421
|
+
if not query.strip() or max_results <= 0:
|
|
422
|
+
return []
|
|
423
|
+
|
|
424
|
+
prompt = (
|
|
425
|
+
f"Search the web for: {query.strip()}\n"
|
|
426
|
+
f"Return up to {max_results} highly relevant results as JSON only.\n"
|
|
427
|
+
"Schema: "
|
|
428
|
+
'{"results":[{"title":"...","url":"...","snippet":"..."}]}\n'
|
|
429
|
+
"Rules:\n"
|
|
430
|
+
"- Use only information grounded in your live search results.\n"
|
|
431
|
+
"- Keep snippets concise and factual.\n"
|
|
432
|
+
"- Do not invent URLs.\n"
|
|
433
|
+
"- Return JSON only.\n"
|
|
434
|
+
)
|
|
435
|
+
|
|
436
|
+
sources = self.sources or [{"type": "web"}, {"type": "x"}]
|
|
437
|
+
try:
|
|
438
|
+
response = self.client.chat.completions.create(
|
|
439
|
+
model=self.model,
|
|
440
|
+
messages=[{"role": "user", "content": prompt}],
|
|
441
|
+
max_tokens=self.max_tokens,
|
|
442
|
+
extra_body={
|
|
443
|
+
"search_parameters": {
|
|
444
|
+
"mode": self.search_mode,
|
|
445
|
+
"max_search_results": self.max_search_results,
|
|
446
|
+
"sources": sources,
|
|
447
|
+
},
|
|
448
|
+
},
|
|
449
|
+
)
|
|
450
|
+
except Exception:
|
|
451
|
+
return []
|
|
452
|
+
|
|
453
|
+
message_text = ""
|
|
454
|
+
if getattr(response, "choices", None):
|
|
455
|
+
choice = response.choices[0]
|
|
456
|
+
message = getattr(choice, "message", None)
|
|
457
|
+
if message is not None:
|
|
458
|
+
message_text = getattr(message, "content", "") or ""
|
|
459
|
+
|
|
460
|
+
raw_obj = _extract_json_object(message_text)
|
|
461
|
+
parsed_results: list[dict[str, str]] = []
|
|
462
|
+
if isinstance(raw_obj, dict):
|
|
463
|
+
parsed_results = _normalise_backend_results(
|
|
464
|
+
raw_obj.get("results", []),
|
|
465
|
+
max_results=max_results,
|
|
466
|
+
)
|
|
467
|
+
|
|
468
|
+
# Fallback: harvest citations (URLs) when JSON parsing fails.
|
|
469
|
+
citation_results = _extract_grok_citation_results(
|
|
470
|
+
response, max_results
|
|
471
|
+
)
|
|
472
|
+
if parsed_results:
|
|
473
|
+
merged: list[dict[str, str]] = []
|
|
474
|
+
seen: set[tuple[str, str]] = set()
|
|
475
|
+
_merge_unique_results(
|
|
476
|
+
merged,
|
|
477
|
+
parsed_results,
|
|
478
|
+
seen=seen,
|
|
479
|
+
max_results=max_results,
|
|
480
|
+
)
|
|
481
|
+
_merge_unique_results(
|
|
482
|
+
merged,
|
|
483
|
+
citation_results,
|
|
484
|
+
seen=seen,
|
|
485
|
+
max_results=max_results,
|
|
486
|
+
)
|
|
487
|
+
return merged[:max_results]
|
|
488
|
+
return citation_results[:max_results]
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def _extract_gemini_grounding_results(
|
|
492
|
+
response: object,
|
|
493
|
+
max_results: int,
|
|
494
|
+
) -> list[dict[str, str]]:
|
|
495
|
+
"""Walk a Gemini response for ``grounding_chunks[].web`` URL+title pairs."""
|
|
496
|
+
candidates = getattr(response, "candidates", None) or []
|
|
497
|
+
out: list[dict[str, str]] = []
|
|
498
|
+
seen_urls: set[str] = set()
|
|
499
|
+
for candidate in candidates:
|
|
500
|
+
meta = getattr(candidate, "grounding_metadata", None)
|
|
501
|
+
if meta is None:
|
|
502
|
+
continue
|
|
503
|
+
chunks = getattr(meta, "grounding_chunks", None) or []
|
|
504
|
+
for chunk in chunks:
|
|
505
|
+
web = getattr(chunk, "web", None)
|
|
506
|
+
if web is None:
|
|
507
|
+
continue
|
|
508
|
+
url = (getattr(web, "uri", "") or "").strip()
|
|
509
|
+
title = (getattr(web, "title", "") or "").strip() or url
|
|
510
|
+
if not url or url in seen_urls:
|
|
511
|
+
continue
|
|
512
|
+
seen_urls.add(url)
|
|
513
|
+
out.append({"title": title, "snippet": "", "url": url})
|
|
514
|
+
if len(out) >= max_results:
|
|
515
|
+
return out
|
|
516
|
+
return out
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
def _extract_grok_citation_results(
|
|
520
|
+
response: object,
|
|
521
|
+
max_results: int,
|
|
522
|
+
) -> list[dict[str, str]]:
|
|
523
|
+
"""Harvest ``response.citations`` (URLs) as fallback web results."""
|
|
524
|
+
citations = getattr(response, "citations", None) or []
|
|
525
|
+
out: list[dict[str, str]] = []
|
|
526
|
+
seen_urls: set[str] = set()
|
|
527
|
+
for cite in citations:
|
|
528
|
+
# `citations` may be a list of strings (URLs) or dict-like objects
|
|
529
|
+
# depending on xAI SDK version. Handle both shapes.
|
|
530
|
+
if isinstance(cite, str):
|
|
531
|
+
url = cite.strip()
|
|
532
|
+
title = url
|
|
533
|
+
snippet = ""
|
|
534
|
+
elif isinstance(cite, dict):
|
|
535
|
+
url = str(cite.get("url") or "").strip()
|
|
536
|
+
title = str(cite.get("title") or url).strip()
|
|
537
|
+
snippet = str(cite.get("snippet") or cite.get("excerpt") or "").strip()
|
|
538
|
+
else:
|
|
539
|
+
url = str(getattr(cite, "url", "") or "").strip()
|
|
540
|
+
title = str(getattr(cite, "title", url) or url).strip()
|
|
541
|
+
snippet = str(getattr(cite, "snippet", "") or "").strip()
|
|
542
|
+
if not url or url in seen_urls:
|
|
543
|
+
continue
|
|
544
|
+
seen_urls.add(url)
|
|
545
|
+
out.append({"title": title, "snippet": snippet, "url": url})
|
|
546
|
+
if len(out) >= max_results:
|
|
547
|
+
break
|
|
548
|
+
return out
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
# -------------------------------------------------------------------
|
|
552
|
+
# Stdlib fallback — zero external dependencies
|
|
553
|
+
# -------------------------------------------------------------------
|
|
554
|
+
|
|
555
|
+
class _DuckDuckGoLiteParser(html.parser.HTMLParser):
|
|
556
|
+
"""Minimal parser for DuckDuckGo Lite HTML results page."""
|
|
557
|
+
|
|
558
|
+
def __init__(self) -> None:
|
|
559
|
+
super().__init__()
|
|
560
|
+
self.results: list[dict[str, str]] = []
|
|
561
|
+
self._in_link = False
|
|
562
|
+
self._in_snippet = False
|
|
563
|
+
self._current: dict[str, str] = {}
|
|
564
|
+
self._text_buf: list[str] = []
|
|
565
|
+
|
|
566
|
+
# DuckDuckGo Lite wraps each result link in
|
|
567
|
+
# <a rel="nofollow" class="result-link" href="…">
|
|
568
|
+
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|
569
|
+
attr = dict(attrs)
|
|
570
|
+
if tag == "a" and attr.get("class") == "result-link":
|
|
571
|
+
self._in_link = True
|
|
572
|
+
self._current = {"url": attr.get("href", ""), "title": "", "snippet": ""}
|
|
573
|
+
self._text_buf = []
|
|
574
|
+
elif tag == "td" and attr.get("class") == "result-snippet":
|
|
575
|
+
self._in_snippet = True
|
|
576
|
+
self._text_buf = []
|
|
577
|
+
|
|
578
|
+
def handle_endtag(self, tag: str) -> None:
|
|
579
|
+
if tag == "a" and self._in_link:
|
|
580
|
+
self._in_link = False
|
|
581
|
+
self._current["title"] = " ".join(self._text_buf).strip()
|
|
582
|
+
elif tag == "td" and self._in_snippet:
|
|
583
|
+
self._in_snippet = False
|
|
584
|
+
self._current["snippet"] = " ".join(self._text_buf).strip()
|
|
585
|
+
if self._current.get("url"):
|
|
586
|
+
self.results.append(self._current)
|
|
587
|
+
self._current = {}
|
|
588
|
+
|
|
589
|
+
def handle_data(self, data: str) -> None:
|
|
590
|
+
if self._in_link or self._in_snippet:
|
|
591
|
+
self._text_buf.append(data)
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
def _resolve_ddg_url(raw: str) -> str:
|
|
595
|
+
"""Extract the real destination URL from a DuckDuckGo redirect link.
|
|
596
|
+
|
|
597
|
+
DuckDuckGo Lite wraps result links as
|
|
598
|
+
``//duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com&rut=…``.
|
|
599
|
+
This function extracts the ``uddg`` parameter value, falling back
|
|
600
|
+
to the original URL if it doesn't match the redirect pattern.
|
|
601
|
+
"""
|
|
602
|
+
if "duckduckgo.com/l/" in raw and "uddg=" in raw:
|
|
603
|
+
parsed = urllib.parse.urlparse(raw if "://" in raw else "https:" + raw)
|
|
604
|
+
params = urllib.parse.parse_qs(parsed.query)
|
|
605
|
+
uddg = params.get("uddg", [None])[0]
|
|
606
|
+
if uddg:
|
|
607
|
+
return uddg
|
|
608
|
+
return raw
|
|
609
|
+
|
|
610
|
+
|
|
611
|
+
def _search_stdlib(query: str, max_results: int) -> list[dict[str, str]]:
|
|
612
|
+
"""Search DuckDuckGo using only the standard library.
|
|
613
|
+
|
|
614
|
+
Handles network errors, timeouts, and malformed HTML gracefully
|
|
615
|
+
by returning an empty list on failure.
|
|
616
|
+
"""
|
|
617
|
+
# Guard against excessively long queries that could exceed URL limits
|
|
618
|
+
if len(query) > 2000:
|
|
619
|
+
query = query[:2000]
|
|
620
|
+
|
|
621
|
+
url = "https://lite.duckduckgo.com/lite/?" + urllib.parse.urlencode({"q": query})
|
|
622
|
+
req = urllib.request.Request(
|
|
623
|
+
url,
|
|
624
|
+
headers={
|
|
625
|
+
"User-Agent": (
|
|
626
|
+
"Mozilla/5.0 (compatible; metacoupling-python/0.1; "
|
|
627
|
+
"+https://github.com/metacoupling/metacoupling)"
|
|
628
|
+
),
|
|
629
|
+
},
|
|
630
|
+
)
|
|
631
|
+
try:
|
|
632
|
+
with urllib.request.urlopen(req, timeout=15) as resp:
|
|
633
|
+
body = resp.read().decode("utf-8", errors="replace")
|
|
634
|
+
except (urllib.error.URLError, urllib.error.HTTPError, socket.timeout,
|
|
635
|
+
OSError) as exc:
|
|
636
|
+
print(f"[WebSearch] stdlib network error: {exc}")
|
|
637
|
+
return []
|
|
638
|
+
|
|
639
|
+
try:
|
|
640
|
+
parser = _DuckDuckGoLiteParser()
|
|
641
|
+
parser.feed(body)
|
|
642
|
+
except Exception as exc:
|
|
643
|
+
print(f"[WebSearch] HTML parsing error: {exc}")
|
|
644
|
+
return []
|
|
645
|
+
|
|
646
|
+
# Resolve redirect URLs to actual destinations
|
|
647
|
+
for r in parser.results:
|
|
648
|
+
r["url"] = _resolve_ddg_url(r["url"])
|
|
649
|
+
|
|
650
|
+
return parser.results[:max_results]
|
|
651
|
+
|
|
652
|
+
|
|
653
|
+
def _search_ddgs(query: str, max_results: int) -> list[dict[str, str]]:
|
|
654
|
+
"""Search using the modern ``ddgs`` backend."""
|
|
655
|
+
from ddgs import DDGS # type: ignore[import-untyped]
|
|
656
|
+
|
|
657
|
+
results: list[dict[str, str]] = []
|
|
658
|
+
for r in DDGS().text(query, max_results=max_results):
|
|
659
|
+
results.append({
|
|
660
|
+
"title": r.get("title", ""),
|
|
661
|
+
"snippet": r.get("body", ""),
|
|
662
|
+
"url": _resolve_ddg_url(r.get("href", "")),
|
|
663
|
+
})
|
|
664
|
+
return results
|
|
665
|
+
|
|
666
|
+
|
|
667
|
+
def _search_duckduckgo_search(
|
|
668
|
+
query: str,
|
|
669
|
+
max_results: int,
|
|
670
|
+
) -> list[dict[str, str]]:
|
|
671
|
+
"""Search using the legacy ``duckduckgo_search`` backend."""
|
|
672
|
+
import warnings
|
|
673
|
+
|
|
674
|
+
with warnings.catch_warnings():
|
|
675
|
+
warnings.filterwarnings(
|
|
676
|
+
"ignore",
|
|
677
|
+
category=RuntimeWarning,
|
|
678
|
+
module=r"duckduckgo_search(\..*)?",
|
|
679
|
+
)
|
|
680
|
+
from duckduckgo_search import DDGS as DDGS2 # type: ignore[import-untyped]
|
|
681
|
+
|
|
682
|
+
results: list[dict[str, str]] = []
|
|
683
|
+
for r in DDGS2().text(query, max_results=max_results):
|
|
684
|
+
results.append({
|
|
685
|
+
"title": r.get("title", ""),
|
|
686
|
+
"snippet": r.get("body", ""),
|
|
687
|
+
"url": _resolve_ddg_url(r.get("href", "")),
|
|
688
|
+
})
|
|
689
|
+
return results
|
|
690
|
+
|
|
691
|
+
|
|
692
|
+
def _result_key(result: dict[str, str]) -> tuple[str, str]:
|
|
693
|
+
"""Build a stable dedupe key for a search result."""
|
|
694
|
+
url = result.get("url", "").strip().lower().rstrip("/")
|
|
695
|
+
if url:
|
|
696
|
+
return ("url", url)
|
|
697
|
+
title = " ".join(result.get("title", "").lower().split())
|
|
698
|
+
snippet = " ".join(result.get("snippet", "").lower().split())
|
|
699
|
+
return ("text", f"{title}|{snippet}")
|
|
700
|
+
|
|
701
|
+
|
|
702
|
+
def _normalise_backend_results(
|
|
703
|
+
results: object,
|
|
704
|
+
*,
|
|
705
|
+
max_results: int,
|
|
706
|
+
) -> list[dict[str, str]]:
|
|
707
|
+
"""Normalise backend output to ``title``/``snippet``/``url`` dicts."""
|
|
708
|
+
if not isinstance(results, list):
|
|
709
|
+
return []
|
|
710
|
+
|
|
711
|
+
normalised: list[dict[str, str]] = []
|
|
712
|
+
for item in results:
|
|
713
|
+
if not isinstance(item, dict):
|
|
714
|
+
continue
|
|
715
|
+
title = str(item.get("title", "")).strip()
|
|
716
|
+
snippet = str(item.get("snippet", "")).strip()
|
|
717
|
+
url = str(item.get("url", "")).strip()
|
|
718
|
+
if not url:
|
|
719
|
+
continue
|
|
720
|
+
normalised.append({
|
|
721
|
+
"title": title,
|
|
722
|
+
"snippet": snippet,
|
|
723
|
+
"url": url,
|
|
724
|
+
})
|
|
725
|
+
if len(normalised) >= max_results:
|
|
726
|
+
break
|
|
727
|
+
return normalised
|
|
728
|
+
|
|
729
|
+
|
|
730
|
+
def _extract_openai_source_results(
|
|
731
|
+
response: object,
|
|
732
|
+
max_results: int,
|
|
733
|
+
) -> list[dict[str, str]]:
|
|
734
|
+
"""Best-effort extraction of ``url``/``title`` source rows from a response."""
|
|
735
|
+
raw: object
|
|
736
|
+
if hasattr(response, "model_dump"):
|
|
737
|
+
try:
|
|
738
|
+
raw = response.model_dump()
|
|
739
|
+
except Exception:
|
|
740
|
+
raw = None
|
|
741
|
+
elif isinstance(response, dict):
|
|
742
|
+
raw = response
|
|
743
|
+
else:
|
|
744
|
+
raw = None
|
|
745
|
+
|
|
746
|
+
if not isinstance(raw, (dict, list)):
|
|
747
|
+
return []
|
|
748
|
+
|
|
749
|
+
seen_urls: set[str] = set()
|
|
750
|
+
found: list[dict[str, str]] = []
|
|
751
|
+
|
|
752
|
+
def _walk(node: object) -> None:
|
|
753
|
+
if len(found) >= max_results:
|
|
754
|
+
return
|
|
755
|
+
if isinstance(node, dict):
|
|
756
|
+
url = node.get("url")
|
|
757
|
+
title = node.get("title")
|
|
758
|
+
if isinstance(url, str) and url.strip():
|
|
759
|
+
clean_url = url.strip()
|
|
760
|
+
if clean_url not in seen_urls:
|
|
761
|
+
seen_urls.add(clean_url)
|
|
762
|
+
found.append({
|
|
763
|
+
"title": str(title).strip() if title else clean_url,
|
|
764
|
+
"snippet": str(
|
|
765
|
+
node.get("snippet")
|
|
766
|
+
or node.get("excerpt")
|
|
767
|
+
or node.get("description")
|
|
768
|
+
or ""
|
|
769
|
+
).strip(),
|
|
770
|
+
"url": clean_url,
|
|
771
|
+
})
|
|
772
|
+
if len(found) >= max_results:
|
|
773
|
+
return
|
|
774
|
+
for value in node.values():
|
|
775
|
+
_walk(value)
|
|
776
|
+
elif isinstance(node, list):
|
|
777
|
+
for item in node:
|
|
778
|
+
_walk(item)
|
|
779
|
+
|
|
780
|
+
_walk(raw)
|
|
781
|
+
return found[:max_results]
|
|
782
|
+
|
|
783
|
+
|
|
784
|
+
def _extract_anthropic_web_results(
|
|
785
|
+
response: object,
|
|
786
|
+
max_results: int,
|
|
787
|
+
) -> list[dict[str, str]]:
|
|
788
|
+
"""Extract ``title``/``snippet``/``url`` rows from an Anthropic response.
|
|
789
|
+
|
|
790
|
+
Primary path: walk the ``citations`` lists on each ``text`` content block
|
|
791
|
+
— every citation carries the source URL, title, and up to 150 chars of
|
|
792
|
+
``cited_text`` (the snippet). Dedupe by URL, take the first citation's
|
|
793
|
+
``cited_text`` as the snippet when a URL is cited multiple times.
|
|
794
|
+
|
|
795
|
+
Fallback path: when Claude returned search results but no citations,
|
|
796
|
+
walk the ``web_search_tool_result`` blocks for ``url`` + ``title``.
|
|
797
|
+
``encrypted_content`` is opaque by design, so snippets in this path
|
|
798
|
+
degrade to just the ``page_age`` timestamp.
|
|
799
|
+
"""
|
|
800
|
+
raw: object
|
|
801
|
+
if hasattr(response, "model_dump"):
|
|
802
|
+
try:
|
|
803
|
+
raw = response.model_dump()
|
|
804
|
+
except Exception:
|
|
805
|
+
raw = None
|
|
806
|
+
elif isinstance(response, dict):
|
|
807
|
+
raw = response
|
|
808
|
+
else:
|
|
809
|
+
raw = None
|
|
810
|
+
|
|
811
|
+
if not isinstance(raw, dict):
|
|
812
|
+
return []
|
|
813
|
+
|
|
814
|
+
content = raw.get("content")
|
|
815
|
+
if not isinstance(content, list):
|
|
816
|
+
return []
|
|
817
|
+
|
|
818
|
+
seen_urls: set[str] = set()
|
|
819
|
+
found: list[dict[str, str]] = []
|
|
820
|
+
|
|
821
|
+
# Primary: citations on text blocks
|
|
822
|
+
for block in content:
|
|
823
|
+
if not isinstance(block, dict) or block.get("type") != "text":
|
|
824
|
+
continue
|
|
825
|
+
citations = block.get("citations")
|
|
826
|
+
if not isinstance(citations, list):
|
|
827
|
+
continue
|
|
828
|
+
for cit in citations:
|
|
829
|
+
if not isinstance(cit, dict):
|
|
830
|
+
continue
|
|
831
|
+
if cit.get("type") != "web_search_result_location":
|
|
832
|
+
continue
|
|
833
|
+
url = str(cit.get("url", "") or "").strip()
|
|
834
|
+
if not url or url in seen_urls:
|
|
835
|
+
continue
|
|
836
|
+
seen_urls.add(url)
|
|
837
|
+
title = str(cit.get("title", "") or "").strip() or url
|
|
838
|
+
snippet = str(cit.get("cited_text", "") or "").strip()
|
|
839
|
+
found.append({"title": title, "snippet": snippet, "url": url})
|
|
840
|
+
if len(found) >= max_results:
|
|
841
|
+
return found
|
|
842
|
+
|
|
843
|
+
# Fallback: web_search_tool_result blocks when no citations were emitted
|
|
844
|
+
if not found:
|
|
845
|
+
for block in content:
|
|
846
|
+
if not isinstance(block, dict):
|
|
847
|
+
continue
|
|
848
|
+
if block.get("type") != "web_search_tool_result":
|
|
849
|
+
continue
|
|
850
|
+
tr_content = block.get("content")
|
|
851
|
+
if not isinstance(tr_content, list):
|
|
852
|
+
continue
|
|
853
|
+
for res in tr_content:
|
|
854
|
+
if not isinstance(res, dict):
|
|
855
|
+
continue
|
|
856
|
+
if res.get("type") != "web_search_result":
|
|
857
|
+
continue
|
|
858
|
+
url = str(res.get("url", "") or "").strip()
|
|
859
|
+
if not url or url in seen_urls:
|
|
860
|
+
continue
|
|
861
|
+
seen_urls.add(url)
|
|
862
|
+
title = str(res.get("title", "") or "").strip() or url
|
|
863
|
+
page_age = str(res.get("page_age", "") or "").strip()
|
|
864
|
+
snippet = f"(page age: {page_age})" if page_age else ""
|
|
865
|
+
found.append({"title": title, "snippet": snippet, "url": url})
|
|
866
|
+
if len(found) >= max_results:
|
|
867
|
+
return found
|
|
868
|
+
|
|
869
|
+
return found[:max_results]
|
|
870
|
+
|
|
871
|
+
|
|
872
|
+
def _merge_unique_results(
|
|
873
|
+
merged: list[dict[str, str]],
|
|
874
|
+
new_results: list[dict[str, str]],
|
|
875
|
+
*,
|
|
876
|
+
seen: set[tuple[str, str]],
|
|
877
|
+
max_results: int,
|
|
878
|
+
) -> None:
|
|
879
|
+
"""Append unique results to *merged* until *max_results* is reached."""
|
|
880
|
+
for result in new_results:
|
|
881
|
+
key = _result_key(result)
|
|
882
|
+
if key in seen:
|
|
883
|
+
continue
|
|
884
|
+
seen.add(key)
|
|
885
|
+
merged.append(result)
|
|
886
|
+
if len(merged) >= max_results:
|
|
887
|
+
return
|
|
888
|
+
|
|
889
|
+
|
|
890
|
+
def _extract_query_phrases(query: str, max_words: int = 4) -> list[str]:
|
|
891
|
+
"""Return overlapping word phrases from longest to shortest."""
|
|
892
|
+
words = re.findall(r"[A-Za-z][A-Za-z'’-]*", query)
|
|
893
|
+
phrases: list[str] = []
|
|
894
|
+
seen: set[str] = set()
|
|
895
|
+
for size in range(max_words, 0, -1):
|
|
896
|
+
for idx in range(0, len(words) - size + 1):
|
|
897
|
+
phrase = " ".join(words[idx: idx + size]).strip()
|
|
898
|
+
if not phrase:
|
|
899
|
+
continue
|
|
900
|
+
key = phrase.lower()
|
|
901
|
+
if key in seen:
|
|
902
|
+
continue
|
|
903
|
+
seen.add(key)
|
|
904
|
+
phrases.append(phrase)
|
|
905
|
+
return phrases
|
|
906
|
+
|
|
907
|
+
|
|
908
|
+
def _strip_possessive(text: str) -> str:
|
|
909
|
+
"""Strip a trailing possessive marker from *text*."""
|
|
910
|
+
return re.sub(r"(?:'s|’s)$", "", text.strip(), flags=re.IGNORECASE)
|
|
911
|
+
|
|
912
|
+
|
|
913
|
+
def _detect_subnational_focus(query: str) -> tuple[str | None, str | None]:
|
|
914
|
+
"""Return ADM1 place and parent country when the query is subnational."""
|
|
915
|
+
try:
|
|
916
|
+
from metacouplingllm.knowledge.adm1_pericoupling import (
|
|
917
|
+
get_adm1_info,
|
|
918
|
+
resolve_adm1_code,
|
|
919
|
+
)
|
|
920
|
+
from metacouplingllm.knowledge.countries import (
|
|
921
|
+
get_country_name,
|
|
922
|
+
resolve_country_code,
|
|
923
|
+
)
|
|
924
|
+
except Exception:
|
|
925
|
+
return None, None
|
|
926
|
+
|
|
927
|
+
for phrase in _extract_query_phrases(query):
|
|
928
|
+
cleaned = _strip_possessive(phrase)
|
|
929
|
+
if not cleaned:
|
|
930
|
+
continue
|
|
931
|
+
code = resolve_adm1_code(cleaned)
|
|
932
|
+
if not code:
|
|
933
|
+
continue
|
|
934
|
+
info = get_adm1_info(code)
|
|
935
|
+
if not info:
|
|
936
|
+
return cleaned, None
|
|
937
|
+
country_name = info.get("country_name")
|
|
938
|
+
country_code = resolve_country_code(country_name or "")
|
|
939
|
+
if country_code:
|
|
940
|
+
country_name = get_country_name(country_code)
|
|
941
|
+
return info.get("name", cleaned), country_name
|
|
942
|
+
|
|
943
|
+
return None, None
|
|
944
|
+
|
|
945
|
+
|
|
946
|
+
def _extract_topic_terms(
|
|
947
|
+
query: str,
|
|
948
|
+
*,
|
|
949
|
+
focal_place: str | None,
|
|
950
|
+
focal_country: str | None,
|
|
951
|
+
max_terms: int = 3,
|
|
952
|
+
) -> list[str]:
|
|
953
|
+
"""Extract likely commodity/topic terms from a research query."""
|
|
954
|
+
lowered = query.replace("’", "'").lower()
|
|
955
|
+
|
|
956
|
+
# Strip focal place/country names AND their ISO codes so they
|
|
957
|
+
# don't appear as topic terms (e.g., "usa" from "USA").
|
|
958
|
+
try:
|
|
959
|
+
from metacouplingllm.knowledge.countries import resolve_country_code
|
|
960
|
+
|
|
961
|
+
_extra_strips: list[str] = []
|
|
962
|
+
for phrase in (focal_place, focal_country):
|
|
963
|
+
if phrase:
|
|
964
|
+
code = resolve_country_code(phrase)
|
|
965
|
+
if code:
|
|
966
|
+
_extra_strips.append(code.lower())
|
|
967
|
+
except Exception:
|
|
968
|
+
_extra_strips = []
|
|
969
|
+
|
|
970
|
+
for phrase in (focal_place, focal_country):
|
|
971
|
+
if not phrase:
|
|
972
|
+
continue
|
|
973
|
+
lowered = re.sub(
|
|
974
|
+
rf"\b{re.escape(phrase.lower())}(?:'s)?\b",
|
|
975
|
+
" ",
|
|
976
|
+
lowered,
|
|
977
|
+
)
|
|
978
|
+
for code_lower in _extra_strips:
|
|
979
|
+
lowered = re.sub(rf"\b{re.escape(code_lower)}\b", " ", lowered)
|
|
980
|
+
|
|
981
|
+
topic_terms: list[str] = []
|
|
982
|
+
seen: set[str] = set()
|
|
983
|
+
for token in re.findall(r"[a-z][a-z'-]{2,}", lowered):
|
|
984
|
+
token = _strip_possessive(token.lower())
|
|
985
|
+
if (
|
|
986
|
+
token in _QUERY_FILLER
|
|
987
|
+
or token in _STRUCTURAL_TERMS
|
|
988
|
+
or token in _TRADE_HINTS
|
|
989
|
+
):
|
|
990
|
+
continue
|
|
991
|
+
if token in seen:
|
|
992
|
+
continue
|
|
993
|
+
seen.add(token)
|
|
994
|
+
topic_terms.append(token)
|
|
995
|
+
if len(topic_terms) >= max_terms:
|
|
996
|
+
break
|
|
997
|
+
|
|
998
|
+
return topic_terms
|
|
999
|
+
|
|
1000
|
+
|
|
1001
|
+
def _has_trade_intent(query: str) -> bool:
|
|
1002
|
+
"""Return True when the research query is about trade/export partners."""
|
|
1003
|
+
tokens = {
|
|
1004
|
+
_strip_possessive(token.lower())
|
|
1005
|
+
for token in re.findall(r"[A-Za-z][A-Za-z’’-]*", query)
|
|
1006
|
+
}
|
|
1007
|
+
return any(token in _TRADE_HINTS for token in tokens)
|
|
1008
|
+
|
|
1009
|
+
|
|
1010
|
+
def _detect_national_focus(query: str) -> str | None:
|
|
1011
|
+
"""Extract a country name from the query when no subnational region found.
|
|
1012
|
+
|
|
1013
|
+
E.g., ``"corn production and exports in USA"`` → ``"United States"``.
|
|
1014
|
+
"""
|
|
1015
|
+
try:
|
|
1016
|
+
from metacouplingllm.knowledge.countries import (
|
|
1017
|
+
get_country_name,
|
|
1018
|
+
resolve_country_code,
|
|
1019
|
+
)
|
|
1020
|
+
except Exception:
|
|
1021
|
+
return None
|
|
1022
|
+
|
|
1023
|
+
for phrase in _extract_query_phrases(query):
|
|
1024
|
+
cleaned = _strip_possessive(phrase)
|
|
1025
|
+
if not cleaned:
|
|
1026
|
+
continue
|
|
1027
|
+
code = resolve_country_code(cleaned)
|
|
1028
|
+
if code:
|
|
1029
|
+
return get_country_name(code) or cleaned
|
|
1030
|
+
|
|
1031
|
+
return None
|
|
1032
|
+
|
|
1033
|
+
|
|
1034
|
+
def _build_search_queries(query: str) -> list[str]:
|
|
1035
|
+
"""Expand a raw research query into higher-signal search variants."""
|
|
1036
|
+
base_query = " ".join(query.split())
|
|
1037
|
+
if not base_query:
|
|
1038
|
+
return []
|
|
1039
|
+
|
|
1040
|
+
if not _has_trade_intent(base_query):
|
|
1041
|
+
return [base_query]
|
|
1042
|
+
|
|
1043
|
+
# Try subnational focus first (e.g., "Michigan pork exports")
|
|
1044
|
+
focal_place, focal_country = _detect_subnational_focus(base_query)
|
|
1045
|
+
|
|
1046
|
+
# If no subnational focus, try national (e.g., "USA corn exports")
|
|
1047
|
+
if not focal_place:
|
|
1048
|
+
focal_country = _detect_national_focus(base_query)
|
|
1049
|
+
if not focal_country:
|
|
1050
|
+
return [base_query]
|
|
1051
|
+
focal_place = focal_country
|
|
1052
|
+
|
|
1053
|
+
topic_terms = _extract_topic_terms(
|
|
1054
|
+
base_query,
|
|
1055
|
+
focal_place=focal_place,
|
|
1056
|
+
focal_country=focal_country,
|
|
1057
|
+
)
|
|
1058
|
+
|
|
1059
|
+
queries: list[str] = []
|
|
1060
|
+
seen: set[str] = set()
|
|
1061
|
+
|
|
1062
|
+
def add(candidate: str) -> None:
|
|
1063
|
+
cleaned = " ".join(candidate.split()).strip()
|
|
1064
|
+
if not cleaned:
|
|
1065
|
+
return
|
|
1066
|
+
key = cleaned.lower()
|
|
1067
|
+
if key in seen:
|
|
1068
|
+
return
|
|
1069
|
+
seen.add(key)
|
|
1070
|
+
queries.append(cleaned)
|
|
1071
|
+
|
|
1072
|
+
if topic_terms:
|
|
1073
|
+
for term in topic_terms:
|
|
1074
|
+
add(f"{focal_place} {term} export destinations")
|
|
1075
|
+
add(f"{focal_place} {term} trade partners")
|
|
1076
|
+
add(f"{focal_place} {term} export markets")
|
|
1077
|
+
if focal_country:
|
|
1078
|
+
add(f"{focal_country} {term} exports top destinations")
|
|
1079
|
+
add(f"{focal_country} {term} export markets")
|
|
1080
|
+
else:
|
|
1081
|
+
add(f"{focal_place} export destinations")
|
|
1082
|
+
add(f"{focal_place} trade partners")
|
|
1083
|
+
add(f"{focal_place} export markets")
|
|
1084
|
+
if focal_country:
|
|
1085
|
+
add(f"{focal_country} exports top destinations")
|
|
1086
|
+
add(f"{focal_country} export markets")
|
|
1087
|
+
|
|
1088
|
+
add(base_query)
|
|
1089
|
+
return queries
|
|
1090
|
+
|
|
1091
|
+
|
|
1092
|
+
def _search_single_query(
|
|
1093
|
+
query: str,
|
|
1094
|
+
max_results: int,
|
|
1095
|
+
) -> list[dict[str, str]]:
|
|
1096
|
+
"""Search one query string across all backends and merge results."""
|
|
1097
|
+
merged: list[dict[str, str]] = []
|
|
1098
|
+
seen: set[tuple[str, str]] = set()
|
|
1099
|
+
backends = (
|
|
1100
|
+
("ddgs", _search_ddgs),
|
|
1101
|
+
("duckduckgo_search", _search_duckduckgo_search),
|
|
1102
|
+
("stdlib fallback", _search_stdlib),
|
|
1103
|
+
)
|
|
1104
|
+
|
|
1105
|
+
for backend_name, backend in backends:
|
|
1106
|
+
try:
|
|
1107
|
+
results = backend(query, max_results)
|
|
1108
|
+
except ImportError:
|
|
1109
|
+
continue
|
|
1110
|
+
except Exception as exc:
|
|
1111
|
+
print(f"[WebSearch] {backend_name} backend failed: {exc}")
|
|
1112
|
+
continue
|
|
1113
|
+
|
|
1114
|
+
if not results:
|
|
1115
|
+
continue
|
|
1116
|
+
|
|
1117
|
+
_merge_unique_results(
|
|
1118
|
+
merged, results, seen=seen, max_results=max_results,
|
|
1119
|
+
)
|
|
1120
|
+
if len(merged) >= max_results:
|
|
1121
|
+
break
|
|
1122
|
+
|
|
1123
|
+
return merged[:max_results]
|
|
1124
|
+
|
|
1125
|
+
|
|
1126
|
+
# -------------------------------------------------------------------
|
|
1127
|
+
# Public API
|
|
1128
|
+
# -------------------------------------------------------------------
|
|
1129
|
+
|
|
1130
|
+
_DUCKDUCKGO_LABEL = "DuckDuckGo (fan-out + top-up)"
|
|
1131
|
+
|
|
1132
|
+
|
|
1133
|
+
def _backend_display_name(backend: object | None) -> str:
|
|
1134
|
+
"""Return a human label describing *backend* for logs/console output."""
|
|
1135
|
+
if backend is None:
|
|
1136
|
+
return _DUCKDUCKGO_LABEL
|
|
1137
|
+
if isinstance(backend, AnthropicWebSearchBackend):
|
|
1138
|
+
return f"Claude web_search ({backend.model})"
|
|
1139
|
+
if isinstance(backend, OpenAIWebSearchBackend):
|
|
1140
|
+
return f"OpenAI web_search ({backend.model})"
|
|
1141
|
+
return type(backend).__name__
|
|
1142
|
+
|
|
1143
|
+
|
|
1144
|
+
def search_web(
|
|
1145
|
+
query: str,
|
|
1146
|
+
max_results: int = 5,
|
|
1147
|
+
*,
|
|
1148
|
+
backend: WebSearchBackend | None = None,
|
|
1149
|
+
metadata: dict[str, object] | None = None,
|
|
1150
|
+
) -> list[dict[str, str]]:
|
|
1151
|
+
"""Search the web and return top results.
|
|
1152
|
+
|
|
1153
|
+
Parameters
|
|
1154
|
+
----------
|
|
1155
|
+
query:
|
|
1156
|
+
Search query string.
|
|
1157
|
+
max_results:
|
|
1158
|
+
Maximum number of results to return.
|
|
1159
|
+
backend:
|
|
1160
|
+
Optional pluggable backend (e.g., ``OpenAIWebSearchBackend``,
|
|
1161
|
+
``AnthropicWebSearchBackend``). Tried first; falls back to the
|
|
1162
|
+
DuckDuckGo cascade on exception or empty result.
|
|
1163
|
+
metadata:
|
|
1164
|
+
Optional out-dict the caller can pass to learn which backend
|
|
1165
|
+
actually produced the returned results. When provided, it is
|
|
1166
|
+
populated with:
|
|
1167
|
+
|
|
1168
|
+
- ``"backend_used"``: human label of the backend that produced
|
|
1169
|
+
the results (e.g. ``"Claude web_search (claude-opus-4-7)"`` or
|
|
1170
|
+
``"DuckDuckGo (fan-out + top-up)"``).
|
|
1171
|
+
- ``"fallback_from"`` *(only present when fallback occurred)*: a
|
|
1172
|
+
short reason string like ``"Claude web_search raised: ..."``
|
|
1173
|
+
or ``"OpenAI web_search returned 0 results"``.
|
|
1174
|
+
|
|
1175
|
+
Returns
|
|
1176
|
+
-------
|
|
1177
|
+
A list of dicts, each with ``title``, ``snippet``, and ``url`` keys.
|
|
1178
|
+
Returns an empty list if the search fails or the library is not
|
|
1179
|
+
installed.
|
|
1180
|
+
|
|
1181
|
+
Notes
|
|
1182
|
+
-----
|
|
1183
|
+
When *backend* is provided, it is tried first. If it yields no usable
|
|
1184
|
+
results or raises, the built-in DuckDuckGo-family backends are used.
|
|
1185
|
+
|
|
1186
|
+
The built-in search path attempts three backends in order, and unique
|
|
1187
|
+
results are merged until *max_results* is reached:
|
|
1188
|
+
|
|
1189
|
+
1. ``ddgs`` (pip install ddgs)
|
|
1190
|
+
2. ``duckduckgo_search`` (pip install duckduckgo-search)
|
|
1191
|
+
3. stdlib-only fallback using ``urllib`` + ``html.parser`` — works
|
|
1192
|
+
everywhere including Google Colab without extra packages.
|
|
1193
|
+
"""
|
|
1194
|
+
if max_results <= 0:
|
|
1195
|
+
return []
|
|
1196
|
+
|
|
1197
|
+
fallback_reason: str | None = None
|
|
1198
|
+
if backend is not None:
|
|
1199
|
+
try:
|
|
1200
|
+
results = _normalise_backend_results(
|
|
1201
|
+
backend.search(query, max_results=max_results),
|
|
1202
|
+
max_results=max_results,
|
|
1203
|
+
)
|
|
1204
|
+
except Exception as exc:
|
|
1205
|
+
print(f"[WebSearch] custom backend failed: {exc}")
|
|
1206
|
+
fallback_reason = (
|
|
1207
|
+
f"{_backend_display_name(backend)} raised: {exc}"
|
|
1208
|
+
)
|
|
1209
|
+
else:
|
|
1210
|
+
if results:
|
|
1211
|
+
if metadata is not None:
|
|
1212
|
+
metadata["backend_used"] = _backend_display_name(backend)
|
|
1213
|
+
return results[:max_results]
|
|
1214
|
+
fallback_reason = (
|
|
1215
|
+
f"{_backend_display_name(backend)} returned 0 results"
|
|
1216
|
+
)
|
|
1217
|
+
|
|
1218
|
+
queries = _build_search_queries(query)
|
|
1219
|
+
if not queries:
|
|
1220
|
+
if metadata is not None:
|
|
1221
|
+
metadata["backend_used"] = _DUCKDUCKGO_LABEL
|
|
1222
|
+
if fallback_reason is not None:
|
|
1223
|
+
metadata["fallback_from"] = fallback_reason
|
|
1224
|
+
return []
|
|
1225
|
+
|
|
1226
|
+
merged: list[dict[str, str]] = []
|
|
1227
|
+
seen: set[tuple[str, str]] = set()
|
|
1228
|
+
per_query_max = (
|
|
1229
|
+
max_results
|
|
1230
|
+
if len(queries) == 1
|
|
1231
|
+
else max(4, min(max_results, 6))
|
|
1232
|
+
)
|
|
1233
|
+
|
|
1234
|
+
for search_query in queries:
|
|
1235
|
+
results = _search_single_query(search_query, per_query_max)
|
|
1236
|
+
if not results:
|
|
1237
|
+
continue
|
|
1238
|
+
|
|
1239
|
+
_merge_unique_results(
|
|
1240
|
+
merged,
|
|
1241
|
+
results,
|
|
1242
|
+
seen=seen,
|
|
1243
|
+
max_results=max_results,
|
|
1244
|
+
)
|
|
1245
|
+
if len(merged) >= max_results:
|
|
1246
|
+
break
|
|
1247
|
+
|
|
1248
|
+
# Top-up: when fan-out is used (multiple variants) AND the per-variant
|
|
1249
|
+
# cap left us short of max_results, re-issue the raw query at the full
|
|
1250
|
+
# budget. The dedup keys ensure variant URLs are not double-counted.
|
|
1251
|
+
# This recovers cases where a single query has plenty of unique hits
|
|
1252
|
+
# available (e.g., "Michigan pork exports" returns ~47 unique at max=50)
|
|
1253
|
+
# that the per-variant cap of ~6 would otherwise throttle away.
|
|
1254
|
+
if len(queries) > 1 and len(merged) < max_results:
|
|
1255
|
+
topup = _search_single_query(queries[-1], max_results)
|
|
1256
|
+
_merge_unique_results(
|
|
1257
|
+
merged,
|
|
1258
|
+
topup,
|
|
1259
|
+
seen=seen,
|
|
1260
|
+
max_results=max_results,
|
|
1261
|
+
)
|
|
1262
|
+
|
|
1263
|
+
if metadata is not None:
|
|
1264
|
+
metadata["backend_used"] = _DUCKDUCKGO_LABEL
|
|
1265
|
+
if fallback_reason is not None:
|
|
1266
|
+
metadata["fallback_from"] = fallback_reason
|
|
1267
|
+
|
|
1268
|
+
return merged[:max_results]
|
|
1269
|
+
|
|
1270
|
+
|
|
1271
|
+
def format_web_context(results: list[dict[str, str]]) -> str:
|
|
1272
|
+
"""Format web search results into a prompt-ready context section.
|
|
1273
|
+
|
|
1274
|
+
Parameters
|
|
1275
|
+
----------
|
|
1276
|
+
results:
|
|
1277
|
+
List of search result dicts from :func:`search_web`.
|
|
1278
|
+
|
|
1279
|
+
Returns
|
|
1280
|
+
-------
|
|
1281
|
+
A formatted string starting with ``## WEB SEARCH CONTEXT``, or an
|
|
1282
|
+
empty string if no results are provided.
|
|
1283
|
+
"""
|
|
1284
|
+
if not results:
|
|
1285
|
+
return ""
|
|
1286
|
+
|
|
1287
|
+
lines: list[str] = [
|
|
1288
|
+
"## WEB SEARCH CONTEXT",
|
|
1289
|
+
"",
|
|
1290
|
+
"The following web search snippets provide real-world context "
|
|
1291
|
+
"for the research topic. Use this information to ground your "
|
|
1292
|
+
"analysis in current facts (e.g., trade volumes, destinations, "
|
|
1293
|
+
"policies), but always apply the metacoupling framework "
|
|
1294
|
+
"rigorously.",
|
|
1295
|
+
"",
|
|
1296
|
+
"For subnational trade topics, some snippets may come from "
|
|
1297
|
+
"broader country-level searches when place-specific trade-partner "
|
|
1298
|
+
"data are sparse. Treat those as proxy context and label them "
|
|
1299
|
+
"clearly rather than presenting them as direct subnational facts.",
|
|
1300
|
+
"",
|
|
1301
|
+
"When you use a fact from a web snippet, cite it inline as "
|
|
1302
|
+
"[W1], [W2], etc. These are distinct from the literature "
|
|
1303
|
+
"citations [1], [2], … that may be added separately.",
|
|
1304
|
+
"",
|
|
1305
|
+
]
|
|
1306
|
+
|
|
1307
|
+
for i, r in enumerate(results, 1):
|
|
1308
|
+
title = r.get("title", "Untitled")
|
|
1309
|
+
snippet = r.get("snippet", "")
|
|
1310
|
+
url = r.get("url", "")
|
|
1311
|
+
lines.append(f"[W{i}] **{title}**")
|
|
1312
|
+
if snippet:
|
|
1313
|
+
lines.append(f" {snippet}")
|
|
1314
|
+
if url:
|
|
1315
|
+
lines.append(f" Source: {url}")
|
|
1316
|
+
lines.append("")
|
|
1317
|
+
|
|
1318
|
+
return "\n".join(lines)
|
|
1319
|
+
|
|
1320
|
+
|
|
1321
|
+
def _extract_json_object(text: str) -> dict[str, object] | None:
|
|
1322
|
+
"""Best-effort extraction of a JSON object from model output.
|
|
1323
|
+
|
|
1324
|
+
Handles common LLM output patterns including:
|
|
1325
|
+
- Bare JSON
|
|
1326
|
+
- JSON inside markdown fenced code blocks
|
|
1327
|
+
- JSON surrounded by explanation text
|
|
1328
|
+
- Truncated JSON (attempts repair by closing brackets)
|
|
1329
|
+
"""
|
|
1330
|
+
if not text or not text.strip():
|
|
1331
|
+
return None
|
|
1332
|
+
|
|
1333
|
+
stripped = text.strip()
|
|
1334
|
+
candidates: list[str] = [stripped]
|
|
1335
|
+
|
|
1336
|
+
# Try fenced code blocks first (most reliable)
|
|
1337
|
+
fenced = re.findall(r"```(?:json)?\s*(\{.*?\})\s*```", stripped, re.DOTALL)
|
|
1338
|
+
candidates.extend(fenced)
|
|
1339
|
+
|
|
1340
|
+
# Try extracting between first { and last }
|
|
1341
|
+
first = stripped.find("{")
|
|
1342
|
+
last = stripped.rfind("}")
|
|
1343
|
+
if first != -1 and last != -1 and last > first:
|
|
1344
|
+
candidates.append(stripped[first:last + 1])
|
|
1345
|
+
|
|
1346
|
+
for candidate in candidates:
|
|
1347
|
+
try:
|
|
1348
|
+
obj = json.loads(candidate)
|
|
1349
|
+
except (json.JSONDecodeError, ValueError):
|
|
1350
|
+
continue
|
|
1351
|
+
if isinstance(obj, dict):
|
|
1352
|
+
return obj
|
|
1353
|
+
|
|
1354
|
+
# Last resort: try to repair truncated JSON by closing open brackets.
|
|
1355
|
+
# This handles the case where max_tokens cut the response mid-JSON.
|
|
1356
|
+
if first != -1:
|
|
1357
|
+
raw = stripped[first:]
|
|
1358
|
+
# Count unclosed braces/brackets and close them
|
|
1359
|
+
open_braces = raw.count("{") - raw.count("}")
|
|
1360
|
+
open_brackets = raw.count("[") - raw.count("]")
|
|
1361
|
+
if open_braces > 0 or open_brackets > 0:
|
|
1362
|
+
# Strip any trailing incomplete key-value pair
|
|
1363
|
+
repaired = re.sub(r',\s*"[^"]*"\s*:\s*$', "", raw)
|
|
1364
|
+
repaired = re.sub(r',\s*$', "", repaired)
|
|
1365
|
+
repaired += "]" * max(0, open_brackets) + "}" * max(0, open_braces)
|
|
1366
|
+
try:
|
|
1367
|
+
obj = json.loads(repaired)
|
|
1368
|
+
except (json.JSONDecodeError, ValueError):
|
|
1369
|
+
pass
|
|
1370
|
+
else:
|
|
1371
|
+
if isinstance(obj, dict):
|
|
1372
|
+
return obj
|
|
1373
|
+
|
|
1374
|
+
return None
|
|
1375
|
+
|
|
1376
|
+
|
|
1377
|
+
def _normalise_web_evidence_ids(
|
|
1378
|
+
evidence: object,
|
|
1379
|
+
available_ids: set[str],
|
|
1380
|
+
) -> list[str]:
|
|
1381
|
+
"""Return valid evidence ids like ``['W1', 'W2']``."""
|
|
1382
|
+
if not isinstance(evidence, list):
|
|
1383
|
+
return []
|
|
1384
|
+
cleaned: list[str] = []
|
|
1385
|
+
for item in evidence:
|
|
1386
|
+
if not isinstance(item, str):
|
|
1387
|
+
continue
|
|
1388
|
+
match = re.search(r"W\d+", item.upper())
|
|
1389
|
+
if not match:
|
|
1390
|
+
continue
|
|
1391
|
+
code = match.group(0)
|
|
1392
|
+
if code in available_ids and code not in cleaned:
|
|
1393
|
+
cleaned.append(code)
|
|
1394
|
+
return cleaned
|
|
1395
|
+
|
|
1396
|
+
|
|
1397
|
+
def _coerce_confidence(value: object) -> float:
|
|
1398
|
+
"""Convert *value* to a bounded confidence score."""
|
|
1399
|
+
try:
|
|
1400
|
+
score = float(value)
|
|
1401
|
+
except Exception:
|
|
1402
|
+
return 0.0
|
|
1403
|
+
return max(0.0, min(1.0, score))
|
|
1404
|
+
|
|
1405
|
+
|
|
1406
|
+
def extract_web_map_signals(
|
|
1407
|
+
query: str,
|
|
1408
|
+
results: list[dict[str, str]],
|
|
1409
|
+
llm_client: object,
|
|
1410
|
+
*,
|
|
1411
|
+
min_confidence: float = 0.7,
|
|
1412
|
+
max_targets: int = 6,
|
|
1413
|
+
temperature: float = 0.0,
|
|
1414
|
+
max_tokens: int = 8192,
|
|
1415
|
+
) -> dict[str, object] | None:
|
|
1416
|
+
"""Use an LLM to extract validated map-ready signals from web snippets.
|
|
1417
|
+
|
|
1418
|
+
Returns a dict containing ``focal_country``, ``receiving_systems``,
|
|
1419
|
+
``spillover_systems``, and ``flows`` when extraction succeeds.
|
|
1420
|
+
"""
|
|
1421
|
+
if not results:
|
|
1422
|
+
return None
|
|
1423
|
+
|
|
1424
|
+
from metacouplingllm.knowledge.countries import (
|
|
1425
|
+
get_country_name,
|
|
1426
|
+
resolve_country_code,
|
|
1427
|
+
)
|
|
1428
|
+
from metacouplingllm.llm.client import Message
|
|
1429
|
+
|
|
1430
|
+
available_ids = {f"W{i}" for i in range(1, len(results) + 1)}
|
|
1431
|
+
snippet_lines: list[str] = []
|
|
1432
|
+
for idx, result in enumerate(results, 1):
|
|
1433
|
+
snippet_lines.append(f"[W{idx}] {result.get('title', '').strip()}")
|
|
1434
|
+
snippet_lines.append(f"Snippet: {result.get('snippet', '').strip()}")
|
|
1435
|
+
snippet_lines.append(f"URL: {result.get('url', '').strip()}")
|
|
1436
|
+
snippet_lines.append("")
|
|
1437
|
+
|
|
1438
|
+
schema = {
|
|
1439
|
+
"focal_country": "country name, ISO code, or null",
|
|
1440
|
+
"receiving_systems": [
|
|
1441
|
+
{
|
|
1442
|
+
"country": "country name or ISO code",
|
|
1443
|
+
"kind": "direct or proxy",
|
|
1444
|
+
"confidence": 0.0,
|
|
1445
|
+
"evidence": ["W1"],
|
|
1446
|
+
"reason": "short explanation",
|
|
1447
|
+
}
|
|
1448
|
+
],
|
|
1449
|
+
"spillover_systems": [
|
|
1450
|
+
{
|
|
1451
|
+
"country": "country name or ISO code",
|
|
1452
|
+
"kind": "direct or proxy",
|
|
1453
|
+
"confidence": 0.0,
|
|
1454
|
+
"evidence": ["W2"],
|
|
1455
|
+
"reason": "short explanation",
|
|
1456
|
+
}
|
|
1457
|
+
],
|
|
1458
|
+
"flows": [
|
|
1459
|
+
{
|
|
1460
|
+
"category": "matter/capital/information/energy/people/organisms",
|
|
1461
|
+
"source_country": "country name or ISO code",
|
|
1462
|
+
"target_country": "country name or ISO code",
|
|
1463
|
+
"kind": "direct or proxy",
|
|
1464
|
+
"confidence": 0.0,
|
|
1465
|
+
"evidence": ["W1"],
|
|
1466
|
+
"description": "short flow description",
|
|
1467
|
+
}
|
|
1468
|
+
],
|
|
1469
|
+
}
|
|
1470
|
+
system_text = (
|
|
1471
|
+
"You extract conservative, map-ready metacoupling signals from web "
|
|
1472
|
+
"search snippets. Use only the provided snippets. Do not invent "
|
|
1473
|
+
"countries, destinations, or flows. Return JSON only."
|
|
1474
|
+
)
|
|
1475
|
+
user_text = (
|
|
1476
|
+
f"Research query:\n{query.strip()}\n\n"
|
|
1477
|
+
"Extract map-ready countries and flows for a metacoupling map.\n"
|
|
1478
|
+
"Rules:\n"
|
|
1479
|
+
"- Use null or empty lists when uncertain.\n"
|
|
1480
|
+
"- Label items as 'proxy' when the snippet is broader than the focal "
|
|
1481
|
+
"study but still useful context.\n"
|
|
1482
|
+
"- Only include countries that are explicitly supported by the "
|
|
1483
|
+
"snippets.\n"
|
|
1484
|
+
"- ONLY extract countries and flows that are DIRECTLY RELEVANT to "
|
|
1485
|
+
"the research query above. Ignore trade data about unrelated "
|
|
1486
|
+
"products, sectors, or commodities, even if they involve the focal "
|
|
1487
|
+
"country. For example, if the research is about feed barley, do NOT "
|
|
1488
|
+
"extract sheep offal exports or unrelated commodity flows.\n"
|
|
1489
|
+
"- receiving_systems = countries that BUY from or RECEIVE "
|
|
1490
|
+
"goods/services/capital from the focal country (trade partners, "
|
|
1491
|
+
"importers)\n"
|
|
1492
|
+
"- spillover_systems = competitors, indirectly affected countries, "
|
|
1493
|
+
"or countries that experience environmental/economic spillover "
|
|
1494
|
+
"effects\n"
|
|
1495
|
+
"- NEVER put competing exporter countries in receiving_systems. "
|
|
1496
|
+
"Competitors belong in spillover_systems.\n"
|
|
1497
|
+
"- Most flows go FROM the focal country TO receiving countries "
|
|
1498
|
+
"(matter, information, energy, people, organisms).\n"
|
|
1499
|
+
"- Capital/payment flows go in REVERSE: FROM receiving countries "
|
|
1500
|
+
"TO the focal country.\n"
|
|
1501
|
+
"- Do NOT create flows from spillover/competitor countries to "
|
|
1502
|
+
"anyone. For example, if USA exports corn to Mexico "
|
|
1503
|
+
"and Brazil is a competitor, do NOT add a Brazil \u2192 Mexico flow.\n"
|
|
1504
|
+
"- Keep at most 6 receiving systems, 6 spillover systems, and 8 flows.\n"
|
|
1505
|
+
"- Every item must include evidence ids like W1.\n\n"
|
|
1506
|
+
f"JSON schema example:\n{json.dumps(schema, ensure_ascii=True)}\n\n"
|
|
1507
|
+
"Web snippets:\n"
|
|
1508
|
+
f"{chr(10).join(snippet_lines)}"
|
|
1509
|
+
)
|
|
1510
|
+
|
|
1511
|
+
response = llm_client.chat(
|
|
1512
|
+
messages=[
|
|
1513
|
+
Message(role="system", content=system_text),
|
|
1514
|
+
Message(role="user", content=user_text),
|
|
1515
|
+
],
|
|
1516
|
+
temperature=temperature,
|
|
1517
|
+
max_tokens=max_tokens,
|
|
1518
|
+
)
|
|
1519
|
+
raw_obj = _extract_json_object(response.content)
|
|
1520
|
+
if raw_obj is None:
|
|
1521
|
+
return None
|
|
1522
|
+
|
|
1523
|
+
def _normalise_country_entry(item: object) -> dict[str, object] | None:
|
|
1524
|
+
if not isinstance(item, dict):
|
|
1525
|
+
return None
|
|
1526
|
+
code = resolve_country_code(str(item.get("country", "")).strip())
|
|
1527
|
+
if not code:
|
|
1528
|
+
return None
|
|
1529
|
+
confidence = _coerce_confidence(item.get("confidence", 0.0))
|
|
1530
|
+
evidence = _normalise_web_evidence_ids(
|
|
1531
|
+
item.get("evidence", []),
|
|
1532
|
+
available_ids,
|
|
1533
|
+
)
|
|
1534
|
+
if confidence < min_confidence or not evidence:
|
|
1535
|
+
return None
|
|
1536
|
+
kind = str(item.get("kind", "direct")).strip().lower()
|
|
1537
|
+
if kind not in {"direct", "proxy"}:
|
|
1538
|
+
kind = "direct"
|
|
1539
|
+
reason = str(item.get("reason", "")).strip()
|
|
1540
|
+
return {
|
|
1541
|
+
"country": code,
|
|
1542
|
+
"kind": kind,
|
|
1543
|
+
"confidence": confidence,
|
|
1544
|
+
"evidence": evidence,
|
|
1545
|
+
"reason": reason,
|
|
1546
|
+
}
|
|
1547
|
+
|
|
1548
|
+
def _normalise_flow_entry(item: object) -> dict[str, object] | None:
|
|
1549
|
+
if not isinstance(item, dict):
|
|
1550
|
+
return None
|
|
1551
|
+
src_code = resolve_country_code(str(item.get("source_country", "")).strip())
|
|
1552
|
+
tgt_code = resolve_country_code(str(item.get("target_country", "")).strip())
|
|
1553
|
+
if not src_code or not tgt_code or src_code == tgt_code:
|
|
1554
|
+
return None
|
|
1555
|
+
category = str(item.get("category", "")).strip().lower()
|
|
1556
|
+
if category not in {
|
|
1557
|
+
"matter", "material", "capital", "financial", "information",
|
|
1558
|
+
"energy", "people", "organisms",
|
|
1559
|
+
}:
|
|
1560
|
+
return None
|
|
1561
|
+
confidence = _coerce_confidence(item.get("confidence", 0.0))
|
|
1562
|
+
evidence = _normalise_web_evidence_ids(
|
|
1563
|
+
item.get("evidence", []),
|
|
1564
|
+
available_ids,
|
|
1565
|
+
)
|
|
1566
|
+
if confidence < min_confidence or not evidence:
|
|
1567
|
+
return None
|
|
1568
|
+
kind = str(item.get("kind", "direct")).strip().lower()
|
|
1569
|
+
if kind not in {"direct", "proxy"}:
|
|
1570
|
+
kind = "direct"
|
|
1571
|
+
description = str(item.get("description", "")).strip()
|
|
1572
|
+
src_name = get_country_name(src_code) or src_code
|
|
1573
|
+
tgt_name = get_country_name(tgt_code) or tgt_code
|
|
1574
|
+
return {
|
|
1575
|
+
"category": category,
|
|
1576
|
+
"source_country": src_code,
|
|
1577
|
+
"target_country": tgt_code,
|
|
1578
|
+
"direction": f"{src_name} \u2192 {tgt_name}",
|
|
1579
|
+
"description": description,
|
|
1580
|
+
"kind": kind,
|
|
1581
|
+
"confidence": confidence,
|
|
1582
|
+
"evidence": evidence,
|
|
1583
|
+
}
|
|
1584
|
+
|
|
1585
|
+
focal_country = resolve_country_code(str(raw_obj.get("focal_country", "")).strip())
|
|
1586
|
+
receiving: list[dict[str, object]] = []
|
|
1587
|
+
seen_receiving: set[str] = set()
|
|
1588
|
+
for item in raw_obj.get("receiving_systems", []) if isinstance(raw_obj.get("receiving_systems", []), list) else []:
|
|
1589
|
+
normalized = _normalise_country_entry(item)
|
|
1590
|
+
if not normalized:
|
|
1591
|
+
continue
|
|
1592
|
+
code = str(normalized["country"])
|
|
1593
|
+
if code in seen_receiving:
|
|
1594
|
+
continue
|
|
1595
|
+
seen_receiving.add(code)
|
|
1596
|
+
receiving.append(normalized)
|
|
1597
|
+
if len(receiving) >= max_targets:
|
|
1598
|
+
break
|
|
1599
|
+
|
|
1600
|
+
spillover: list[dict[str, object]] = []
|
|
1601
|
+
seen_spillover: set[str] = set()
|
|
1602
|
+
for item in raw_obj.get("spillover_systems", []) if isinstance(raw_obj.get("spillover_systems", []), list) else []:
|
|
1603
|
+
normalized = _normalise_country_entry(item)
|
|
1604
|
+
if not normalized:
|
|
1605
|
+
continue
|
|
1606
|
+
code = str(normalized["country"])
|
|
1607
|
+
if code in seen_spillover:
|
|
1608
|
+
continue
|
|
1609
|
+
seen_spillover.add(code)
|
|
1610
|
+
spillover.append(normalized)
|
|
1611
|
+
if len(spillover) >= max_targets:
|
|
1612
|
+
break
|
|
1613
|
+
|
|
1614
|
+
flows: list[dict[str, object]] = []
|
|
1615
|
+
seen_flows: set[tuple[str, str, str]] = set()
|
|
1616
|
+
raw_flows = raw_obj.get("flows", [])
|
|
1617
|
+
if isinstance(raw_flows, list):
|
|
1618
|
+
for item in raw_flows:
|
|
1619
|
+
normalized = _normalise_flow_entry(item)
|
|
1620
|
+
if not normalized:
|
|
1621
|
+
continue
|
|
1622
|
+
key = (
|
|
1623
|
+
str(normalized["category"]),
|
|
1624
|
+
str(normalized["source_country"]),
|
|
1625
|
+
str(normalized["target_country"]),
|
|
1626
|
+
)
|
|
1627
|
+
if key in seen_flows:
|
|
1628
|
+
continue
|
|
1629
|
+
seen_flows.add(key)
|
|
1630
|
+
flows.append(normalized)
|
|
1631
|
+
if len(flows) >= max_targets + 2:
|
|
1632
|
+
break
|
|
1633
|
+
|
|
1634
|
+
if not focal_country and receiving and flows:
|
|
1635
|
+
focal_country = str(flows[0]["source_country"])
|
|
1636
|
+
|
|
1637
|
+
if not any((focal_country, receiving, spillover, flows)):
|
|
1638
|
+
return None
|
|
1639
|
+
|
|
1640
|
+
return {
|
|
1641
|
+
"focal_country": focal_country,
|
|
1642
|
+
"receiving_systems": receiving,
|
|
1643
|
+
"spillover_systems": spillover,
|
|
1644
|
+
"flows": flows,
|
|
1645
|
+
}
|
|
1646
|
+
|
|
1647
|
+
|
|
1648
|
+
def format_web_map_signals_context(signals: dict[str, object] | None) -> str:
|
|
1649
|
+
"""Format structured web map signals into a prompt-ready section."""
|
|
1650
|
+
if not signals:
|
|
1651
|
+
return ""
|
|
1652
|
+
|
|
1653
|
+
lines = [
|
|
1654
|
+
"## STRUCTURED WEB MAP SIGNALS",
|
|
1655
|
+
"",
|
|
1656
|
+
"The following countries and flows were conservatively extracted "
|
|
1657
|
+
"from the web snippets and validated for mapping. Treat these as "
|
|
1658
|
+
"high-priority map hints, while still reasoning carefully about "
|
|
1659
|
+
"direct versus proxy evidence.",
|
|
1660
|
+
"",
|
|
1661
|
+
]
|
|
1662
|
+
|
|
1663
|
+
focal = signals.get("focal_country")
|
|
1664
|
+
if isinstance(focal, str) and focal:
|
|
1665
|
+
lines.append(f"- Focal country: {focal}")
|
|
1666
|
+
|
|
1667
|
+
receiving = signals.get("receiving_systems", [])
|
|
1668
|
+
if isinstance(receiving, list) and receiving:
|
|
1669
|
+
lines.append("- Receiving systems:")
|
|
1670
|
+
for item in receiving:
|
|
1671
|
+
if not isinstance(item, dict):
|
|
1672
|
+
continue
|
|
1673
|
+
lines.append(
|
|
1674
|
+
" - "
|
|
1675
|
+
f"{item.get('country')} ({item.get('kind', 'direct')}, "
|
|
1676
|
+
f"confidence={item.get('confidence', 0):.2f}, "
|
|
1677
|
+
f"evidence={','.join(item.get('evidence', []))})"
|
|
1678
|
+
)
|
|
1679
|
+
|
|
1680
|
+
spillover = signals.get("spillover_systems", [])
|
|
1681
|
+
if isinstance(spillover, list) and spillover:
|
|
1682
|
+
lines.append("- Spillover systems:")
|
|
1683
|
+
for item in spillover:
|
|
1684
|
+
if not isinstance(item, dict):
|
|
1685
|
+
continue
|
|
1686
|
+
lines.append(
|
|
1687
|
+
" - "
|
|
1688
|
+
f"{item.get('country')} ({item.get('kind', 'direct')}, "
|
|
1689
|
+
f"confidence={item.get('confidence', 0):.2f}, "
|
|
1690
|
+
f"evidence={','.join(item.get('evidence', []))})"
|
|
1691
|
+
)
|
|
1692
|
+
|
|
1693
|
+
flows = signals.get("flows", [])
|
|
1694
|
+
if isinstance(flows, list) and flows:
|
|
1695
|
+
lines.append("- Map-ready flows:")
|
|
1696
|
+
for item in flows:
|
|
1697
|
+
if not isinstance(item, dict):
|
|
1698
|
+
continue
|
|
1699
|
+
lines.append(
|
|
1700
|
+
" - "
|
|
1701
|
+
f"{item.get('category')}: {item.get('direction')} "
|
|
1702
|
+
f"({item.get('kind', 'direct')}, "
|
|
1703
|
+
f"confidence={item.get('confidence', 0):.2f}, "
|
|
1704
|
+
f"evidence={','.join(item.get('evidence', []))})"
|
|
1705
|
+
)
|
|
1706
|
+
|
|
1707
|
+
return "\n".join(lines)
|
|
1708
|
+
|
|
1709
|
+
|
|
1710
|
+
# -------------------------------------------------------------------
|
|
1711
|
+
# Post-processing: add inline [WN] citations
|
|
1712
|
+
# -------------------------------------------------------------------
|
|
1713
|
+
|
|
1714
|
+
# Stopwords excluded from keyword matching
|
|
1715
|
+
_STOP = frozenset(
|
|
1716
|
+
"a an the and or but in on of to for is are was were be been by "
|
|
1717
|
+
"at as it its with from this that these those not no do does did "
|
|
1718
|
+
"has have had will would shall should can could may might must "
|
|
1719
|
+
"so if than too very also about more most such each every all "
|
|
1720
|
+
"any both few many much some their them they which what when "
|
|
1721
|
+
"where who how because between into through during before after "
|
|
1722
|
+
"above below up down out off over under again further then once "
|
|
1723
|
+
"here there just only own same other another new old well still "
|
|
1724
|
+
"even us our we you your he she him her his".split()
|
|
1725
|
+
)
|
|
1726
|
+
|
|
1727
|
+
|
|
1728
|
+
def _web_tokenise(text: str) -> set[str]:
|
|
1729
|
+
"""Extract distinctive lowercase keywords from *text*."""
|
|
1730
|
+
tokens = re.findall(r"[a-z0-9]{3,}", text.lower())
|
|
1731
|
+
return {t for t in tokens if t not in _STOP}
|
|
1732
|
+
|
|
1733
|
+
|
|
1734
|
+
def annotate_web_citations(
|
|
1735
|
+
formatted: str,
|
|
1736
|
+
web_results: list[dict[str, str]],
|
|
1737
|
+
min_keyword_overlap: int = 3,
|
|
1738
|
+
min_overlap_ratio: float = 0.25,
|
|
1739
|
+
) -> str:
|
|
1740
|
+
"""Add inline ``[WN]`` citations to formatted analysis lines.
|
|
1741
|
+
|
|
1742
|
+
Works the same way as the literature ``annotate_citations`` but uses
|
|
1743
|
+
web-search snippets and the ``[W1]``, ``[W2]``, … prefix so that
|
|
1744
|
+
web and literature references are visually distinct.
|
|
1745
|
+
|
|
1746
|
+
Parameters
|
|
1747
|
+
----------
|
|
1748
|
+
formatted:
|
|
1749
|
+
The formatted analysis text.
|
|
1750
|
+
web_results:
|
|
1751
|
+
Web search result dicts (each with ``title``, ``snippet``, ``url``).
|
|
1752
|
+
min_keyword_overlap:
|
|
1753
|
+
Minimum shared keywords for a citation to be added.
|
|
1754
|
+
min_overlap_ratio:
|
|
1755
|
+
Minimum fraction of the line's keywords that must appear in the
|
|
1756
|
+
web snippet.
|
|
1757
|
+
|
|
1758
|
+
Returns
|
|
1759
|
+
-------
|
|
1760
|
+
The text with ``[WN]`` citations appended to matching lines.
|
|
1761
|
+
"""
|
|
1762
|
+
if not web_results:
|
|
1763
|
+
return formatted
|
|
1764
|
+
|
|
1765
|
+
# Build keyword sets for each web snippet
|
|
1766
|
+
snippet_keywords: list[set[str]] = []
|
|
1767
|
+
for r in web_results:
|
|
1768
|
+
combined = (r.get("title", "") + " " + r.get("snippet", "")).strip()
|
|
1769
|
+
snippet_keywords.append(_web_tokenise(combined))
|
|
1770
|
+
|
|
1771
|
+
_skip_prefixes = (
|
|
1772
|
+
"COUPLING CLASSIFICATION", "SYSTEMS IDENTIFICATION",
|
|
1773
|
+
"FLOWS ANALYSIS", "AGENTS", "CAUSES", "EFFECTS",
|
|
1774
|
+
"RESEARCH GAPS", "PERICOUPLING DATABASE",
|
|
1775
|
+
"METACOUPLING FRAMEWORK", "SUPPORTING EVIDENCE",
|
|
1776
|
+
"RECOMMENDED LITERATURE", "WEB SOURCES", "Note:",
|
|
1777
|
+
)
|
|
1778
|
+
|
|
1779
|
+
lines = formatted.split("\n")
|
|
1780
|
+
annotated: list[str] = []
|
|
1781
|
+
|
|
1782
|
+
in_ref_block = False
|
|
1783
|
+
|
|
1784
|
+
for line in lines:
|
|
1785
|
+
stripped = line.strip()
|
|
1786
|
+
|
|
1787
|
+
# Don't annotate inside reference blocks (evidence or web sources)
|
|
1788
|
+
if any(h in stripped for h in (
|
|
1789
|
+
"SUPPORTING EVIDENCE FROM LITERATURE",
|
|
1790
|
+
"WEB SOURCES",
|
|
1791
|
+
)):
|
|
1792
|
+
in_ref_block = True
|
|
1793
|
+
if in_ref_block:
|
|
1794
|
+
annotated.append(line)
|
|
1795
|
+
continue
|
|
1796
|
+
|
|
1797
|
+
# Skip empty lines, separators, and section headers
|
|
1798
|
+
if (
|
|
1799
|
+
not stripped
|
|
1800
|
+
or stripped.startswith(tuple(_skip_prefixes))
|
|
1801
|
+
or set(stripped) <= {"=", "-", " "}
|
|
1802
|
+
):
|
|
1803
|
+
annotated.append(line)
|
|
1804
|
+
continue
|
|
1805
|
+
|
|
1806
|
+
# Skip sub-headings
|
|
1807
|
+
if stripped.startswith("[") and stripped.endswith("]"):
|
|
1808
|
+
annotated.append(line)
|
|
1809
|
+
continue
|
|
1810
|
+
if stripped.endswith(":") and len(stripped.split()) <= 3:
|
|
1811
|
+
annotated.append(line)
|
|
1812
|
+
continue
|
|
1813
|
+
|
|
1814
|
+
line_tokens = _web_tokenise(stripped)
|
|
1815
|
+
if len(line_tokens) < 3:
|
|
1816
|
+
annotated.append(line)
|
|
1817
|
+
continue
|
|
1818
|
+
|
|
1819
|
+
# Find matching web snippets
|
|
1820
|
+
citations: list[int] = []
|
|
1821
|
+
for idx, sk in enumerate(snippet_keywords):
|
|
1822
|
+
overlap = line_tokens & sk
|
|
1823
|
+
n_overlap = len(overlap)
|
|
1824
|
+
ratio = n_overlap / len(line_tokens) if line_tokens else 0
|
|
1825
|
+
if n_overlap >= min_keyword_overlap and ratio >= min_overlap_ratio:
|
|
1826
|
+
citations.append(idx + 1)
|
|
1827
|
+
|
|
1828
|
+
if citations:
|
|
1829
|
+
# Avoid duplicates if LLM already cited [WN] in the line
|
|
1830
|
+
existing = set()
|
|
1831
|
+
for m in re.finditer(r"\[W(\d+)\]", line):
|
|
1832
|
+
existing.add(int(m.group(1)))
|
|
1833
|
+
new_cites = [c for c in citations if c not in existing]
|
|
1834
|
+
if new_cites:
|
|
1835
|
+
cite_str = " " + " ".join(f"[W{n}]" for n in new_cites)
|
|
1836
|
+
annotated.append(f"{line}{cite_str}")
|
|
1837
|
+
else:
|
|
1838
|
+
annotated.append(line)
|
|
1839
|
+
else:
|
|
1840
|
+
annotated.append(line)
|
|
1841
|
+
|
|
1842
|
+
return "\n".join(annotated)
|