web-forager 0.0.0-stage → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/.claude-plugin/marketplace.json +16 -0
  2. package/.claude-plugin/plugin.json +20 -0
  3. package/.clinerules/web-forager.md +85 -0
  4. package/.codebuddy-plugin/marketplace.json +14 -0
  5. package/.codebuddy-plugin/plugin.json +25 -0
  6. package/.codex-plugin/marketplace.json +12 -0
  7. package/.codex-plugin/mcp.json +20 -0
  8. package/.codex-plugin/plugin.json +15 -0
  9. package/.continue/rules/web-forager.md +90 -0
  10. package/.cursor/rules/web-forager.mdc +80 -0
  11. package/.dsh-plugin/index.js +11 -0
  12. package/.dsh-plugin/plugin.json +5 -0
  13. package/.github/copilot-instructions.md +85 -0
  14. package/.mcp.json +12 -0
  15. package/.roo/rules/web-forager.md +85 -0
  16. package/.windsurf/rules/web-forager.md +90 -0
  17. package/AGENTS.md +112 -0
  18. package/CHANGELOG.md +454 -0
  19. package/LICENSE +21 -0
  20. package/README.md +477 -3
  21. package/README_zh.md +477 -0
  22. package/SKILL.md +351 -0
  23. package/THIRD_PARTY_LICENSES.txt +35 -0
  24. package/browser/bridges/playwright-bridge.js +375 -0
  25. package/browser/launch_chrome.py +144 -0
  26. package/browser/lib/backend.js +302 -0
  27. package/browser/lib/bili-api.js +79 -0
  28. package/browser/lib/bridge.js +83 -0
  29. package/browser/lib/context.js +60 -0
  30. package/browser/lib/doctor.js +97 -0
  31. package/browser/lib/errors.js +17 -0
  32. package/browser/lib/netguard.js +149 -0
  33. package/browser/lib/playbooks.js +30 -0
  34. package/browser/lib/record.js +109 -0
  35. package/browser/lib/robots.js +195 -0
  36. package/browser/lib/safety.js +115 -0
  37. package/browser/lib/selectors.js +80 -0
  38. package/browser/lib/session.js +116 -0
  39. package/browser/lib/snapshot.js +229 -0
  40. package/browser/lib/untrusted.js +56 -0
  41. package/browser/lib/workspace.js +40 -0
  42. package/browser/mcp_server.js +272 -0
  43. package/browser/package.json +40 -0
  44. package/browser/playbooks/bili-chain.js +58 -0
  45. package/browser/playbooks/bili-danmaku.js +102 -0
  46. package/browser/playbooks/bili-feed.js +66 -0
  47. package/browser/playbooks/bili-video.js +38 -0
  48. package/browser/playbooks/search-dig.js +105 -0
  49. package/browser/playbooks/shared.js +138 -0
  50. package/browser/playbooks/wechat-dig.js +78 -0
  51. package/browser/playbooks/xhs-dig.js +89 -0
  52. package/browser/playbooks/zhihu-dig.js +89 -0
  53. package/browser/run.js +432 -0
  54. package/commands/browse.md +53 -0
  55. package/commands/fetch.md +29 -0
  56. package/commands/map.md +24 -0
  57. package/commands/scholar.md +28 -0
  58. package/commands/surf.md +28 -0
  59. package/cordis.patch.yml +19 -0
  60. package/docs/ACKNOWLEDGEMENTS.md +15 -0
  61. package/docs/ACKNOWLEDGEMENTS_zh.md +15 -0
  62. package/docs/DEPENDENCIES.md +40 -0
  63. package/docs/INSTALL_COMMANDS.md +113 -0
  64. package/docs/INSTALL_COMMANDS_zh.md +113 -0
  65. package/docs/REFERENCE.md +308 -0
  66. package/docs/REFERENCE_zh.md +309 -0
  67. package/index.js +7 -0
  68. package/install.py +254 -0
  69. package/manifest.json +97 -0
  70. package/mcp_server.py +597 -0
  71. package/package.json +80 -6
  72. package/requirements.txt +12 -0
  73. package/run.py +852 -0
  74. package/scripts/__init__.py +1 -0
  75. package/scripts/bilibili.py +874 -0
  76. package/scripts/documents.py +437 -0
  77. package/scripts/engine.py +4220 -0
  78. package/scripts/feeds.py +180 -0
  79. package/scripts/pdf.py +135 -0
  80. package/scripts/scholar.py +354 -0
  81. package/scripts/sitemap.py +180 -0
  82. package/scripts/untrusted.py +95 -0
@@ -0,0 +1,4220 @@
1
+ """web-forager — clean-room web fetching and cleaning engine.
2
+
3
+ Design notes
4
+ ------------
5
+ This module is an original implementation written from scratch. It borrows no
6
+ source code from Crawlee, Scrapy, or any other crawling framework. Only general
7
+ ideas are reused (politeness delay, a robots-policy cache, an extraction
8
+ pipeline); every line below was written for this plugin.
9
+
10
+ Permitted third-party packages
11
+ ------------------------------
12
+ requests Apache-2.0
13
+ beautifulsoup4 MIT
14
+ trafilatura Apache-2.0 (preferred extractor, optional at runtime)
15
+ playwright Apache-2.0 (optional, dynamic rendering only)
16
+
17
+ No GPL / AGPL dependency is used.
18
+
19
+ Compliance
20
+ ----------
21
+ * robots.txt is always consulted before fetching a target page (RFC 9309
22
+ matching: longest rule wins, ``*`` wildcard, ``$`` end anchor, UA groups).
23
+ * Every request carries an explicit information-gathering User-Agent.
24
+ * Requests are rate-limited per host; the default delay is one second.
25
+ """
26
+
27
+ from __future__ import annotations
28
+
29
+ import base64
30
+ import dataclasses
31
+ import html as html_lib
32
+ import ipaddress
33
+ import json
34
+ import os
35
+ import re
36
+ import socket
37
+ import sys
38
+ import threading
39
+ import time
40
+ import urllib.parse
41
+ import xml.etree.ElementTree as ET
42
+ from concurrent.futures import ThreadPoolExecutor
43
+ from dataclasses import dataclass, field
44
+ from datetime import datetime, timezone
45
+ from pathlib import Path
46
+ from typing import Any, Iterable, Iterator, Optional
47
+
48
+ import requests
49
+ from bs4 import BeautifulSoup
50
+ from bs4.element import NavigableString, Tag
51
+
52
+ from . import untrusted
53
+
54
+ __version__ = "1.3.0"
55
+
56
+ DEFAULT_USER_AGENT = (
57
+ f"WebForager/{__version__} (Information Gathering; "
58
+ "+https://github.com/JiamingShi-JsPhys/web-forager)"
59
+ )
60
+
61
+ DISCLAIMER = (
62
+ "本插件仅供学习与信息收集使用。使用者需遵守目标网站的 Terms of Service 及相关版权法律。"
63
+ "禁止用于恶意并发攻击。严禁收集、聚合或推断特定自然人的个人信息,或将其用于人肉搜索、骚扰、跟踪等侵害他人隐私的活动;此类用法产生的法律责任由使用者自行承担。"
64
+ )
65
+
66
+ # ---------------------------------------------------------------------------
67
+ # Errors
68
+ # ---------------------------------------------------------------------------
69
+
70
+
71
+ class WebForagerError(Exception):
72
+ """Base class for all engine errors."""
73
+
74
+
75
+ class InvalidURLError(WebForagerError):
76
+ """The supplied URL is missing, malformed, or uses a forbidden scheme."""
77
+
78
+
79
+ class RobotsDenied(WebForagerError):
80
+ """robots.txt forbids fetching this URL."""
81
+
82
+
83
+ class FetchError(WebForagerError):
84
+ """Network failure, non-success status, or unsupported content type."""
85
+
86
+
87
+ class ExtractionError(WebForagerError):
88
+ """No readable content could be extracted from the document."""
89
+
90
+
91
+ class VerificationRequired(WebForagerError):
92
+ """The site answered with a verification or anti-bot page, not content.
93
+
94
+ HTTP mode cannot pass these interstitials: doing so would mean solving a
95
+ challenge, which this tool does not do. Browser mode may succeed when the
96
+ page is readable in a real profile that has already passed verification.
97
+ """
98
+
99
+
100
+ class RenderUnavailable(WebForagerError):
101
+ """Dynamic rendering was requested but Playwright is not usable."""
102
+
103
+
104
+ class PdfUnavailable(WebForagerError):
105
+ """PDF support was requested but pypdf is not usable."""
106
+
107
+
108
+ class PdfExtractionError(WebForagerError):
109
+ """The PDF could not be parsed or contained no text."""
110
+
111
+
112
+ class ProhibitedQuery(WebForagerError):
113
+ """The query asks to identify a private person and is refused."""
114
+
115
+
116
+ def parse_xml(text: str) -> ET.Element:
117
+ """Parse untrusted XML without XXE or entity-expansion risk.
118
+
119
+ ``defusedxml`` is used when it happens to be installed; otherwise the
120
+ input is rejected outright if it declares a DOCTYPE or an ENTITY, which
121
+ blocks both external entities and billion-laughs expansion.
122
+ """
123
+
124
+ if "<!ENTITY" in text:
125
+ raise FetchError("refusing XML with ENTITY declarations")
126
+ # An external DTD is harmless here (defusedxml forbids external entities;
127
+ # stdlib ElementTree never fetches the DTD), but an internal subset can
128
+ # declare entities, so only a bare DOCTYPE is stripped.
129
+ doctype = re.search(r"<!DOCTYPE[^>]*>", text, re.I | re.S)
130
+ if doctype:
131
+ if "[" in doctype.group(0):
132
+ raise FetchError("refusing XML with an internal DTD subset")
133
+ text = text.replace(doctype.group(0), "", 1)
134
+ try:
135
+ import defusedxml.ElementTree as DefusedET # type: ignore
136
+ except ImportError:
137
+ DefusedET = None # type: ignore[assignment]
138
+ if DefusedET is not None:
139
+ try:
140
+ return DefusedET.fromstring(text)
141
+ except Exception as exc:
142
+ raise FetchError(f"refusing unsafe XML: {exc}") from exc
143
+ try:
144
+ return ET.fromstring(text)
145
+ except ET.ParseError as exc:
146
+ raise FetchError(f"cannot parse XML: {exc}") from exc
147
+
148
+
149
+ # ---------------------------------------------------------------------------
150
+ # Configuration
151
+ # ---------------------------------------------------------------------------
152
+
153
+
154
+ def _allow_private_from_env() -> bool:
155
+ """Local fixtures and trusted intranet tools set WEB_FORAGER_ALLOW_PRIVATE=1."""
156
+
157
+ return os.environ.get("WEB_FORAGER_ALLOW_PRIVATE", "").strip().lower() in (
158
+ "1",
159
+ "true",
160
+ "yes",
161
+ "on",
162
+ )
163
+
164
+
165
+ @dataclass
166
+ class FetchConfig:
167
+ """Runtime knobs for one CLI invocation."""
168
+
169
+ user_agent: str = DEFAULT_USER_AGENT
170
+ timeout: float = 20.0
171
+ retries: int = 2
172
+ delay: float = 1.0
173
+ max_bytes: int = 8 * 1024 * 1024
174
+ max_chars: int = 20_000
175
+ respect_robots: bool = True
176
+ # respect: refuse a disallowed path; warn: read it and mark the record;
177
+ # ignore: do not consult robots.txt (same as respect_robots=False).
178
+ robots_policy: str = "respect"
179
+ render: str = "auto" # auto | never | always
180
+ include_tables: bool = True
181
+ keep_links: bool = False
182
+ include_images: bool = False
183
+ max_pages: int = 3
184
+ num_results: int = 8
185
+ verbose: bool = False
186
+ extract_links: bool = False
187
+ max_links: int = 20
188
+ follow: str = "same-site" # same-site | all | none
189
+ search_engine: str = "auto"
190
+ search_pages: int = 1
191
+ # Engines are different hosts, so they may run at once without breaking the
192
+ # per-origin delay. Set False for a strictly serial diagnostic run.
193
+ parallel_engines: bool = True
194
+ cache: bool = False
195
+ cache_ttl: int = 3600
196
+ cache_dir: str = ""
197
+ chunk_size: int = 0
198
+ wait_for: str = ""
199
+ scroll: int = 0
200
+ proxy: str = "" # http(s) proxy for every request; "" uses the environment
201
+ # Refuse private/loopback/link-local targets unless the caller opts in.
202
+ allow_private: bool = field(default_factory=_allow_private_from_env)
203
+ # off | mark | strict: what to do with instruction-shaped page text.
204
+ input_gate: str = "mark"
205
+
206
+ def validate(self) -> None:
207
+ if self.render not in ("auto", "never", "always"):
208
+ raise ValueError("render must be auto, never or always")
209
+ if self.robots_policy not in ("respect", "warn", "ignore"):
210
+ raise ValueError("robots_policy must be respect, warn or ignore")
211
+ if self.input_gate not in ("off", "mark", "strict"):
212
+ raise ValueError("input_gate must be off, mark or strict")
213
+ if self.follow not in ("same-site", "all", "none"):
214
+ raise ValueError("follow must be same-site, all or none")
215
+ if self.search_engine not in SEARCH_ENGINES:
216
+ raise ValueError(
217
+ "search_engine must be one of: " + ", ".join(SEARCH_ENGINES)
218
+ )
219
+ if self.timeout <= 0:
220
+ raise ValueError("timeout must be positive")
221
+ if self.delay < 0:
222
+ raise ValueError("delay must be >= 0")
223
+ if self.max_chars < 0:
224
+ raise ValueError("max_chars must be >= 0")
225
+ if self.max_pages < 0:
226
+ raise ValueError("max_pages must be >= 0")
227
+ if self.max_links < 1:
228
+ raise ValueError("max_links must be >= 1")
229
+ if self.search_pages < 1:
230
+ raise ValueError("search_pages must be >= 1")
231
+ if self.cache_ttl < 0:
232
+ raise ValueError("cache_ttl must be >= 0")
233
+ if self.chunk_size < 0:
234
+ raise ValueError("chunk_size must be >= 0")
235
+ if self.scroll < 0:
236
+ raise ValueError("scroll must be >= 0")
237
+ if self.proxy and "://" in self.proxy:
238
+ scheme = self.proxy.split("://", 1)[0].lower()
239
+ if scheme not in ("http", "https", "socks5", "socks5h"):
240
+ raise ValueError("proxy must be an http(s) or socks5 URL")
241
+ if not self.user_agent or len(self.user_agent) < 8:
242
+ raise ValueError("user_agent must be a descriptive string")
243
+
244
+
245
+ def log(message: str, config: Optional[FetchConfig] = None) -> None:
246
+ """Write a diagnostic line to stderr (never to stdout)."""
247
+
248
+ if config is not None and not config.verbose:
249
+ return
250
+ print(f"[web-forager] {message}", file=sys.stderr, flush=True)
251
+
252
+
253
+ def utc_now() -> str:
254
+ return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
255
+
256
+
257
+ # ---------------------------------------------------------------------------
258
+ # URL helpers
259
+ # ---------------------------------------------------------------------------
260
+
261
+ _ALLOWED_SCHEMES = ("http", "https")
262
+ _FORBIDDEN_PREFIXES = ("javascript:", "data:", "file:", "mailto:", "tel:", "ftp:")
263
+
264
+
265
+ def normalize_url(url: str, base: Optional[str] = None) -> str:
266
+ """Return a canonical absolute http(s) URL or raise InvalidURLError.
267
+
268
+ Fragments are dropped, whitespace trimmed, and scheme-less input such as
269
+ ``example.com/page`` is promoted to https. ``base`` resolves relative
270
+ links found while parsing a page.
271
+ """
272
+
273
+ if url is None:
274
+ raise InvalidURLError("URL is required")
275
+ raw = str(url).strip().strip("<>").strip()
276
+ if not raw:
277
+ raise InvalidURLError("URL is empty")
278
+ lowered = raw.lower()
279
+ if lowered.startswith(_FORBIDDEN_PREFIXES):
280
+ raise InvalidURLError(f"unsupported URL scheme: {raw.split(':', 1)[0]}")
281
+
282
+ if base:
283
+ raw = urllib.parse.urljoin(base, raw)
284
+
285
+ parts = urllib.parse.urlsplit(raw)
286
+ if not parts.scheme:
287
+ raw = "https://" + raw.lstrip("/")
288
+ parts = urllib.parse.urlsplit(raw)
289
+
290
+ scheme = parts.scheme.lower()
291
+ if scheme not in _ALLOWED_SCHEMES:
292
+ raise InvalidURLError(f"unsupported URL scheme: {scheme}")
293
+ if not parts.netloc:
294
+ raise InvalidURLError(f"URL has no host: {raw}")
295
+
296
+ # Normalise default ports and strip fragments.
297
+ netloc = parts.netloc
298
+ if (scheme == "http" and netloc.endswith(":80")) or (
299
+ scheme == "https" and netloc.endswith(":443")
300
+ ):
301
+ netloc = netloc.rsplit(":", 1)[0]
302
+
303
+ path = parts.path or "/"
304
+ rebuilt = urllib.parse.urlunsplit((scheme, netloc, path, parts.query, ""))
305
+ return rebuilt
306
+
307
+
308
+ def origin_of(url: str) -> str:
309
+ """Return ``scheme://host[:port]`` for rate limiting and robots lookup."""
310
+
311
+ parts = urllib.parse.urlsplit(url)
312
+ return f"{parts.scheme.lower()}://{parts.netloc.lower()}"
313
+
314
+
315
+ def absolute_url(href: Optional[str], base: Optional[str]) -> str:
316
+ """Best-effort absolute link; returns '' when it cannot be resolved."""
317
+
318
+ if not href:
319
+ return ""
320
+ text = str(href).strip()
321
+ if not text:
322
+ return ""
323
+ lowered = text.lower()
324
+ if lowered.startswith(_FORBIDDEN_PREFIXES):
325
+ return ""
326
+ # Without a base there is nothing to resolve a relative link against.
327
+ if base is None and not (text.startswith("//") or re.match(r"^https?://", text, re.I)):
328
+ return ""
329
+ try:
330
+ return normalize_url(text, base=base)
331
+ except InvalidURLError:
332
+ return ""
333
+
334
+
335
+ # ---------------------------------------------------------------------------
336
+ # Private-address guard (SSRF)
337
+ # ---------------------------------------------------------------------------
338
+
339
+ # Loopback, RFC 1918, link-local, CGNAT, documentation, multicast and reserved
340
+ # ranges. ``ipaddress`` flags cover most of them; the explicit list keeps the
341
+ # same answer on every supported Python version.
342
+ _BLOCKED_NETWORKS = tuple(
343
+ ipaddress.ip_network(cidr)
344
+ for cidr in (
345
+ "0.0.0.0/8",
346
+ "10.0.0.0/8",
347
+ "100.64.0.0/10",
348
+ "127.0.0.0/8",
349
+ "169.254.0.0/16",
350
+ "172.16.0.0/12",
351
+ "192.0.0.0/24",
352
+ "192.0.2.0/24",
353
+ "192.168.0.0/16",
354
+ "198.18.0.0/15",
355
+ "198.51.100.0/24",
356
+ "203.0.113.0/24",
357
+ "240.0.0.0/4",
358
+ "255.255.255.255/32",
359
+ "::1/128",
360
+ "64:ff9b::/96",
361
+ "100::/64",
362
+ "2001:db8::/32",
363
+ "fc00::/7",
364
+ "fe80::/10",
365
+ "ff00::/8",
366
+ )
367
+ )
368
+
369
+
370
+ class UnsafeAddress(InvalidURLError):
371
+ """The URL points at a loopback, private or link-local address."""
372
+
373
+
374
+ def _blocked_ip(ip: Any) -> bool:
375
+ if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None:
376
+ ip = ip.ipv4_mapped
377
+ if any(ip in network for network in _BLOCKED_NETWORKS if network.version == ip.version):
378
+ return True
379
+ return bool(
380
+ ip.is_loopback
381
+ or ip.is_link_local
382
+ or ip.is_private
383
+ or ip.is_multicast
384
+ or ip.is_reserved
385
+ or ip.is_unspecified
386
+ )
387
+
388
+
389
+ def _host_is_local_name(host: str) -> bool:
390
+ host = host.rstrip(".").lower()
391
+ return (
392
+ host == "localhost"
393
+ or host.endswith(".localhost")
394
+ or host.endswith(".local")
395
+ or host.endswith(".internal")
396
+ or host == "metadata.google.internal"
397
+ )
398
+
399
+
400
+ def assert_public_url(url: str, *, allow_private: bool = False) -> None:
401
+ """Refuse URLs that resolve to this machine, the LAN or a metadata service.
402
+
403
+ The host as written and every address DNS returns are checked, so decimal
404
+ IPv4 spellings and IPv6-mapped IPv4 are caught as well. The guard runs
405
+ before each request and on every redirect hop. A host that does not resolve
406
+ is left to the HTTP client, which fails on its own.
407
+
408
+ ``allow_private`` exists for local fixtures and trusted intranet tools; it
409
+ is off by default and the CLI exposes it as ``--allow-private``.
410
+ """
411
+
412
+ if allow_private:
413
+ return
414
+ parts = urllib.parse.urlsplit(url)
415
+ host = parts.hostname or ""
416
+ if not host:
417
+ raise UnsafeAddress(f"URL has no host: {url}")
418
+ if _host_is_local_name(host):
419
+ raise UnsafeAddress(f"refusing a local host: {host}")
420
+ try:
421
+ port = parts.port or (443 if parts.scheme.lower() == "https" else 80)
422
+ except ValueError as exc:
423
+ raise UnsafeAddress(f"invalid port in {url}") from exc
424
+
425
+ try:
426
+ literal = ipaddress.ip_address(host.strip("[]").split("%", 1)[0])
427
+ except ValueError:
428
+ literal = None
429
+ if literal is not None:
430
+ if _blocked_ip(literal):
431
+ raise UnsafeAddress(f"refusing a private address: {host}")
432
+ return
433
+
434
+ try:
435
+ infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
436
+ except OSError:
437
+ return # unresolvable: the HTTP client reports the failure
438
+ for info in infos:
439
+ address = str(info[4][0])
440
+ try:
441
+ ip = ipaddress.ip_address(address.split("%", 1)[0])
442
+ except ValueError:
443
+ continue
444
+ if _blocked_ip(ip):
445
+ raise UnsafeAddress(f"refusing {host}: it resolves to {address}")
446
+
447
+
448
+ def _guarded_request(
449
+ getter: Any,
450
+ url: str,
451
+ *,
452
+ allow_private: bool = False,
453
+ max_redirects: int = 5,
454
+ **kwargs: Any,
455
+ ) -> Any:
456
+ """GET ``url`` and follow redirects one hop at a time, checking each target.
457
+
458
+ ``requests`` follows redirects internally, which would let a public URL
459
+ bounce the client to ``127.0.0.1`` or a metadata endpoint without the guard
460
+ ever seeing it. Every hop is therefore resolved and checked here.
461
+ """
462
+
463
+ current = url
464
+ for _ in range(max_redirects + 1):
465
+ assert_public_url(current, allow_private=allow_private)
466
+ response = getter(current, allow_redirects=False, **kwargs)
467
+ location = response.headers.get("Location") if response.headers else None
468
+ if response.status_code in (301, 302, 303, 307, 308) and location:
469
+ current = normalize_url(urllib.parse.urljoin(current, location))
470
+ response.close()
471
+ continue
472
+ return response
473
+ raise FetchError(f"too many redirects for {url}")
474
+
475
+
476
+ # ---------------------------------------------------------------------------
477
+ # robots.txt (RFC 9309 compatible subset, written from scratch)
478
+ # ---------------------------------------------------------------------------
479
+
480
+ _ROBOTS_CHARSET_RE = re.compile(rb"charset\s*=\s*[\"']?([A-Za-z0-9._:-]+)", re.I)
481
+
482
+
483
+ @dataclass(frozen=True)
484
+ class RobotsRule:
485
+ allow: bool
486
+ pattern: str
487
+ regex: re.Pattern
488
+ specificity: int
489
+
490
+
491
+ class RobotsPolicy:
492
+ """An immutable parsed robots.txt policy for one user agent."""
493
+
494
+ def __init__(
495
+ self,
496
+ rules: Iterable[RobotsRule] = (),
497
+ crawl_delay: Optional[float] = None,
498
+ *,
499
+ allow_all: bool = False,
500
+ deny_all: bool = False,
501
+ source: str = "",
502
+ note: str = "",
503
+ ) -> None:
504
+ self.rules = tuple(rules)
505
+ self.crawl_delay = crawl_delay
506
+ self.allow_all = allow_all
507
+ self.deny_all = deny_all
508
+ self.source = source
509
+ self.note = note
510
+
511
+ @classmethod
512
+ def allowing_all(cls, source: str = "", note: str = "") -> "RobotsPolicy":
513
+ return cls(allow_all=True, source=source, note=note)
514
+
515
+ @classmethod
516
+ def denying_all(cls, source: str = "", note: str = "") -> "RobotsPolicy":
517
+ return cls(deny_all=True, source=source, note=note)
518
+
519
+ def allowed(self, url: str) -> bool:
520
+ if self.deny_all:
521
+ return False
522
+ if self.allow_all:
523
+ return True
524
+ if not self.rules:
525
+ return True
526
+
527
+ path = _robots_path(url)
528
+ winner: Optional[RobotsRule] = None
529
+ for rule in self.rules:
530
+ if not rule.regex.match(path):
531
+ continue
532
+ if winner is None or rule.specificity > winner.specificity:
533
+ winner = rule
534
+ elif rule.specificity == winner.specificity and rule.allow and not winner.allow:
535
+ # RFC 9309: ties are resolved in favour of Allow.
536
+ winner = rule
537
+ return True if winner is None else winner.allow
538
+
539
+
540
+ def _robots_path(url: str) -> str:
541
+ parts = urllib.parse.urlsplit(url)
542
+ path = parts.path or "/"
543
+ if parts.query:
544
+ path += "?" + parts.query
545
+ # robots.txt patterns are conventionally unencoded; compare unencoded text.
546
+ try:
547
+ path = urllib.parse.unquote(path)
548
+ except Exception:
549
+ pass
550
+ return path
551
+
552
+
553
+ def _compile_rule(pattern: str) -> re.Pattern:
554
+ """Translate a robots.txt path pattern into a regular expression."""
555
+
556
+ anchored = pattern.endswith("$")
557
+ core = pattern[:-1] if anchored else pattern
558
+ if not core.startswith(("/", "*")):
559
+ core = "/" + core
560
+ escaped = re.escape(core).replace(r"\*", ".*")
561
+ return re.compile("^" + escaped + ("$" if anchored else ""), re.S)
562
+
563
+
564
+ def _parse_robots_groups(text: str) -> list[dict[str, Any]]:
565
+ """Split robots.txt into groups of {agents, rules, delay}."""
566
+
567
+ groups: list[dict[str, Any]] = []
568
+ current: Optional[dict[str, Any]] = None
569
+
570
+ for raw_line in text.splitlines():
571
+ line = raw_line.split("#", 1)[0].strip()
572
+ if not line or ":" not in line:
573
+ continue
574
+ key, _, value = line.partition(":")
575
+ key = key.strip().lower()
576
+ value = value.strip()
577
+
578
+ if key == "user-agent":
579
+ # A user-agent line after rules starts a new group.
580
+ if current is None or current["rules"]:
581
+ current = {"agents": [], "rules": [], "delay": None}
582
+ groups.append(current)
583
+ current["agents"].append(value.lower())
584
+ elif key in ("allow", "disallow"):
585
+ if current is None:
586
+ continue
587
+ if key == "disallow" and value == "":
588
+ # An empty Disallow means "no restriction"; ignore it.
589
+ continue
590
+ current["rules"].append((key == "allow", value))
591
+ elif key == "crawl-delay" and current is not None:
592
+ try:
593
+ delay = float(value)
594
+ except ValueError:
595
+ continue
596
+ if delay >= 0:
597
+ current["delay"] = delay
598
+
599
+ return groups
600
+
601
+
602
+ def _agent_match_score(agent_token: str, user_agent: str) -> int:
603
+ """Return a match score for a robots.txt user-agent token.
604
+
605
+ ``*`` scores 0; a product token found in the request UA scores its length.
606
+ ``-1`` means no match.
607
+ """
608
+
609
+ token = agent_token.strip().lower()
610
+ if not token:
611
+ return -1
612
+ if token == "*":
613
+ return 0
614
+ token = token.split("/", 1)[0].strip()
615
+ if token and token in user_agent.lower():
616
+ return len(token)
617
+ return -1
618
+
619
+
620
+ def parse_robots(text: str, user_agent: str, source: str = "") -> RobotsPolicy:
621
+ """Parse robots.txt text and return the policy for ``user_agent``."""
622
+
623
+ groups = _parse_robots_groups(text)
624
+ best_score = -1
625
+ selected: list[dict[str, Any]] = []
626
+
627
+ for group in groups:
628
+ score = max((_agent_match_score(a, user_agent) for a in group["agents"]), default=-1)
629
+ if score < 0:
630
+ continue
631
+ if score > best_score:
632
+ best_score = score
633
+ selected = [group]
634
+ elif score == best_score:
635
+ selected.append(group)
636
+
637
+ if not selected:
638
+ return RobotsPolicy.allowing_all(source=source, note="no matching user-agent group")
639
+
640
+ rules: list[RobotsRule] = []
641
+ delay: Optional[float] = None
642
+ for group in selected:
643
+ if group["delay"] is not None:
644
+ delay = group["delay"] if delay is None else max(delay, group["delay"])
645
+ for allow, pattern in group["rules"]:
646
+ rules.append(
647
+ RobotsRule(
648
+ allow=allow,
649
+ pattern=pattern,
650
+ regex=_compile_rule(pattern),
651
+ specificity=len(pattern.replace("*", "")),
652
+ )
653
+ )
654
+
655
+ return RobotsPolicy(rules=rules, crawl_delay=delay, source=source)
656
+
657
+
658
+ class RobotsCache:
659
+ """Fetch robots.txt once per origin and cache the parsed policy."""
660
+
661
+ def __init__(self, config: FetchConfig, ttl: float = 3600.0) -> None:
662
+ self.config = config
663
+ self.ttl = ttl
664
+ self._cache: dict[str, tuple[float, RobotsPolicy]] = {}
665
+ self._lock = threading.Lock()
666
+
667
+ def policy_for(self, url: str) -> RobotsPolicy:
668
+ origin = origin_of(url)
669
+ now = time.monotonic()
670
+ with self._lock:
671
+ hit = self._cache.get(origin)
672
+ if hit and hit[0] > now:
673
+ return hit[1]
674
+
675
+ policy = self._download(origin)
676
+ with self._lock:
677
+ self._cache[origin] = (now + self.ttl, policy)
678
+ return policy
679
+
680
+ def _download(self, origin: str) -> RobotsPolicy:
681
+ robots_url = origin + "/robots.txt"
682
+ headers = {
683
+ "User-Agent": self.config.user_agent,
684
+ "Accept": "text/plain,*/*;q=0.5",
685
+ }
686
+ try:
687
+ response = _guarded_request(
688
+ requests.get,
689
+ robots_url,
690
+ allow_private=self.config.allow_private,
691
+ headers=headers,
692
+ timeout=min(max(self.config.timeout, 1.0), 10.0),
693
+ )
694
+ except requests.RequestException as exc:
695
+ # RFC 9309 §2.3.1: an unreachable robots.txt is treated as a
696
+ # server error, so we take the conservative route and deny.
697
+ return RobotsPolicy.denying_all(
698
+ source=robots_url,
699
+ note=f"robots.txt unreachable ({exc.__class__.__name__}); denied conservatively",
700
+ )
701
+
702
+ status = response.status_code
703
+ if status == 200:
704
+ raw = response.content[:512_000]
705
+ text = _decode_robots_bytes(raw, response)
706
+ return parse_robots(text, self.config.user_agent, source=robots_url)
707
+ if status in (401, 403):
708
+ return RobotsPolicy.denying_all(source=robots_url, note=f"HTTP {status} on robots.txt")
709
+ if 500 <= status < 600:
710
+ return RobotsPolicy.denying_all(
711
+ source=robots_url,
712
+ note=(
713
+ f"HTTP {status} on robots.txt; denied per RFC 9309. The site's "
714
+ "robots.txt is broken, so no page on it can be read. Contact the "
715
+ "site owner, or use --ignore-robots only for resources you are "
716
+ "authorised to access."
717
+ ),
718
+ )
719
+ # 404/410 and other 4xx mean "no restrictions".
720
+ return RobotsPolicy.allowing_all(source=robots_url, note=f"HTTP {status} on robots.txt")
721
+
722
+
723
+ def _decode_robots_bytes(raw: bytes, response: Any) -> str:
724
+ encoding = None
725
+ try:
726
+ match = _ROBOTS_CHARSET_RE.search(raw[:512])
727
+ if match:
728
+ encoding = match.group(1).decode("ascii", "ignore")
729
+ except Exception:
730
+ encoding = None
731
+ if not encoding:
732
+ content_type = ""
733
+ try:
734
+ content_type = response.headers.get("Content-Type", "")
735
+ except Exception:
736
+ content_type = ""
737
+ match = re.search(r"charset=([A-Za-z0-9._:-]+)", content_type, re.I)
738
+ if match:
739
+ encoding = match.group(1)
740
+ if not encoding:
741
+ encoding = getattr(response, "encoding", None) or "utf-8"
742
+ try:
743
+ return raw.decode(encoding, errors="replace")
744
+ except (LookupError, TypeError):
745
+ return raw.decode("utf-8", errors="replace")
746
+
747
+
748
+ # ---------------------------------------------------------------------------
749
+ # Politeness
750
+ # ---------------------------------------------------------------------------
751
+
752
+
753
+ class RateLimiter:
754
+ """Per-origin politeness delay, capped so robots.txt cannot stall a run."""
755
+
756
+ MAX_EXTRA_DELAY = 30.0
757
+
758
+ def __init__(self, delay: float) -> None:
759
+ self.delay = max(0.0, float(delay))
760
+ self._last: dict[str, float] = {}
761
+ self._lock = threading.Lock()
762
+
763
+ def wait(self, url: str, extra_delay: float = 0.0) -> None:
764
+ target = max(self.delay, min(extra_delay, self.MAX_EXTRA_DELAY))
765
+ if target <= 0:
766
+ return
767
+ origin = origin_of(url)
768
+ with self._lock:
769
+ now = time.monotonic()
770
+ last = self._last.get(origin, 0.0)
771
+ pause = max(0.0, last + target - now)
772
+ self._last[origin] = now + pause
773
+ if pause > 0:
774
+ time.sleep(pause)
775
+
776
+
777
+ # ---------------------------------------------------------------------------
778
+ # HTTP fetching
779
+ # ---------------------------------------------------------------------------
780
+
781
+ _HTML_CONTENT_TYPES = (
782
+ "text/html",
783
+ "application/xhtml+xml",
784
+ "application/xml",
785
+ "text/xml",
786
+ "application/rss+xml",
787
+ "application/atom+xml",
788
+ )
789
+ _TEXT_CONTENT_TYPES = ("text/plain", "text/markdown", "text/csv")
790
+ _JSON_CONTENT_TYPES = ("application/json", "application/ld+json")
791
+ _PDF_CONTENT_TYPES = ("application/pdf",)
792
+ _DOCUMENT_CONTENT_TYPES = (
793
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
794
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
795
+ "application/epub+zip",
796
+ )
797
+ _DOCUMENT_EXTENSIONS = {
798
+ ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
799
+ ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
800
+ ".epub": "application/epub+zip",
801
+ ".csv": "text/csv",
802
+ }
803
+ _FEED_CONTENT_TYPES = ("application/rss+xml", "application/atom+xml", "application/xml", "text/xml")
804
+
805
+
806
+ @dataclass
807
+ class HttpResponse:
808
+ url: str
809
+ final_url: str
810
+ status: int
811
+ headers: dict[str, str]
812
+ content: bytes
813
+ text: str
814
+ encoding: str
815
+ content_type: str
816
+ from_cache: bool = False
817
+
818
+
819
+ def _cache_root(config: FetchConfig) -> Path:
820
+ if config.cache_dir:
821
+ return Path(config.cache_dir).expanduser()
822
+ return Path.home() / ".web-forager" / "cache"
823
+
824
+
825
+ def _cache_key(url: str) -> str:
826
+ import hashlib
827
+
828
+ return hashlib.sha256(url.encode("utf-8")).hexdigest()
829
+
830
+
831
+ def decode_js_redirect(text: str) -> str:
832
+ """Read the target URL out of a search engine's ``/link`` interstitial.
833
+
834
+ Two shapes exist. One writes the address literally
835
+ (``window.location.replace('https://...')``). The other — used by
836
+ weixin.sogou.com — assembles it from ``url += '...'`` chunks so that the
837
+ address never appears in the HTML as a whole; concatenating the chunks
838
+ recovers it without running any JavaScript.
839
+ """
840
+
841
+ if not text:
842
+ return ""
843
+ chunks = re.findall(r"""url\s*\+=\s*['"]([^'"]*)['"]""", text)
844
+ if chunks:
845
+ joined = "".join(chunks)
846
+ if joined.startswith("http"):
847
+ return joined
848
+ match = re.search(r"""window\.location\.replace\(\s*['"]([^'"]+)""", text)
849
+ if not match:
850
+ match = re.search(r"""URL\s*=\s*['"]([^'"]+)""", text)
851
+ return match.group(1) if match else ""
852
+
853
+
854
+ class HttpFetcher:
855
+ """A polite, robots-aware HTTP client built on requests."""
856
+
857
+ def __init__(self, config: FetchConfig) -> None:
858
+ self.config = config
859
+ self.session = requests.Session()
860
+ self.session.max_redirects = 5
861
+ self.session.headers.update(
862
+ {
863
+ "User-Agent": config.user_agent,
864
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,"
865
+ "text/plain;q=0.8,application/json;q=0.7,*/*;q=0.5",
866
+ "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
867
+ "Accept-Encoding": "gzip, deflate",
868
+ }
869
+ )
870
+ if config.proxy:
871
+ # An explicit proxy wins over the environment; requests itself still
872
+ # honours HTTPS_PROXY / ALL_PROXY when this is empty.
873
+ self.session.proxies.update({"http": config.proxy, "https": config.proxy})
874
+ self.robots = RobotsCache(config)
875
+ self.limiter = RateLimiter(config.delay)
876
+
877
+ def close(self) -> None:
878
+ try:
879
+ self.session.close()
880
+ except Exception:
881
+ pass
882
+
883
+ # -- robots ---------------------------------------------------------
884
+
885
+ def assert_allowed(self, url: str) -> RobotsPolicy:
886
+ """Raise RobotsDenied when policy is ``respect`` and robots forbids it.
887
+
888
+ ``warn`` returns the denying policy instead of raising, so the caller
889
+ can still read the page and must record ``robots_overridden: true``.
890
+ """
891
+
892
+ if not self.config.respect_robots or self.config.robots_policy == "ignore":
893
+ return RobotsPolicy.allowing_all(note="robots checking disabled by policy")
894
+ policy = self.robots.policy_for(url)
895
+ if not policy.allowed(url):
896
+ if self.config.robots_policy == "warn":
897
+ return policy
898
+ detail = policy.note or policy.source or "robots.txt"
899
+ raise RobotsDenied(f"robots.txt disallows {url} ({detail})")
900
+ return policy
901
+
902
+ # -- main entry -----------------------------------------------------
903
+
904
+ def fetch(
905
+ self, url: str, *, check_robots: bool = True, use_cache: bool = True
906
+ ) -> HttpResponse:
907
+ """Fetch one URL.
908
+
909
+ ``use_cache=False`` skips both lookup and write. A bootstrap request
910
+ whose only job is to set cookies must not be answered from the cache:
911
+ a cached anti-bot page would silently supply no cookie at all.
912
+ """
913
+
914
+ url = normalize_url(url)
915
+ assert_public_url(url, allow_private=self.config.allow_private)
916
+ policy: Optional[RobotsPolicy] = None
917
+ if check_robots:
918
+ policy = self.assert_allowed(url)
919
+
920
+ cached = self._read_cache(url) if use_cache else None
921
+ if cached is not None:
922
+ return cached
923
+
924
+ extra = policy.crawl_delay if policy and policy.crawl_delay else 0.0
925
+ self.limiter.wait(url, extra)
926
+
927
+ last_error: Optional[Exception] = None
928
+ for attempt in range(self.config.retries + 1):
929
+ try:
930
+ response = _guarded_request(
931
+ self.session.get,
932
+ url,
933
+ allow_private=self.config.allow_private,
934
+ max_redirects=self.session.max_redirects,
935
+ timeout=self.config.timeout,
936
+ stream=True,
937
+ )
938
+ except requests.RequestException as exc:
939
+ last_error = exc
940
+ if attempt >= self.config.retries:
941
+ break
942
+ time.sleep(min(0.5 * (2**attempt), 4.0))
943
+ continue
944
+
945
+ status = response.status_code
946
+ if status in (429, 500, 502, 503, 504) and attempt < self.config.retries:
947
+ retry_after = _retry_after_seconds(response.headers.get("Retry-After"))
948
+ response.close()
949
+ time.sleep(retry_after if retry_after is not None else min(1.0 * (2**attempt), 8.0))
950
+ continue
951
+
952
+ if status >= 400:
953
+ response.close()
954
+ raise FetchError(f"HTTP {status} for {url}")
955
+
956
+ result = self._read_response(response, url)
957
+ if use_cache:
958
+ self._write_cache(url, result)
959
+ return result
960
+
961
+ raise FetchError(f"request failed for {url}: {last_error}") from last_error
962
+
963
+ # -- on-disk response cache ----------------------------------------
964
+
965
+ def _cache_paths(self, url: str) -> tuple[Path, Path]:
966
+ root = _cache_root(self.config)
967
+ key = _cache_key(url)
968
+ return root / f"{key}.json", root / f"{key}.bin"
969
+
970
+ def _read_cache(self, url: str) -> Optional[HttpResponse]:
971
+ if not self.config.cache or self.config.cache_ttl <= 0:
972
+ return None
973
+ meta_path, body_path = self._cache_paths(url)
974
+ try:
975
+ if not meta_path.is_file() or not body_path.is_file():
976
+ return None
977
+ age = time.time() - meta_path.stat().st_mtime
978
+ if age > self.config.cache_ttl:
979
+ return None
980
+ meta = json.loads(meta_path.read_text(encoding="utf-8"))
981
+ body = body_path.read_bytes()
982
+ except (OSError, json.JSONDecodeError):
983
+ return None
984
+ content_type = str(meta.get("content_type") or "text/html")
985
+ encoding = str(meta.get("encoding") or "utf-8")
986
+ text = body.decode(encoding, errors="replace")
987
+ log(f"cache hit: {url}", self.config)
988
+ return HttpResponse(
989
+ url=url,
990
+ final_url=str(meta.get("final_url") or url),
991
+ status=int(meta.get("status") or 200),
992
+ headers={str(k): str(v) for k, v in (meta.get("headers") or {}).items()},
993
+ content=body,
994
+ text=text,
995
+ encoding=encoding,
996
+ content_type=content_type,
997
+ from_cache=True,
998
+ )
999
+
1000
+ def _write_cache(self, url: str, response: HttpResponse) -> None:
1001
+ if not self.config.cache or response.status != 200:
1002
+ return
1003
+ lowered = {str(key).lower() for key in response.headers}
1004
+ if lowered & {"set-cookie", "set-cookie2"}:
1005
+ # A response that plants a session cookie is not a neutral copy of a
1006
+ # public page; keep it (and the credential) out of the on-disk cache.
1007
+ log("cache skipped: response sets cookies", self.config)
1008
+ return
1009
+ safe_headers = {
1010
+ key: value
1011
+ for key, value in response.headers.items()
1012
+ if str(key).lower() not in ("authorization", "proxy-authorization")
1013
+ }
1014
+ meta_path, body_path = self._cache_paths(url)
1015
+ try:
1016
+ meta_path.parent.mkdir(parents=True, exist_ok=True)
1017
+ body_path.write_bytes(response.content)
1018
+ meta_path.write_text(
1019
+ json.dumps(
1020
+ {
1021
+ "url": url,
1022
+ "final_url": response.final_url,
1023
+ "status": response.status,
1024
+ "headers": safe_headers,
1025
+ "content_type": response.content_type,
1026
+ "encoding": response.encoding,
1027
+ },
1028
+ ensure_ascii=False,
1029
+ ),
1030
+ encoding="utf-8",
1031
+ )
1032
+ except OSError as exc: # a cache failure must never fail the fetch
1033
+ log(f"cache write skipped ({exc})", self.config)
1034
+
1035
+ def _read_response(self, response: requests.Response, requested_url: str) -> HttpResponse:
1036
+ content_type = (response.headers.get("Content-Type") or "").split(";", 1)[0].strip().lower()
1037
+ path = requested_url.lower().split("?", 1)[0]
1038
+ if content_type == "application/octet-stream":
1039
+ for extension, mapped in _DOCUMENT_EXTENSIONS.items():
1040
+ if path.endswith(extension):
1041
+ content_type = mapped
1042
+ break
1043
+ else:
1044
+ if path.endswith(".pdf"):
1045
+ content_type = "application/pdf"
1046
+ if content_type and not _is_supported_content_type(content_type):
1047
+ response.close()
1048
+ raise FetchError(
1049
+ f"unsupported content type '{content_type}' for {requested_url}; "
1050
+ "web-forager handles HTML, XML/feeds, plain text, CSV, JSON, PDF "
1051
+ "and DOCX/XLSX/EPUB only"
1052
+ )
1053
+
1054
+ chunks: list[bytes] = []
1055
+ total = 0
1056
+ truncated = False
1057
+ try:
1058
+ for chunk in response.iter_content(chunk_size=65_536):
1059
+ if not chunk:
1060
+ continue
1061
+ chunks.append(chunk)
1062
+ total += len(chunk)
1063
+ if total >= self.config.max_bytes:
1064
+ truncated = True
1065
+ break
1066
+ finally:
1067
+ response.close()
1068
+
1069
+ raw = b"".join(chunks)[: self.config.max_bytes]
1070
+ if raw[:5] == b"%PDF-":
1071
+ content_type = "application/pdf"
1072
+ if content_type in ("application/octet-stream", "binary/octet-stream"):
1073
+ if raw[:4] == b"PK\x03\x04":
1074
+ pass # DOCX/XLSX/EPUB is detected after reading
1075
+ elif not _looks_like_text(raw):
1076
+ response.close()
1077
+ raise FetchError(
1078
+ f"unsupported binary payload for {requested_url}; "
1079
+ "web-forager handles text, HTML, XML/feeds, JSON, PDF and "
1080
+ "DOCX/XLSX/EPUB only"
1081
+ )
1082
+ encoding = _detect_encoding(raw, content_type, response.encoding)
1083
+ text = raw.decode(encoding, errors="replace")
1084
+ if truncated:
1085
+ text += "\n<!-- web-forager: response truncated at max_bytes -->"
1086
+
1087
+ headers = {k.lower(): v for k, v in response.headers.items()}
1088
+ return HttpResponse(
1089
+ url=requested_url,
1090
+ final_url=response.url,
1091
+ status=response.status_code,
1092
+ headers=headers,
1093
+ content=raw,
1094
+ text=text,
1095
+ encoding=encoding,
1096
+ content_type=content_type or "text/html",
1097
+ )
1098
+
1099
+ def resolve(self, url: str) -> str:
1100
+ """Follow redirects cheaply and decode Sogou's JS ``/link`` hop.
1101
+
1102
+ Used only for search-result URLs; the returned URL is still passed
1103
+ through robots.txt before its content is fetched.
1104
+ """
1105
+
1106
+ url = normalize_url(url)
1107
+ self.limiter.wait(url)
1108
+ response: Optional[requests.Response] = None
1109
+ try:
1110
+ response = _guarded_request(
1111
+ self.session.get,
1112
+ url,
1113
+ allow_private=self.config.allow_private,
1114
+ max_redirects=self.session.max_redirects,
1115
+ timeout=self.config.timeout,
1116
+ stream=True,
1117
+ )
1118
+ final = response.url
1119
+ text = ""
1120
+ if response.status_code == 200:
1121
+ try:
1122
+ text = response.content[:4096].decode("utf-8", "ignore")
1123
+ except Exception:
1124
+ text = ""
1125
+ is_js_redirect = bool(
1126
+ text and ("sogou.com/link" in final or "window.location.replace" in text)
1127
+ )
1128
+ if is_js_redirect:
1129
+ final = decode_js_redirect(text) or final
1130
+ return final
1131
+ except (requests.RequestException, InvalidURLError):
1132
+ return url
1133
+ finally:
1134
+ if response is not None:
1135
+ try:
1136
+ response.close()
1137
+ except Exception:
1138
+ pass
1139
+
1140
+
1141
+ def _is_supported_content_type(content_type: str) -> bool:
1142
+ if (
1143
+ content_type in _JSON_CONTENT_TYPES
1144
+ or content_type in _PDF_CONTENT_TYPES
1145
+ or content_type in _DOCUMENT_CONTENT_TYPES
1146
+ or content_type in ("application/octet-stream", "binary/octet-stream")
1147
+ ):
1148
+ return True
1149
+ return any(content_type.startswith(prefix) for prefix in _HTML_CONTENT_TYPES + _TEXT_CONTENT_TYPES)
1150
+
1151
+
1152
+ def _looks_like_text(raw: bytes) -> bool:
1153
+ sample = raw[:4096]
1154
+ if not sample:
1155
+ return True
1156
+ if b"\x00" in sample:
1157
+ return False
1158
+ controls = sum(1 for byte in sample if byte < 9 or 13 < byte < 32)
1159
+ return controls / len(sample) < 0.02
1160
+
1161
+
1162
+ def _retry_after_seconds(value: Optional[str]) -> Optional[float]:
1163
+ if not value:
1164
+ return None
1165
+ try:
1166
+ return max(0.0, min(float(value), 10.0))
1167
+ except (TypeError, ValueError):
1168
+ return None
1169
+
1170
+
1171
+ _META_CHARSET_RE = re.compile(
1172
+ rb"""<meta[^>]+charset\s*=\s*["']?\s*([A-Za-z0-9._:-]+)""", re.I
1173
+ )
1174
+
1175
+
1176
+ def _detect_encoding(raw: bytes, content_type: str, header_encoding: Optional[str]) -> str:
1177
+ # 1. An explicit charset in Content-Type always wins.
1178
+ match = re.search(r"charset=([A-Za-z0-9._:-]+)", content_type or "", re.I)
1179
+ if match:
1180
+ return match.group(1)
1181
+ # 2. A <meta charset> declaration beats requests' RFC 2616 fallback of
1182
+ # ISO-8859-1 for text/* responses without a charset parameter.
1183
+ match = _META_CHARSET_RE.search(raw[:4096])
1184
+ if match:
1185
+ try:
1186
+ return match.group(1).decode("ascii", "ignore")
1187
+ except Exception:
1188
+ pass
1189
+ # 3. Validate a UTF-8 sample, then common CJK encodings.
1190
+ sample = raw[:4096]
1191
+ try:
1192
+ sample.decode("utf-8")
1193
+ return "utf-8"
1194
+ except UnicodeDecodeError:
1195
+ pass
1196
+ for candidate in ("gb18030", "big5"):
1197
+ try:
1198
+ sample.decode(candidate)
1199
+ return candidate
1200
+ except (UnicodeDecodeError, LookupError):
1201
+ continue
1202
+ if header_encoding and header_encoding.lower() not in ("iso-8859-1", "latin-1"):
1203
+ return header_encoding
1204
+ return "utf-8"
1205
+
1206
+
1207
+ def decode_bytes(data: bytes, content_type: str = "") -> str:
1208
+ """Decode a local HTML/text file using the same heuristics as the fetcher."""
1209
+
1210
+ return data.decode(_detect_encoding(data, content_type, None), errors="replace")
1211
+
1212
+
1213
+ # ---------------------------------------------------------------------------
1214
+ # Search back-ends (public result pages; parsed from scratch)
1215
+ # ---------------------------------------------------------------------------
1216
+
1217
+ _BING_RSS = "https://www.bing.com/search?q={query}&format=rss&count={count}&first={first}"
1218
+ _BING_HTML = (
1219
+ "https://www.bing.com/search?q={query}&count={count}&first={first}"
1220
+ "&setlang=zh-hans&mkt=zh-CN"
1221
+ )
1222
+ _DDG_HTML = "https://html.duckduckgo.com/html/?q={query}"
1223
+ _SO_360 = "https://www.so.com/s?q={query}&pn={page}"
1224
+ _SOGOU = "https://www.sogou.com/web?query={query}&page={page}"
1225
+ _WEIXIN = "https://weixin.sogou.com/weixin?type=2&query={query}&page={page}"
1226
+
1227
+ SEARCH_ENGINES = ("auto", "all", "360", "bing", "sogou", "weixin", "duckduckgo")
1228
+
1229
+ _360_INTERNAL_HOSTS = (
1230
+ "so.com",
1231
+ "360kan.com",
1232
+ "hao.360.com",
1233
+ "e.so.com",
1234
+ "image.so.com",
1235
+ "tv.360kan.com",
1236
+ )
1237
+ _SOGOU_BLOCK_MARKERS = ("antispider", "安全验证", "人机验证", "验证码")
1238
+
1239
+
1240
+ _CJK_RE = re.compile(r"[㐀-䶿一-鿿豈-﫿぀-ヿ가-힯]")
1241
+ _USEFUL_TITLE_RE = re.compile(r"[\w㐀-䶿一-鿿]")
1242
+ _MOJIBAKE_RE = re.compile(r"[�\u0000-\u0008\u000b\u000c\u000e-\u001f]")
1243
+
1244
+
1245
+ def looks_garbled(text: str, threshold: float = 0.2) -> bool:
1246
+ """True when text is mostly replacement or control characters.
1247
+
1248
+ This catches an engine handing back an anti-bot page instead of results.
1249
+ Real mojibake (UTF-8 read as Latin-1) is not caught here: it looks like
1250
+ ordinary punctuation to a character-class check.
1251
+ """
1252
+
1253
+ value = str(text or "")
1254
+ if not value:
1255
+ return False
1256
+ return len(_MOJIBAKE_RE.findall(value)) / len(value) >= threshold
1257
+
1258
+
1259
+ _CJK_CHAR_RE = re.compile(r"[㐀-䶿一-鿿]")
1260
+
1261
+ _CHARACTER_PAGE_MARKERS = (
1262
+ "百科",
1263
+ "字典",
1264
+ "词典",
1265
+ "拼音",
1266
+ "笔顺",
1267
+ "笔画",
1268
+ "部首",
1269
+ "读音",
1270
+ "组词",
1271
+ "汉字",
1272
+ "汉语",
1273
+ "释义",
1274
+ "字义",
1275
+ "意思",
1276
+ "解释",
1277
+ )
1278
+
1279
+
1280
+ def looks_like_character_page(title: str, query: str) -> bool:
1281
+ """True for a dictionary entry that shares only one character with the query.
1282
+
1283
+ Engines that split a Chinese phrase into characters rank the dictionary
1284
+ page of its first character — ``反(汉语汉字)_百度百科`` for a query about
1285
+ 反逆向工程. That page shares a single character with the query and is never
1286
+ the one the caller asked for, so it is dropped instead of padding the list.
1287
+ """
1288
+
1289
+ query_chars = set(_CJK_CHAR_RE.findall(query or ""))
1290
+ if len(query_chars) < 2:
1291
+ return False
1292
+ title_text = title or ""
1293
+ title_chars = set(_CJK_CHAR_RE.findall(title_text))
1294
+ if len(title_chars & query_chars) != 1:
1295
+ return False
1296
+ return any(marker in title_text for marker in _CHARACTER_PAGE_MARKERS)
1297
+
1298
+
1299
+ _DICTIONARY_MARKERS = (
1300
+ "词典",
1301
+ "翻译",
1302
+ "音标",
1303
+ "是什么意思",
1304
+ "英语单词",
1305
+ "汉语词典",
1306
+ "dictionary",
1307
+ "translation",
1308
+ )
1309
+
1310
+
1311
+ def looks_like_dictionary_page(result: dict[str, str], query: str) -> bool:
1312
+ """True for a word page that matches only one token of a longer query.
1313
+
1314
+ Bing occasionally answers a Latin phrase with the dictionary entry of its
1315
+ first word (``reinforcement`` for ``reinforcement learning survey``). The
1316
+ page is not wrong, but it is not an answer to the query; dropping it lets
1317
+ the next result through instead of filling the list with word pages.
1318
+ """
1319
+
1320
+ tokens = _query_tokens(query)
1321
+ if len(tokens) < 2:
1322
+ return False
1323
+ title = (result.get("title") or "").lower()
1324
+ if not any(marker in title for marker in _DICTIONARY_MARKERS):
1325
+ return False
1326
+ return result_coverage(result, tokens) <= 1 / len(tokens)
1327
+
1328
+
1329
+ _DOXXING_PHRASES = (
1330
+ "人肉搜索",
1331
+ "人肉",
1332
+ "开盒",
1333
+ "社工库",
1334
+ "查户籍",
1335
+ "户籍信息",
1336
+ "开房记录",
1337
+ "查开房",
1338
+ "家庭住址",
1339
+ "具体住址",
1340
+ "亲属关系",
1341
+ "查身份证",
1342
+ "查手机号",
1343
+ "查住址",
1344
+ "查家人",
1345
+ "查个人资料",
1346
+ "定位某人",
1347
+ )
1348
+ _DOXXING_LATIN = ("doxxing", "doxx", "doxed", "doxing", "dox ")
1349
+ _SENSITIVE_ATTRIBUTES = (
1350
+ "手机号",
1351
+ "手机号码",
1352
+ "电话号码",
1353
+ "住址",
1354
+ "身份证",
1355
+ "照片",
1356
+ "微信号",
1357
+ "学号",
1358
+ "工号",
1359
+ "行踪",
1360
+ "轨迹",
1361
+ )
1362
+ _PERSON_REFERENCES = (
1363
+ "这个人",
1364
+ "此人",
1365
+ "该人",
1366
+ "某人",
1367
+ "对方",
1368
+ "本人",
1369
+ "他的",
1370
+ "她的",
1371
+ "家人",
1372
+ "家属",
1373
+ )
1374
+ _LATIN_PERSON_RE = re.compile(r"\b[A-Z][a-z]+(?: [A-Z][a-z]+){1,2}\b")
1375
+ _LATIN_SENSITIVE = ("phone number", "home address", "email address", "social security", "id number")
1376
+ _PERSON_NAME_RE = re.compile(r"^[一-鿿]{2,3}$")
1377
+ _LATIN_NAME_RE = re.compile(r"^[A-Z][a-z]+(?: [A-Z][a-z]+){1,2}$")
1378
+
1379
+
1380
+ def doxxing_reason(query: str) -> str:
1381
+ """Return a reason when a query asks to identify a private person, else "".
1382
+
1383
+ The check refuses *intent*, not names: an explicit doxxing phrase, a
1384
+ sensitive attribute asked about a specific person, or a Latin personal name
1385
+ combined with a contact field. A plain name on its own is not refused,
1386
+ because it cannot be told apart from a paper author or a public figure; it
1387
+ draws a warning instead.
1388
+ """
1389
+
1390
+ text = (query or "").lower()
1391
+ for phrase in _DOXXING_PHRASES:
1392
+ if phrase in text:
1393
+ return (
1394
+ "the query asks for information about a person in a way that "
1395
+ "amounts to doxxing or profiling"
1396
+ )
1397
+ for word in _DOXXING_LATIN:
1398
+ if word in text:
1399
+ return "the query names doxxing as its purpose"
1400
+ if any(word in text for word in _LATIN_SENSITIVE) and _LATIN_PERSON_RE.search(query or ""):
1401
+ return "the query asks for contact details about a named individual"
1402
+ if any(word in text for word in _SENSITIVE_ATTRIBUTES):
1403
+ if any(ref in text for ref in _PERSON_REFERENCES):
1404
+ return "the query asks for a sensitive attribute of a specific person"
1405
+ # A short Chinese token beside an attribute word is a name: 王五 身份证.
1406
+ for token in re.split(r"[\s,,;;|/]+", query or ""):
1407
+ if not _PERSON_NAME_RE.match(token):
1408
+ continue
1409
+ if any(token in word or word in token for word in _SENSITIVE_ATTRIBUTES):
1410
+ continue
1411
+ return "the query pairs a person's name with a sensitive attribute"
1412
+ return ""
1413
+
1414
+
1415
+ def looks_like_person_name(query: str) -> bool:
1416
+ """True for a bare personal name, which draws a warning but is not refused."""
1417
+
1418
+ text = (query or "").strip()
1419
+ if not text:
1420
+ return False
1421
+ return bool(_PERSON_NAME_RE.match(text) or _LATIN_NAME_RE.match(text))
1422
+
1423
+
1424
+ def prefers_chinese_engines(query: str) -> bool:
1425
+ """True when a query is mostly CJK text.
1426
+
1427
+ Such queries rank better on 360 and Sogou; Bing's tokeniser treats quoted
1428
+ Chinese phrases as single characters, which buries the precise hits.
1429
+ """
1430
+
1431
+ text = str(query or "")
1432
+ if not text:
1433
+ return False
1434
+ cjk = len(_CJK_RE.findall(text))
1435
+ if not cjk:
1436
+ return False
1437
+ return cjk >= len(re.findall(r"[A-Za-z]", text))
1438
+
1439
+
1440
+ def search(
1441
+ query: str,
1442
+ config: FetchConfig,
1443
+ engine: str = "auto",
1444
+ limit: Optional[int] = None,
1445
+ fetcher: Optional[HttpFetcher] = None,
1446
+ report: Optional[list[dict[str, Any]]] = None,
1447
+ ) -> list[dict[str, str]]:
1448
+ """Return ``[{title, url, snippet, source}]`` for a keyword query.
1449
+
1450
+ ``engine`` is one of ``auto`` (360 + Bing, and 360 + Sogou + the WeChat
1451
+ index for a mostly-CJK query), ``all`` (every engine), ``360``, ``bing``,
1452
+ ``sogou``, ``weixin`` or ``duckduckgo``. ``auto`` asks one more engine when
1453
+ the answer holds fewer than three distinct results.
1454
+
1455
+ Pass ``report`` to collect one entry per engine tried — name, whether it
1456
+ answered, how many results and how long it took — so a caller in a hostile
1457
+ network can see which backend is actually reachable.
1458
+
1459
+ Search endpoints are query interfaces, not target pages. robots.txt
1460
+ therefore applies to the *result pages* we later fetch, not to the query
1461
+ request itself. Queries are still rate-limited per host.
1462
+ """
1463
+
1464
+ query = (query or "").strip()
1465
+ if not query:
1466
+ raise ValueError("query must not be empty")
1467
+ reason = doxxing_reason(query)
1468
+ if reason:
1469
+ raise ProhibitedQuery(
1470
+ f"refused: {reason}. web-forager does not collect or assemble "
1471
+ "profiles of identifiable people; search for a topic, a paper "
1472
+ "title or an organisation instead."
1473
+ )
1474
+ limit = limit or config.num_results or 8
1475
+ engine = (engine or "auto").lower()
1476
+ if engine not in SEARCH_ENGINES:
1477
+ raise ValueError(f"unsupported search engine: {engine}")
1478
+
1479
+ fallback_engines: list[str] = []
1480
+ if engine == "auto":
1481
+ # Bing tokenises Chinese phrases into single characters: a query about
1482
+ # 反逆向工程 comes back as the dictionary page of 反. For a CJK query,
1483
+ # ask 360, Sogou and 搜狗微信 and stop there — the article index is the
1484
+ # one channel that still carries a Chinese person's or institution's
1485
+ # name. Bing stays available through --engine bing.
1486
+ if prefers_chinese_engines(query):
1487
+ engines = ["360", "sogou", "weixin"]
1488
+ fallback_engines = []
1489
+ else:
1490
+ engines = ["360", "bing"]
1491
+ # A Latin query answered with a handful of results (or none) is
1492
+ # often a narrow topic the first two engines do not index; Sogou
1493
+ # widens the net without being asked for every query.
1494
+ fallback_engines = ["sogou"]
1495
+ elif engine == "all":
1496
+ engines = ["360", "bing", "sogou", "weixin"]
1497
+ else:
1498
+ engines = [engine]
1499
+
1500
+ own_fetcher = fetcher is None
1501
+ fetcher = fetcher or HttpFetcher(config)
1502
+ parallel = own_fetcher and config.parallel_engines
1503
+ groups: list[tuple[str, list[dict[str, str]]]] = []
1504
+ errors: list[str] = []
1505
+
1506
+ def run_engine(
1507
+ name: str, fallback: bool, qtext: str = query
1508
+ ) -> tuple[dict[str, Any], Optional[list[dict[str, str]]]]:
1509
+ started = time.monotonic()
1510
+ entry: dict[str, Any] = {"engine": name}
1511
+ if fallback:
1512
+ entry["fallback"] = True
1513
+ if qtext != query:
1514
+ entry["query_variant"] = qtext
1515
+ # A requests.Session is not safe to share across engine threads. Each
1516
+ # engine gets its own fetcher; the disk cache and per-host limiters are
1517
+ # still shared through the filesystem and the host split.
1518
+ local = HttpFetcher(config) if parallel else fetcher
1519
+ try:
1520
+ items = _search_one(
1521
+ name, qtext, config, limit, local, pages=config.search_pages
1522
+ )
1523
+ # An engine that split a Chinese phrase into characters returns
1524
+ # the dictionary page of the first one; drop it before it can
1525
+ # pad the list or inflate the per-engine count.
1526
+ items = [
1527
+ item
1528
+ for item in items
1529
+ if not looks_like_character_page(item.get("title", ""), query)
1530
+ and not looks_like_dictionary_page(item, query)
1531
+ ]
1532
+ entry.update(
1533
+ ok=True,
1534
+ results=len(items),
1535
+ ms=round((time.monotonic() - started) * 1000),
1536
+ )
1537
+ return entry, items
1538
+ except Exception as exc: # one engine failing must not kill the query
1539
+ errors.append(f"{name}: {exc}")
1540
+ entry.update(
1541
+ ok=False,
1542
+ results=0,
1543
+ error=str(exc),
1544
+ ms=round((time.monotonic() - started) * 1000),
1545
+ )
1546
+ return entry, None
1547
+ finally:
1548
+ if parallel:
1549
+ local.close()
1550
+
1551
+ def collect(names: list[str], fallback: bool = False, qtext: str = query) -> None:
1552
+ if parallel and len(names) > 1:
1553
+ with ThreadPoolExecutor(max_workers=len(names)) as pool:
1554
+ outcomes = list(pool.map(lambda n: run_engine(n, fallback, qtext), names))
1555
+ else:
1556
+ outcomes = [run_engine(name, fallback, qtext) for name in names]
1557
+ for entry, items in outcomes:
1558
+ if items is not None:
1559
+ groups.append((entry["engine"], items))
1560
+ if report is not None:
1561
+ report.append(entry)
1562
+
1563
+ try:
1564
+ collect(engines)
1565
+ if fallback_engines and (not groups or _thin_results(groups, limit)):
1566
+ collect(fallback_engines, fallback=True)
1567
+ # A thin first pass is often a formatting mismatch rather than a
1568
+ # missing topic. One simplified variant costs two or three more
1569
+ # requests and is skipped entirely when the first pass was healthy, or
1570
+ # when the caller pinned a single engine on purpose.
1571
+ if groups and engine in ("auto", "all") and _thin_results(groups, limit):
1572
+ for variant in query_variants(query)[1:]:
1573
+ collect(engines, fallback=True, qtext=variant)
1574
+ finally:
1575
+ if own_fetcher:
1576
+ fetcher.close()
1577
+
1578
+ if not groups:
1579
+ raise FetchError("all search engines failed: " + "; ".join(errors) or "no engine available")
1580
+
1581
+ if len(groups) == 1:
1582
+ merged = groups[0][1]
1583
+ else:
1584
+ merged = merge_search_results(groups, query, limit=limit)
1585
+ return diversify_results(
1586
+ dedupe_results(merged, limit=max(limit * 3, limit)), limit=limit
1587
+ )
1588
+
1589
+
1590
+ def _thin_results(groups: list[tuple[str, list[dict[str, str]]]], limit: int) -> bool:
1591
+ """True when the engines answered with fewer than a useful number of hits.
1592
+
1593
+ Three is the floor: below it a caller almost always has to run a second
1594
+ query by hand. The check counts distinct URLs, not raw rows, because one
1595
+ engine repeats the same result across pages.
1596
+ """
1597
+
1598
+ items = dedupe_results(
1599
+ [item for _, group in groups for item in group], limit=max(limit, 3)
1600
+ )
1601
+ return len(items) < min(limit, 3)
1602
+
1603
+
1604
+ def is_360_blocked(html: str) -> bool:
1605
+ """True when so.com answered with a verification page instead of results.
1606
+
1607
+ The page is small and has no result list; a real result page always carries
1608
+ ``res-list``. Only used to report the engine honestly, never to retry past
1609
+ the check.
1610
+ """
1611
+
1612
+ lowered = (html or "").lower()
1613
+ if not lowered or "res-list" in lowered or len(html) > 60000:
1614
+ return False
1615
+ if any(marker in html for marker in ("安全验证", "人机验证", "请输入验证码", "访问异常", "滑动验证")):
1616
+ return True
1617
+ return "验证" in html and len(html) < 12000
1618
+
1619
+
1620
+ def _search_one(
1621
+ name: str,
1622
+ query: str,
1623
+ config: FetchConfig,
1624
+ limit: int,
1625
+ fetcher: HttpFetcher,
1626
+ pages: int = 1,
1627
+ ) -> list[dict[str, str]]:
1628
+ quoted = urllib.parse.quote_plus(query)
1629
+ pages = max(1, pages)
1630
+ collected: list[dict[str, str]] = []
1631
+
1632
+ for page in range(1, pages + 1):
1633
+ if name == "duckduckgo" and page > 1:
1634
+ break
1635
+ if name == "360":
1636
+ response = fetcher.fetch(
1637
+ _SO_360.format(query=quoted, page=page), check_robots=False
1638
+ )
1639
+ if is_360_blocked(response.text):
1640
+ raise FetchError("360 returned a verification page; wait and retry later")
1641
+ found = parse_360_html(response.text, limit=limit)
1642
+ elif name == "sogou":
1643
+ response = fetcher.fetch(
1644
+ _SOGOU.format(query=quoted, page=page), check_robots=False
1645
+ )
1646
+ if is_sogou_blocked(response.text):
1647
+ raise FetchError("Sogou returned a verification page (antispider)")
1648
+ parsed = parse_sogou_html(
1649
+ response.text, base="https://www.sogou.com/", limit=limit
1650
+ )
1651
+ found = resolve_sogou_results(parsed, fetcher, limit=limit)
1652
+ elif name == "weixin":
1653
+ response = fetcher.fetch(
1654
+ _WEIXIN.format(query=quoted, page=page), check_robots=False
1655
+ )
1656
+ if is_sogou_blocked(response.text):
1657
+ raise FetchError("weixin.sogou.com returned a verification page (antispider)")
1658
+ parsed = parse_weixin_html(
1659
+ response.text, base="https://weixin.sogou.com/", limit=limit
1660
+ )
1661
+ found = resolve_sogou_results(parsed, fetcher, limit=limit)
1662
+ elif name == "duckduckgo":
1663
+ response = fetcher.fetch(_DDG_HTML.format(query=quoted), check_robots=False)
1664
+ found = parse_ddg_html(response.text, limit=limit)
1665
+ else:
1666
+ first = 1 + (page - 1) * 10
1667
+ # Bing: RSS is cheaper and gives direct URLs; HTML is the fallback.
1668
+ try:
1669
+ response = fetcher.fetch(
1670
+ _BING_RSS.format(query=quoted, count=limit, first=first),
1671
+ check_robots=False,
1672
+ )
1673
+ found = parse_bing_rss(response.text, limit=limit)
1674
+ except FetchError:
1675
+ found = []
1676
+ if not found:
1677
+ response = fetcher.fetch(
1678
+ _BING_HTML.format(query=quoted, count=limit, first=first),
1679
+ check_robots=False,
1680
+ )
1681
+ found = parse_bing_html(response.text, limit=limit)
1682
+
1683
+ if not found:
1684
+ break
1685
+ collected.extend(found)
1686
+ collected = dedupe_results(collected, limit=limit * pages)
1687
+ if len(collected) >= limit * pages:
1688
+ break
1689
+
1690
+ return collected[: limit * pages]
1691
+
1692
+
1693
+ def _query_tokens(query: str) -> list[str]:
1694
+ return [token.lower() for token in re.split(r"[\s,,;;|]+", query.strip()) if token]
1695
+
1696
+
1697
+ def result_coverage(result: dict[str, str], tokens: list[str]) -> float:
1698
+ """Fraction of query tokens present in title/snippet/url (0..1)."""
1699
+
1700
+ if not tokens:
1701
+ return 0.0
1702
+ haystack = " ".join(
1703
+ (result.get("title", ""), result.get("snippet", ""), result.get("url", ""))
1704
+ ).lower()
1705
+ return sum(1 for token in tokens if token in haystack) / len(tokens)
1706
+
1707
+
1708
+ _HIGH_VALUE_SUFFIXES = (
1709
+ "docs.python.org",
1710
+ "developer.mozilla.org",
1711
+ "wikipedia.org",
1712
+ "arxiv.org",
1713
+ "github.com",
1714
+ "stackoverflow.com",
1715
+ "nature.com",
1716
+ "science.org",
1717
+ "who.int",
1718
+ "worldbank.org",
1719
+ "gov.cn",
1720
+ "edu.cn",
1721
+ "ac.uk",
1722
+ "edu",
1723
+ )
1724
+ _LOW_VALUE_SUFFIXES = (
1725
+ "doc88.com",
1726
+ "docin.com",
1727
+ "wenku.baidu.com",
1728
+ "360doc.com",
1729
+ "book118.com",
1730
+ )
1731
+
1732
+
1733
+ def _host_prior(host: str) -> float:
1734
+ """A small prior for domains that usually answer a query well.
1735
+
1736
+ It only breaks ties and nudges; query coverage carries most of the score,
1737
+ so a content farm with the exact answer still outranks an off-topic paper.
1738
+ """
1739
+
1740
+ host = (host or "").lower()
1741
+ for suffix in _HIGH_VALUE_SUFFIXES:
1742
+ if host == suffix or host.endswith("." + suffix):
1743
+ return 0.08
1744
+ for suffix in _LOW_VALUE_SUFFIXES:
1745
+ if host == suffix or host.endswith("." + suffix):
1746
+ return -0.08
1747
+ return 0.0
1748
+
1749
+
1750
+ def _freshness_bonus(text: str) -> float:
1751
+ """Prefer results whose snippet carries a recent year."""
1752
+
1753
+ years = [int(match) for match in re.findall(r"\b(20\d{2})\b", text or "")]
1754
+ if not years:
1755
+ return 0.0
1756
+ newest = max(years)
1757
+ current = datetime.now(timezone.utc).year
1758
+ if newest >= current:
1759
+ return 0.06
1760
+ if newest == current - 1:
1761
+ return 0.03
1762
+ return 0.0
1763
+
1764
+
1765
+ def query_variants(query: str) -> list[str]:
1766
+ """Return the original query and at most one simplified variant.
1767
+
1768
+ Only used when the first pass came back thin. It strips quotes and common
1769
+ English stopwords, and joins a spaced CJK query, because result pages treat
1770
+ those two forms differently.
1771
+ """
1772
+
1773
+ base = (query or "").strip()
1774
+ if not base:
1775
+ return []
1776
+ variants = [base]
1777
+ simplified = re.sub(r"[\"'“”‘’]", " ", base)
1778
+ simplified = re.sub(r"[\s,,;;|、]+", " ", simplified).strip()
1779
+ tokens = [
1780
+ token
1781
+ for token in simplified.split()
1782
+ if token.lower() not in _QUERY_STOPWORDS
1783
+ ]
1784
+ candidate = " ".join(tokens).strip()
1785
+ if candidate and candidate.lower() != base.lower():
1786
+ variants.append(candidate)
1787
+ if re.search(r"[㐀-鿿]", base) and " " in base:
1788
+ joined = re.sub(r"\s+", "", base)
1789
+ if joined and joined not in variants:
1790
+ variants.append(joined)
1791
+ return variants[:2]
1792
+
1793
+
1794
+ _QUERY_STOPWORDS = {
1795
+ "a",
1796
+ "an",
1797
+ "the",
1798
+ "of",
1799
+ "for",
1800
+ "to",
1801
+ "in",
1802
+ "on",
1803
+ "and",
1804
+ "or",
1805
+ "with",
1806
+ "about",
1807
+ "please",
1808
+ "如何",
1809
+ "怎么",
1810
+ "什么",
1811
+ "怎样",
1812
+ }
1813
+
1814
+
1815
+ def diversify_results(
1816
+ results: list[dict[str, str]], limit: int, per_host: int = 2
1817
+ ) -> list[dict[str, str]]:
1818
+ """Keep at most ``per_host`` results per host, then fill from the overflow.
1819
+
1820
+ A single host that dominates the list (WeChat, one content farm) no longer
1821
+ crowds out every other source; if too few hosts answered, the overflow is
1822
+ appended so the caller still receives ``limit`` results.
1823
+ """
1824
+
1825
+ chosen: list[dict[str, str]] = []
1826
+ overflow: list[dict[str, str]] = []
1827
+ counts: dict[str, int] = {}
1828
+ for item in results:
1829
+ host = (urllib.parse.urlsplit(item.get("url", "")).netloc or "").lower()
1830
+ if counts.get(host, 0) < per_host:
1831
+ counts[host] = counts.get(host, 0) + 1
1832
+ chosen.append(item)
1833
+ else:
1834
+ overflow.append(item)
1835
+ return (chosen + overflow)[:limit]
1836
+
1837
+
1838
+ def merge_search_results(
1839
+ groups: list[tuple[str, list[dict[str, str]]]],
1840
+ query: str,
1841
+ limit: int = 8,
1842
+ ) -> list[dict[str, str]]:
1843
+ """Merge several engines by coverage, title match, domain prior and freshness.
1844
+
1845
+ Engine order only breaks exact ties, so two engines that return the same
1846
+ page still produce one stable ordering.
1847
+ """
1848
+
1849
+ tokens = _query_tokens(query)
1850
+ phrase = " ".join((query or "").lower().split())
1851
+ scored: list[tuple[float, int, int, dict[str, str]]] = []
1852
+ for group_index, (name, items) in enumerate(groups):
1853
+ for rank, item in enumerate(items):
1854
+ candidate = dict(item)
1855
+ candidate.setdefault("source", name)
1856
+ coverage = result_coverage(candidate, tokens)
1857
+ title = (candidate.get("title") or "").lower()
1858
+ title_hits = (
1859
+ sum(1 for token in tokens if token in title) / len(tokens)
1860
+ if tokens
1861
+ else 0.0
1862
+ )
1863
+ exact = 0.1 if phrase and phrase in title else 0.0
1864
+ host = urllib.parse.urlsplit(candidate.get("url", "")).netloc
1865
+ score = (
1866
+ coverage * 0.55
1867
+ + title_hits * 0.2
1868
+ + exact
1869
+ + _host_prior(host)
1870
+ + _freshness_bonus(
1871
+ f"{candidate.get('title', '')} {candidate.get('snippet', '')}"
1872
+ )
1873
+ )
1874
+ scored.append((score, group_index, rank, candidate))
1875
+ scored.sort(key=lambda entry: (-entry[0], entry[1], entry[2]))
1876
+ return [entry[3] for entry in scored[: max(limit * 3, limit)]]
1877
+
1878
+
1879
+
1880
+ def parse_360_html(html: str, limit: int = 8) -> list[dict[str, str]]:
1881
+ """Parse 360 Search organic results (``li.res-list``)."""
1882
+
1883
+ soup = make_soup(html)
1884
+ results: list[dict[str, str]] = []
1885
+ seen: set[str] = set()
1886
+ for item in soup.select("li.res-list"):
1887
+ anchor = item.select_one("h3 a") or item.select_one("a[data-mdurl]")
1888
+ if anchor is None:
1889
+ continue
1890
+ href = _attr_text(anchor, "data-mdurl") or _attr_text(anchor, "href")
1891
+ url = absolute_url(href, base="https://www.so.com/")
1892
+ if not url:
1893
+ continue
1894
+ host = urllib.parse.urlsplit(url).netloc.lower()
1895
+ if any(host == internal or host.endswith("." + internal) for internal in _360_INTERNAL_HOSTS):
1896
+ continue
1897
+ heading = item.select_one("h3")
1898
+ title = _clean_result_text(heading.get_text(" ", strip=True) if heading else "")
1899
+ if not title:
1900
+ continue
1901
+ snippet = ""
1902
+ for selector in (".res-desc", ".res-rich", ".res-comm-con", ".g-ellipsis2", ".mh-title"):
1903
+ node = item.select_one(selector)
1904
+ if node is not None:
1905
+ snippet = _clean_result_text(node.get_text(" ", strip=True))
1906
+ if snippet:
1907
+ break
1908
+ if url in seen:
1909
+ continue
1910
+ seen.add(url)
1911
+ results.append({"title": title, "url": url, "snippet": snippet, "source": "360"})
1912
+ if len(results) >= limit:
1913
+ break
1914
+ return results
1915
+
1916
+
1917
+ def is_sogou_blocked(html: str) -> bool:
1918
+ lowered = (html or "").lower()
1919
+ return "antispider" in lowered or (
1920
+ any(marker in html for marker in _SOGOU_BLOCK_MARKERS) and "vrwrap" not in lowered
1921
+ )
1922
+
1923
+
1924
+ def parse_sogou_html(
1925
+ html: str, base: str = "https://www.sogou.com/", limit: int = 8
1926
+ ) -> list[dict[str, str]]:
1927
+ """Parse Sogou organic results (``div.vrwrap``)."""
1928
+
1929
+ soup = make_soup(html)
1930
+ results: list[dict[str, str]] = []
1931
+ seen: set[str] = set()
1932
+ for item in soup.select("div.vrwrap"):
1933
+ classes = " ".join(str(value) for value in (item.get("class") or []))
1934
+ if "hintBox" in classes:
1935
+ continue
1936
+ anchor = item.select_one("h3 a")
1937
+ if anchor is None:
1938
+ continue
1939
+ href = _attr_text(anchor, "href")
1940
+ url = absolute_url(href, base=base)
1941
+ if not url:
1942
+ continue
1943
+ heading = item.select_one("h3")
1944
+ title = _clean_result_text(heading.get_text(" ", strip=True) if heading else "")
1945
+ if not title:
1946
+ continue
1947
+ snippet = ""
1948
+ for selector in (".str_info", ".space-txt", ".fz-mid", ".text-layout", ".str-text"):
1949
+ node = item.select_one(selector)
1950
+ if node is not None:
1951
+ snippet = _clean_result_text(node.get_text(" ", strip=True))
1952
+ if snippet:
1953
+ break
1954
+ if url in seen:
1955
+ continue
1956
+ seen.add(url)
1957
+ results.append({"title": title, "url": url, "snippet": snippet, "source": "sogou"})
1958
+ if len(results) >= limit:
1959
+ break
1960
+ return results
1961
+
1962
+
1963
+ def _join_cjk(text: str) -> str:
1964
+ """Remove the spaces a highlight tag leaves inside a Chinese sentence.
1965
+
1966
+ ``示例<em>名字</em>解析`` reads as "示例 名字 解析" once tags are replaced with
1967
+ spaces; Chinese text has no word spaces, so joining across CJK characters
1968
+ restores the original phrase without touching spaces between Latin words.
1969
+ """
1970
+
1971
+ return re.sub(r"(?<=[㐀-䶿一-鿿])\s+(?=[㐀-䶿一-鿿])", "", text)
1972
+
1973
+
1974
+ def parse_weixin_html(
1975
+ html: str, base: str = "https://weixin.sogou.com/", limit: int = 8
1976
+ ) -> list[dict[str, str]]:
1977
+ """Parse 搜狗微信 article results (``ul.news-list > li``).
1978
+
1979
+ Each entry is one WeChat official-account article. The link is a Sogou
1980
+ ``/link?url=`` wrapper that resolves to ``mp.weixin.qq.com`` (handled by
1981
+ :func:`resolve_sogou_results`); the account name and publish date travel
1982
+ with the result so a caller can tell a lab's own post from a re-post.
1983
+ """
1984
+
1985
+ soup = make_soup(html)
1986
+ results: list[dict[str, str]] = []
1987
+ seen: set[str] = set()
1988
+ for item in soup.select("ul.news-list li"):
1989
+ anchor = item.select_one("h3 a")
1990
+ if anchor is None:
1991
+ continue
1992
+ href = _attr_text(anchor, "href")
1993
+ url = absolute_url(href, base=base)
1994
+ if not url:
1995
+ continue
1996
+ heading = item.select_one("h3")
1997
+ title = _join_cjk(_clean_result_text(heading.get_text(" ", strip=True) if heading else ""))
1998
+ if not title:
1999
+ continue
2000
+ snippet = ""
2001
+ node = item.select_one("p.txt-info")
2002
+ if node is not None:
2003
+ snippet = _join_cjk(_clean_result_text(node.get_text(" ", strip=True)))
2004
+ entry = {
2005
+ "title": title,
2006
+ "url": url,
2007
+ "snippet": snippet,
2008
+ "source": "weixin",
2009
+ }
2010
+ account = item.select_one("div.s-p span.all-time-y2") or item.select_one(
2011
+ "div.s-p a.account"
2012
+ )
2013
+ if account is not None:
2014
+ name = _clean_result_text(account.get_text(" ", strip=True))
2015
+ if name:
2016
+ entry["account"] = name
2017
+ stamp = re.search(r"timeConvert\('(\d+)'\)", str(item))
2018
+ if stamp:
2019
+ try:
2020
+ entry["date"] = time.strftime(
2021
+ "%Y-%m-%d", time.localtime(int(stamp.group(1)))
2022
+ )
2023
+ except (OverflowError, OSError, ValueError):
2024
+ pass
2025
+ if url in seen:
2026
+ continue
2027
+ seen.add(url)
2028
+ results.append(entry)
2029
+ if len(results) >= limit:
2030
+ break
2031
+ return results
2032
+
2033
+
2034
+ def resolve_sogou_results(
2035
+ results: list[dict[str, str]],
2036
+ fetcher: HttpFetcher,
2037
+ limit: int = 8,
2038
+ ) -> list[dict[str, str]]:
2039
+ """Resolve Sogou ``/link?url=`` wrappers to the real target URL."""
2040
+
2041
+ resolved: list[dict[str, str]] = []
2042
+ for item in results[:limit]:
2043
+ url = item.get("url", "")
2044
+ if "sogou.com/link" in url or "/link?url=" in url:
2045
+ real = fetcher.resolve(url)
2046
+ if real and "sogou.com/link" not in real:
2047
+ item = dict(item)
2048
+ item["url"] = real
2049
+ resolved.append(item)
2050
+ return resolved
2051
+
2052
+
2053
+ def dedupe_results(results: Iterable[dict[str, str]], limit: int = 8) -> list[dict[str, str]]:
2054
+ """Remove duplicate URLs and unusable titles while preserving rank order."""
2055
+
2056
+ seen: set[str] = set()
2057
+ unique: list[dict[str, str]] = []
2058
+ for item in results:
2059
+ url = item.get("url", "")
2060
+ if not url or url in seen:
2061
+ continue
2062
+ title = str(item.get("title") or "")
2063
+ # Engines sometimes answer a query with an anti-bot interstitial and
2064
+ # the "title" comes back as replacement characters or bare punctuation.
2065
+ # An entry with no title at all is left alone: other callers feed
2066
+ # dedupe_results bare URLs.
2067
+ if title and (looks_garbled(title) or not _USEFUL_TITLE_RE.search(title)):
2068
+ continue
2069
+ seen.add(url)
2070
+ unique.append(item)
2071
+ if len(unique) >= limit:
2072
+ break
2073
+ return unique
2074
+
2075
+
2076
+ def _clean_result_text(value: Optional[str]) -> str:
2077
+ if not value:
2078
+ return ""
2079
+ text = html_lib.unescape(str(value))
2080
+ if "<" in text:
2081
+ text = BeautifulSoup(text, "html.parser").get_text(" ", strip=True)
2082
+ return re.sub(r"\s+", " ", text).strip()
2083
+
2084
+
2085
+ def _attr_text(tag: Tag, name: str, default: str = "") -> str:
2086
+ """Read an HTML attribute as a plain string, tolerating list values."""
2087
+
2088
+ value: Any = tag.attrs.get(name, default)
2089
+ if isinstance(value, list):
2090
+ return " ".join(str(item) for item in value).strip()
2091
+ if value is None:
2092
+ return default
2093
+ return str(value).strip()
2094
+
2095
+
2096
+ def _int_attr(tag: Tag, name: str, default: int) -> int:
2097
+ """Read an integer HTML attribute with a safe fallback."""
2098
+
2099
+ raw = _attr_text(tag, name)
2100
+ try:
2101
+ return int(raw) if raw else default
2102
+ except ValueError:
2103
+ return default
2104
+
2105
+
2106
+ def parse_bing_rss(xml_text: str, limit: int = 8) -> list[dict[str, str]]:
2107
+ """Parse a Bing ``format=rss`` response.
2108
+
2109
+ The parser is only ever handed a metered search-engine response, but we
2110
+ still refuse documents that declare a DTD or custom entities. This blocks
2111
+ XXE / entity-expansion payloads without pulling in a non-permitted XML
2112
+ library.
2113
+ """
2114
+
2115
+ results: list[dict[str, str]] = []
2116
+ lowered = xml_text[:20_000].lower()
2117
+ if "<!doctype" in lowered or "<!entity" in lowered:
2118
+ return results
2119
+ try:
2120
+ root = ET.fromstring(xml_text.encode("utf-8") if isinstance(xml_text, str) else xml_text)
2121
+ except ET.ParseError:
2122
+ return results
2123
+
2124
+ for item in root.iter():
2125
+ if not item.tag.lower().endswith("item"):
2126
+ continue
2127
+ title = _clean_result_text(_xml_text(item, "title"))
2128
+ link = (_xml_text(item, "link") or "").strip()
2129
+ description = _clean_result_text(_xml_text(item, "description"))
2130
+ if not link:
2131
+ continue
2132
+ url = absolute_url(_decode_bing_redirect(link), base=None)
2133
+ if not url:
2134
+ continue
2135
+ results.append({"title": title, "url": url, "snippet": description, "source": "bing"})
2136
+ if len(results) >= limit:
2137
+ break
2138
+ return results
2139
+
2140
+
2141
+ def _xml_text(node: ET.Element, name: str) -> str:
2142
+ for child in node:
2143
+ if child.tag.lower().split("}")[-1] == name.lower():
2144
+ return child.text or ""
2145
+ return ""
2146
+
2147
+
2148
+ def parse_bing_html(html: str, limit: int = 8) -> list[dict[str, str]]:
2149
+ """Parse the HTML Bing result list."""
2150
+
2151
+ soup = make_soup(html)
2152
+ results: list[dict[str, str]] = []
2153
+ for block in soup.select("li.b_algo"):
2154
+ anchor = block.select_one("h2 a") or block.select_one("a")
2155
+ if not anchor:
2156
+ continue
2157
+ href = _attr_text(anchor, "href")
2158
+ url = absolute_url(_decode_bing_redirect(href), base="https://www.bing.com/")
2159
+ if not url:
2160
+ continue
2161
+ title = _clean_result_text(anchor.get_text(" ", strip=True))
2162
+ snippet_tag = block.select_one(".b_caption p") or block.select_one(".b_snippet")
2163
+ snippet = _clean_result_text(snippet_tag.get_text(" ", strip=True)) if snippet_tag else ""
2164
+ results.append({"title": title, "url": url, "snippet": snippet, "source": "bing"})
2165
+ if len(results) >= limit:
2166
+ break
2167
+ return results
2168
+
2169
+
2170
+ def parse_ddg_html(html: str, limit: int = 8) -> list[dict[str, str]]:
2171
+ """Parse the DuckDuckGo HTML result page."""
2172
+
2173
+ soup = make_soup(html)
2174
+ results: list[dict[str, str]] = []
2175
+ for block in soup.select("div.result"):
2176
+ anchor = block.select_one("a.result__a")
2177
+ if not anchor:
2178
+ continue
2179
+ href = _attr_text(anchor, "href")
2180
+ url = absolute_url(_decode_ddg_redirect(href), base="https://duckduckgo.com/")
2181
+ if not url:
2182
+ continue
2183
+ title = _clean_result_text(anchor.get_text(" ", strip=True))
2184
+ snippet_tag = block.select_one(".result__snippet")
2185
+ snippet = _clean_result_text(snippet_tag.get_text(" ", strip=True)) if snippet_tag else ""
2186
+ results.append({"title": title, "url": url, "snippet": snippet, "source": "duckduckgo"})
2187
+ if len(results) >= limit:
2188
+ break
2189
+ return results
2190
+
2191
+
2192
+ def _decode_bing_redirect(href: str) -> str:
2193
+ """Unwrap Bing's ``/ck/a?...&u=a1<base64url>`` redirect wrapper."""
2194
+
2195
+ if not href:
2196
+ return href
2197
+ parts = urllib.parse.urlsplit(href)
2198
+ if "bing.com" not in parts.netloc.lower() or not parts.path.startswith("/ck/"):
2199
+ return href
2200
+ query = urllib.parse.parse_qs(parts.query)
2201
+ value = (query.get("u") or [""])[0]
2202
+ if not value.startswith("a1"):
2203
+ return href
2204
+ payload = value[2:]
2205
+ payload += "=" * (-len(payload) % 4)
2206
+ try:
2207
+ decoded = base64.urlsafe_b64decode(payload.encode("ascii")).decode("utf-8", "ignore")
2208
+ except Exception:
2209
+ return href
2210
+ return decoded if decoded.startswith(("http://", "https://")) else href
2211
+
2212
+
2213
+ def _decode_ddg_redirect(href: str) -> str:
2214
+ if not href:
2215
+ return href
2216
+ parts = urllib.parse.urlsplit(href)
2217
+ if "duckduckgo.com" not in parts.netloc.lower():
2218
+ return href
2219
+ query = urllib.parse.parse_qs(parts.query)
2220
+ value = (query.get("uddg") or [""])[0]
2221
+ return urllib.parse.unquote(value) if value else href
2222
+
2223
+
2224
+ # ---------------------------------------------------------------------------
2225
+ # HTML inspection
2226
+ # ---------------------------------------------------------------------------
2227
+
2228
+ _SKIP_TAGS = {
2229
+ "script",
2230
+ "style",
2231
+ "noscript",
2232
+ "template",
2233
+ "svg",
2234
+ "canvas",
2235
+ "iframe",
2236
+ "form",
2237
+ "button",
2238
+ "input",
2239
+ "select",
2240
+ "textarea",
2241
+ "nav",
2242
+ "footer",
2243
+ "aside",
2244
+ "header",
2245
+ }
2246
+
2247
+ _DYNAMIC_MARKERS = (
2248
+ "__NEXT_DATA__",
2249
+ "__NUXT__",
2250
+ 'id="root"',
2251
+ "id='root'",
2252
+ 'id="app"',
2253
+ "id='app'",
2254
+ "window.__INITIAL_STATE__",
2255
+ "data-reactroot",
2256
+ )
2257
+
2258
+
2259
+ def make_soup(html: str) -> BeautifulSoup:
2260
+ try:
2261
+ return BeautifulSoup(html or "", "lxml")
2262
+ except Exception:
2263
+ return BeautifulSoup(html or "", "html.parser")
2264
+
2265
+
2266
+ def visible_text(html: str) -> str:
2267
+ soup = make_soup(html)
2268
+ for tag in soup(list(_SKIP_TAGS)):
2269
+ tag.decompose()
2270
+ return re.sub(r"\s+", " ", soup.get_text(" ", strip=True)).strip()
2271
+
2272
+
2273
+ def looks_like_html_shell(html: str) -> bool:
2274
+ """A tiny HTML document whose body is a script, cookie check or redirect.
2275
+
2276
+ php.cn and similar sites answer a plain HTTP request with a few dozen bytes
2277
+ that only a browser can resolve; without this, extraction reports an empty
2278
+ page even though the rendered page is readable.
2279
+ """
2280
+
2281
+ if not html or len(html) > 4000:
2282
+ return False
2283
+ if len(visible_text(html)) >= 120:
2284
+ return False
2285
+ lowered = html.lower()
2286
+ return "<script" in lowered or "location" in lowered or "window." in lowered
2287
+
2288
+
2289
+ def looks_dynamic(html: str) -> bool:
2290
+ """Heuristic: a short visible body plus SPA markers suggests JS rendering."""
2291
+
2292
+ if not html:
2293
+ return False
2294
+ head = html.lstrip()[:1000].lower()
2295
+ if head.startswith("<?xml") or head.startswith("<rss") or head.startswith("<feed"):
2296
+ return False
2297
+ script_count = html.lower().count("<script")
2298
+ marker_hit = any(marker in html for marker in _DYNAMIC_MARKERS)
2299
+ text = visible_text(html)
2300
+ if len(text) < 400:
2301
+ return marker_hit or script_count >= 3
2302
+ return False
2303
+
2304
+
2305
+ # ---------------------------------------------------------------------------
2306
+ # Markdown conversion
2307
+ # ---------------------------------------------------------------------------
2308
+
2309
+ _INLINE_WS_RE = re.compile(r"[ \t\r\f\v]+")
2310
+ _MULTI_NEWLINE_RE = re.compile(r"\n{3,}")
2311
+
2312
+
2313
+ def _normalize_markdown(markdown: str) -> str:
2314
+ text = (markdown or "").replace("\r\n", "\n").replace("\r", "\n")
2315
+ text = _MULTI_NEWLINE_RE.sub("\n\n", text)
2316
+ return text.strip()
2317
+
2318
+
2319
+ def _hard_split_block(block: str, size: int) -> list[str]:
2320
+ pieces: list[str] = []
2321
+ remaining = block.strip()
2322
+ while len(remaining) > size:
2323
+ cut = remaining.rfind("\n", 0, size)
2324
+ if cut < size // 2:
2325
+ for marker in ("。", "!", "?", ". ", "! ", "? "):
2326
+ candidate = remaining.rfind(marker, 0, size)
2327
+ if candidate >= size // 2:
2328
+ cut = candidate + len(marker)
2329
+ break
2330
+ if cut < size // 2:
2331
+ cut = remaining.rfind(" ", 0, size)
2332
+ if cut < size // 2:
2333
+ cut = size
2334
+ pieces.append(remaining[:cut].strip())
2335
+ remaining = remaining[cut:].strip()
2336
+ if remaining:
2337
+ pieces.append(remaining)
2338
+ return [piece for piece in pieces if piece]
2339
+
2340
+
2341
+ def chunk_markdown(markdown: str, size: int = 4000, overlap: int = 0) -> list[dict[str, Any]]:
2342
+ """Split Markdown into context-sized chunks at paragraph boundaries.
2343
+
2344
+ Each chunk records the nearest preceding heading so an agent can tell
2345
+ which section it is reading without re-reading the whole document.
2346
+ """
2347
+
2348
+ if size <= 0:
2349
+ return []
2350
+ text = _normalize_markdown(markdown)
2351
+ if not text:
2352
+ return []
2353
+ blocks = re.split(r"\n{2,}", text)
2354
+ chunks: list[dict[str, Any]] = []
2355
+ current: list[str] = []
2356
+ current_len = 0
2357
+ heading = ""
2358
+ overlap = max(0, min(overlap, size // 2))
2359
+
2360
+ def flush() -> None:
2361
+ nonlocal current, current_len
2362
+ body = "\n\n".join(current).strip()
2363
+ if body:
2364
+ chunks.append(
2365
+ {
2366
+ "index": len(chunks) + 1,
2367
+ "heading": heading,
2368
+ "chars": len(body),
2369
+ "text": body,
2370
+ }
2371
+ )
2372
+ current = []
2373
+ current_len = 0
2374
+
2375
+ for block in blocks:
2376
+ block = block.strip()
2377
+ if not block:
2378
+ continue
2379
+ match = re.match(r"^(#{1,6})\s+(.+)$", block)
2380
+ if match and len(block) < 300:
2381
+ heading = match.group(2).strip()
2382
+ pieces = _hard_split_block(block, size) if len(block) > size else [block]
2383
+ for piece in pieces:
2384
+ if current and current_len + len(piece) + 2 > size:
2385
+ flush()
2386
+ if overlap and chunks:
2387
+ tail = chunks[-1]["text"][-overlap:]
2388
+ current = [tail]
2389
+ current_len = len(tail)
2390
+ current.append(piece)
2391
+ current_len += len(piece) + 2
2392
+ flush()
2393
+ return chunks
2394
+
2395
+
2396
+ def html_to_markdown(
2397
+ html: str,
2398
+ url: Optional[str] = None,
2399
+ *,
2400
+ include_tables: bool = True,
2401
+ keep_links: bool = False,
2402
+ include_images: bool = False,
2403
+ ) -> str:
2404
+ """Convert HTML to clean Markdown, preferring trafilatura when available."""
2405
+
2406
+ markdown = _trafilatura_markdown(
2407
+ html,
2408
+ url=url,
2409
+ include_tables=include_tables,
2410
+ keep_links=keep_links,
2411
+ include_images=include_images,
2412
+ )
2413
+ if markdown and markdown.strip():
2414
+ return _normalize_markdown(markdown)
2415
+
2416
+ return _bs4_to_markdown(
2417
+ html,
2418
+ url=url,
2419
+ include_tables=include_tables,
2420
+ keep_links=keep_links,
2421
+ include_images=include_images,
2422
+ )
2423
+
2424
+
2425
+ def _trafilatura_markdown(
2426
+ html: str,
2427
+ *,
2428
+ url: Optional[str],
2429
+ include_tables: bool,
2430
+ keep_links: bool,
2431
+ include_images: bool,
2432
+ ) -> str:
2433
+ try:
2434
+ import trafilatura # type: ignore
2435
+ except ImportError:
2436
+ return ""
2437
+ try:
2438
+ return trafilatura.extract(
2439
+ html,
2440
+ url=url,
2441
+ output_format="markdown",
2442
+ include_tables=include_tables,
2443
+ include_links=keep_links,
2444
+ include_images=include_images,
2445
+ include_comments=False,
2446
+ include_formatting=True,
2447
+ favor_recall=True,
2448
+ # A global deduplication cache would silently drop sentences seen in
2449
+ # an earlier page during a long agent session. Duplicate removal
2450
+ # inside one document is already handled by trafilatura itself.
2451
+ deduplicate=False,
2452
+ ) or ""
2453
+ except Exception:
2454
+ return ""
2455
+
2456
+
2457
+ @dataclass
2458
+ class _RenderContext:
2459
+ url: Optional[str]
2460
+ include_tables: bool
2461
+ keep_links: bool
2462
+ include_images: bool
2463
+
2464
+
2465
+ def _bs4_to_markdown(
2466
+ html: str,
2467
+ url: Optional[str],
2468
+ *,
2469
+ include_tables: bool,
2470
+ keep_links: bool,
2471
+ include_images: bool,
2472
+ ) -> str:
2473
+ """A dependency-light fallback converter used when trafilatura is absent."""
2474
+
2475
+ soup = make_soup(html)
2476
+ for tag in soup(list(_SKIP_TAGS)):
2477
+ tag.decompose()
2478
+ for tag in soup.find_all(attrs={"role": ["navigation", "banner", "contentinfo", "complementary"]}):
2479
+ tag.decompose()
2480
+
2481
+ root: Tag = soup.find("article") or soup.find("main") or soup.body or soup # type: ignore[assignment]
2482
+ ctx = _RenderContext(
2483
+ url=url,
2484
+ include_tables=include_tables,
2485
+ keep_links=keep_links,
2486
+ include_images=include_images,
2487
+ )
2488
+ text = _render_children(root, ctx)
2489
+ return _normalize_markdown(text)
2490
+
2491
+
2492
+ def _render_children(node: Tag, ctx: _RenderContext) -> str:
2493
+ return "".join(_render_node(child, ctx) for child in node.children)
2494
+
2495
+
2496
+ def _render_node(node: Any, ctx: _RenderContext) -> str:
2497
+ if isinstance(node, NavigableString):
2498
+ text = str(node)
2499
+ if not text.strip():
2500
+ return "" if "\n" in text else " "
2501
+ return _INLINE_WS_RE.sub(" ", text)
2502
+
2503
+ if not isinstance(node, Tag):
2504
+ return ""
2505
+
2506
+ name = (node.name or "").lower()
2507
+ if name in _SKIP_TAGS:
2508
+ return ""
2509
+ if name in ("h1", "h2", "h3", "h4", "h5", "h6"):
2510
+ heading = _inline(node, ctx).strip()
2511
+ if not heading:
2512
+ return ""
2513
+ return f"\n\n{'#' * int(name[1])} {heading}\n\n"
2514
+ if name == "p":
2515
+ paragraph = _inline(node, ctx).strip()
2516
+ return f"\n\n{paragraph}\n\n" if paragraph else ""
2517
+ if name == "br":
2518
+ return " \n"
2519
+ if name == "hr":
2520
+ return "\n\n---\n\n"
2521
+ if name in ("ul", "ol"):
2522
+ listing = _render_list(node, ctx)
2523
+ return f"\n\n{listing}\n\n" if listing else ""
2524
+ if name == "pre":
2525
+ code = node.get_text().strip("\n")
2526
+ if not code.strip():
2527
+ return ""
2528
+ return f"\n\n```\n{code}\n```\n\n"
2529
+ if name == "blockquote":
2530
+ body = _render_children(node, ctx).strip()
2531
+ if not body:
2532
+ return ""
2533
+ quoted = "\n".join(f"> {line}" if line else ">" for line in body.split("\n"))
2534
+ return f"\n\n{quoted}\n\n"
2535
+ if name == "table":
2536
+ if ctx.include_tables:
2537
+ table_md = _table_markdown(node)
2538
+ if table_md:
2539
+ return f"\n\n{table_md}\n\n"
2540
+ return f"\n\n{_render_children(node, ctx).strip()}\n\n"
2541
+ if name == "img":
2542
+ if not ctx.include_images:
2543
+ return ""
2544
+ src = absolute_url(_attr_text(node, "src"), ctx.url)
2545
+ if not src:
2546
+ return ""
2547
+ alt = _clean_result_text(_attr_text(node, "alt"))
2548
+ return f"![{alt}]({src})"
2549
+ if name == "a":
2550
+ return _inline(node, ctx)
2551
+ if name == "code":
2552
+ code = node.get_text().strip("\n")
2553
+ return f"`{code}`" if code else ""
2554
+ if name in ("strong", "b"):
2555
+ inner = _inline(node, ctx).strip()
2556
+ return f"**{inner}**" if inner else ""
2557
+ if name in ("em", "i"):
2558
+ inner = _inline(node, ctx).strip()
2559
+ return f"*{inner}*" if inner else ""
2560
+ return _render_children(node, ctx)
2561
+
2562
+
2563
+ def _inline(node: Tag, ctx: _RenderContext) -> str:
2564
+ parts: list[str] = []
2565
+ for child in node.children:
2566
+ if isinstance(child, NavigableString):
2567
+ parts.append(_INLINE_WS_RE.sub(" ", str(child)))
2568
+ continue
2569
+ if not isinstance(child, Tag):
2570
+ continue
2571
+ name = (child.name or "").lower()
2572
+ if name in _SKIP_TAGS:
2573
+ continue
2574
+ if name == "br":
2575
+ parts.append(" ")
2576
+ elif name == "a":
2577
+ label = _clean_result_text(child.get_text(" ", strip=True))
2578
+ href = absolute_url(_attr_text(child, "href"), ctx.url)
2579
+ if ctx.keep_links and href and label:
2580
+ parts.append(f"[{label}]({href})")
2581
+ else:
2582
+ parts.append(label)
2583
+ elif name in ("strong", "b"):
2584
+ inner = _inline(child, ctx).strip()
2585
+ if inner:
2586
+ parts.append(f"**{inner}**")
2587
+ elif name in ("em", "i"):
2588
+ inner = _inline(child, ctx).strip()
2589
+ if inner:
2590
+ parts.append(f"*{inner}*")
2591
+ elif name == "code":
2592
+ code = child.get_text().strip("\n")
2593
+ if code:
2594
+ parts.append(f"`{code}`")
2595
+ elif name == "img":
2596
+ if ctx.include_images:
2597
+ src = absolute_url(_attr_text(child, "src"), ctx.url)
2598
+ if src:
2599
+ parts.append(f"![{_clean_result_text(_attr_text(child, 'alt'))}]({src})")
2600
+ else:
2601
+ parts.append(_inline(child, ctx))
2602
+ text = "".join(parts)
2603
+ return re.sub(r"[ \t]{2,}", " ", text)
2604
+
2605
+
2606
+ def _render_list(node: Tag, ctx: _RenderContext) -> str:
2607
+ ordered = (node.name or "").lower() == "ol"
2608
+ try:
2609
+ index = _int_attr(node, "start", 1)
2610
+ except (TypeError, ValueError):
2611
+ index = 1
2612
+
2613
+ lines: list[str] = []
2614
+ for item in node.find_all("li", recursive=False):
2615
+ body = _render_children(item, ctx).strip()
2616
+ body = re.sub(r"\n{2,}", "\n", body)
2617
+ body = body.replace("\n", "\n ")
2618
+ marker = f"{index}." if ordered else "-"
2619
+ lines.append(f"{marker} {body}".rstrip())
2620
+ index += 1
2621
+ return "\n".join(lines)
2622
+
2623
+
2624
+ def _escape_table_cell(value: str) -> str:
2625
+ return value.replace("|", r"\|").replace("\n", " ").strip()
2626
+
2627
+
2628
+ def _table_markdown(table: Tag) -> str:
2629
+ rows: list[list[str]] = []
2630
+ for tr in table.find_all("tr"):
2631
+ # Skip rows that belong to a nested table.
2632
+ if tr.find_parent("table") is not table:
2633
+ continue
2634
+ cells: list[str] = []
2635
+ for cell in tr.find_all(["th", "td"]):
2636
+ if cell.find_parent("tr") is not tr:
2637
+ continue
2638
+ cells.append(_clean_result_text(cell.get_text(" ", strip=True)))
2639
+ if cells:
2640
+ rows.append(cells)
2641
+
2642
+ if len(rows) < 2:
2643
+ return ""
2644
+ width = max(len(row) for row in rows)
2645
+ if width < 2:
2646
+ return ""
2647
+ rows = [row + [""] * (width - len(row)) for row in rows]
2648
+
2649
+ header = rows[0]
2650
+ body = rows[1:]
2651
+ lines = [
2652
+ "| " + " | ".join(_escape_table_cell(cell) for cell in header) + " |",
2653
+ "| " + " | ".join("---" for _ in range(width)) + " |",
2654
+ ]
2655
+ for row in body:
2656
+ lines.append("| " + " | ".join(_escape_table_cell(cell) for cell in row) + " |")
2657
+ return "\n".join(lines)
2658
+
2659
+
2660
+ def extract_tables(html: str, *, max_tables: int = 20, max_columns: int = 40) -> list[dict[str, Any]]:
2661
+ """Extract data tables from HTML as structured rows plus Markdown."""
2662
+
2663
+ soup = make_soup(html)
2664
+ tables: list[dict[str, Any]] = []
2665
+ for table in soup.find_all("table"):
2666
+ caption_tag = table.find("caption")
2667
+ caption = _clean_result_text(caption_tag.get_text(" ", strip=True)) if caption_tag else ""
2668
+ rows: list[list[str]] = []
2669
+ for tr in table.find_all("tr"):
2670
+ if tr.find_parent("table") is not table:
2671
+ continue
2672
+ cells = [
2673
+ _clean_result_text(cell.get_text(" ", strip=True))
2674
+ for cell in tr.find_all(["th", "td"])
2675
+ if cell.find_parent("tr") is tr
2676
+ ]
2677
+ if cells:
2678
+ rows.append(cells[:max_columns])
2679
+ if len(rows) < 2:
2680
+ continue
2681
+ width = max(len(row) for row in rows)
2682
+ if width < 2:
2683
+ continue
2684
+ rows = [row + [""] * (width - len(row)) for row in rows]
2685
+ tables.append(
2686
+ {
2687
+ "caption": caption,
2688
+ "headers": rows[0],
2689
+ "rows": rows[1:],
2690
+ "markdown": _table_markdown(table),
2691
+ }
2692
+ )
2693
+ if len(tables) >= max_tables:
2694
+ break
2695
+ return tables
2696
+
2697
+
2698
+ # ---------------------------------------------------------------------------
2699
+ # Metadata extraction
2700
+ # ---------------------------------------------------------------------------
2701
+
2702
+ _DOI_RE = re.compile(r"\b10\.\d{4,9}/[^\s\"'<>]+", re.I)
2703
+ _ARXIV_RE = re.compile(r"arxiv\.org/(?:abs|pdf)/([0-9]{4}\.[0-9]{4,5}|[a-z\-]+/[0-9]{7})", re.I)
2704
+
2705
+ _ARTICLE_TYPES = {
2706
+ "article",
2707
+ "scholarlyarticle",
2708
+ "newsarticle",
2709
+ "blogposting",
2710
+ "techarticle",
2711
+ "report",
2712
+ "webpage",
2713
+ }
2714
+
2715
+
2716
+ def _meta_map(soup: BeautifulSoup) -> dict[str, list[str]]:
2717
+ mapping: dict[str, list[str]] = {}
2718
+ for tag in soup.find_all("meta"):
2719
+ key = tag.get("name") or tag.get("property") or tag.get("itemprop")
2720
+ value = tag.get("content")
2721
+ if key and value:
2722
+ mapping.setdefault(str(key).strip().lower(), []).append(str(value).strip())
2723
+ return mapping
2724
+
2725
+
2726
+ def _first(mapping: dict[str, list[str]], *keys: str) -> str:
2727
+ for key in keys:
2728
+ values = mapping.get(key.lower())
2729
+ if values:
2730
+ for value in values:
2731
+ if value:
2732
+ return value
2733
+ return ""
2734
+
2735
+
2736
+ def _all(mapping: dict[str, list[str]], *keys: str) -> list[str]:
2737
+ values: list[str] = []
2738
+ for key in keys:
2739
+ values.extend(mapping.get(key.lower(), []))
2740
+ return [value for value in values if value]
2741
+
2742
+
2743
+ def _iter_jsonld(soup: BeautifulSoup) -> Iterator[dict[str, Any]]:
2744
+ for tag in soup.find_all("script"):
2745
+ script_type = (tag.get("type") or "").lower()
2746
+ if "ld+json" not in script_type:
2747
+ continue
2748
+ raw = tag.string or tag.get_text()
2749
+ if not raw:
2750
+ continue
2751
+ try:
2752
+ data = json.loads(raw)
2753
+ except (json.JSONDecodeError, TypeError):
2754
+ continue
2755
+ yield from _flatten_jsonld(data)
2756
+
2757
+
2758
+ def _flatten_jsonld(data: Any) -> Iterator[dict[str, Any]]:
2759
+ if isinstance(data, list):
2760
+ for item in data:
2761
+ yield from _flatten_jsonld(item)
2762
+ elif isinstance(data, dict):
2763
+ if "@graph" in data:
2764
+ yield from _flatten_jsonld(data["@graph"])
2765
+ yield data
2766
+
2767
+
2768
+ def _jsonld_article(soup: BeautifulSoup) -> dict[str, Any]:
2769
+ fallback: dict[str, Any] = {}
2770
+ for item in _iter_jsonld(soup):
2771
+ raw_type = item.get("@type")
2772
+ types = raw_type if isinstance(raw_type, list) else [raw_type]
2773
+ lowered = {str(value).lower() for value in types if value}
2774
+ if lowered & _ARTICLE_TYPES:
2775
+ return item
2776
+ if not fallback and lowered & {"organization", "website", "product"}:
2777
+ fallback = item
2778
+ return fallback
2779
+
2780
+
2781
+ def _jsonld_names(value: Any) -> list[str]:
2782
+ if not value:
2783
+ return []
2784
+ if isinstance(value, str):
2785
+ return [value.strip()] if value.strip() else []
2786
+ if isinstance(value, dict):
2787
+ name = value.get("name") or value.get("legalName")
2788
+ return [str(name).strip()] if name else []
2789
+ if isinstance(value, list):
2790
+ names: list[str] = []
2791
+ for item in value:
2792
+ names.extend(_jsonld_names(item))
2793
+ return names
2794
+ return []
2795
+
2796
+
2797
+ def _jsonld_publisher(value: Any) -> str:
2798
+ if isinstance(value, dict):
2799
+ name = value.get("name")
2800
+ if name:
2801
+ return str(name).strip()
2802
+ parent = value.get("parentOrganization")
2803
+ return _jsonld_publisher(parent) if parent else ""
2804
+ if isinstance(value, list) and value:
2805
+ return _jsonld_publisher(value[0])
2806
+ if isinstance(value, str):
2807
+ return value.strip()
2808
+ return ""
2809
+
2810
+
2811
+ def _clean_doi(value: str) -> str:
2812
+ return value.rstrip(".,;:)]}。,;").strip()
2813
+
2814
+
2815
+ def extract_metadata(html: str, url: Optional[str] = None) -> dict[str, Any]:
2816
+ """Extract bibliographic and social metadata without external services."""
2817
+
2818
+ soup = make_soup(html)
2819
+ meta = _meta_map(soup)
2820
+ jsonld = _jsonld_article(soup)
2821
+
2822
+ title = _first(meta, "citation_title", "og:title", "twitter:title", "dc.title")
2823
+ if not title:
2824
+ title_tag = soup.find("title")
2825
+ title = _clean_result_text(title_tag.get_text() if title_tag else "")
2826
+ if not title:
2827
+ title = str(jsonld.get("headline") or jsonld.get("name") or "").strip()
2828
+
2829
+ authors = _all(meta, "citation_author", "dc.creator", "dcterms.creator", "author")
2830
+ if not authors:
2831
+ authors = _jsonld_names(jsonld.get("author"))
2832
+
2833
+ published = _first(
2834
+ meta,
2835
+ "citation_publication_date",
2836
+ "citation_date",
2837
+ "article:published_time",
2838
+ "date",
2839
+ "dc.date",
2840
+ "dcterms.date",
2841
+ ) or str(jsonld.get("datePublished") or "").strip()
2842
+
2843
+ modified = _first(meta, "article:modified_time", "og:updated_time") or str(
2844
+ jsonld.get("dateModified") or ""
2845
+ ).strip()
2846
+
2847
+ journal = _first(meta, "citation_journal_title", "citation_conference_title")
2848
+ if not journal:
2849
+ journal = _jsonld_publisher(jsonld.get("isPartOf"))
2850
+
2851
+ doi = _first(meta, "citation_doi", "dc.identifier", "dcterms.identifier")
2852
+ if not doi:
2853
+ candidate = ""
2854
+ if url:
2855
+ match = _DOI_RE.search(url)
2856
+ if match:
2857
+ candidate = match.group(0)
2858
+ if not candidate:
2859
+ match = _DOI_RE.search(html[:500_000])
2860
+ if match:
2861
+ candidate = match.group(0)
2862
+ doi = candidate
2863
+ doi = _clean_doi(doi)
2864
+
2865
+ abstract = _first(
2866
+ meta, "citation_abstract", "dc.description", "og:description", "description", "twitter:description"
2867
+ ) or str(jsonld.get("description") or "").strip()
2868
+
2869
+ keywords_raw = _all(meta, "citation_keywords", "keywords")
2870
+ jsonld_keywords = jsonld.get("keywords")
2871
+ if isinstance(jsonld_keywords, str):
2872
+ keywords_raw.append(jsonld_keywords)
2873
+ elif isinstance(jsonld_keywords, list):
2874
+ keywords_raw.extend(str(item) for item in jsonld_keywords)
2875
+ keywords: list[str] = []
2876
+ for chunk in keywords_raw:
2877
+ for word in re.split(r"[,;,;]", chunk):
2878
+ word = word.strip()
2879
+ if word and word not in keywords:
2880
+ keywords.append(word)
2881
+
2882
+ canonical = ""
2883
+ for link in soup.find_all("link"):
2884
+ rel = [str(value).lower() for value in (link.attrs.get("rel") or [])]
2885
+ if "canonical" in rel and _attr_text(link, "href"):
2886
+ canonical = absolute_url(_attr_text(link, "href"), url)
2887
+ if canonical:
2888
+ break
2889
+ canonical = canonical or _first(meta, "og:url")
2890
+
2891
+ language = ""
2892
+ html_tag = soup.find("html")
2893
+ if html_tag and _attr_text(html_tag, "lang"):
2894
+ language = _attr_text(html_tag, "lang")
2895
+ language = language or _first(meta, "content-language", "dc.language") or str(
2896
+ jsonld.get("inLanguage") or ""
2897
+ ).strip()
2898
+
2899
+ arxiv_id = _first(meta, "citation_arxiv_id")
2900
+ if not arxiv_id:
2901
+ match = _ARXIV_RE.search(url or "") or _ARXIV_RE.search(html[:200_000])
2902
+ if match:
2903
+ arxiv_id = match.group(1)
2904
+
2905
+ return {
2906
+ "title": title,
2907
+ "authors": [author for author in authors if author],
2908
+ "published": published,
2909
+ "modified": modified,
2910
+ "journal": journal,
2911
+ "volume": _first(meta, "citation_volume"),
2912
+ "issue": _first(meta, "citation_issue"),
2913
+ "first_page": _first(meta, "citation_firstpage"),
2914
+ "last_page": _first(meta, "citation_lastpage"),
2915
+ "doi": doi,
2916
+ "arxiv_id": arxiv_id,
2917
+ "abstract": abstract,
2918
+ "keywords": keywords,
2919
+ "language": language,
2920
+ "site_name": _first(meta, "og:site_name", "application-name") or str(
2921
+ jsonld.get("publisher", {}).get("name")
2922
+ if isinstance(jsonld.get("publisher"), dict)
2923
+ else ""
2924
+ ).strip(),
2925
+ "canonical_url": canonical,
2926
+ "og_type": _first(meta, "og:type"),
2927
+ }
2928
+
2929
+
2930
+ # ---------------------------------------------------------------------------
2931
+ # Optional dynamic rendering
2932
+ # ---------------------------------------------------------------------------
2933
+
2934
+
2935
+ def render_html(url: str, config: FetchConfig) -> str:
2936
+ """Render a page with Playwright/Chromium (optional dependency)."""
2937
+
2938
+ assert_public_url(url, allow_private=config.allow_private)
2939
+ try:
2940
+ from playwright.sync_api import sync_playwright # type: ignore
2941
+ except ImportError as exc:
2942
+ raise RenderUnavailable(
2943
+ "Playwright is not installed; dynamic rendering is disabled. "
2944
+ "Enable it with: pip install playwright && playwright install chromium"
2945
+ ) from exc
2946
+ except Exception as exc:
2947
+ raise RenderUnavailable(
2948
+ f"Playwright could not be imported ({exc}). Reinstall it with: "
2949
+ "pip install playwright && playwright install chromium"
2950
+ ) from exc
2951
+
2952
+ timeout_ms = max(5_000, int(config.timeout * 1000))
2953
+ try:
2954
+ with sync_playwright() as playwright:
2955
+ browser = playwright.chromium.launch(headless=True)
2956
+ try:
2957
+ context = browser.new_context(user_agent=config.user_agent)
2958
+ page = context.new_page()
2959
+ page.set_default_timeout(timeout_ms)
2960
+ page.goto(url, wait_until="domcontentloaded", timeout=timeout_ms)
2961
+ # A JS or meta redirect can land on a private address; check
2962
+ # where the document actually ended up before reading it.
2963
+ assert_public_url(page.url, allow_private=config.allow_private)
2964
+ try:
2965
+ page.wait_for_load_state("networkidle", timeout=min(5_000, timeout_ms))
2966
+ except Exception:
2967
+ pass
2968
+ if config.wait_for:
2969
+ try:
2970
+ page.wait_for_selector(config.wait_for, timeout=min(10_000, timeout_ms))
2971
+ except Exception:
2972
+ pass
2973
+ for _ in range(config.scroll):
2974
+ try:
2975
+ page.mouse.wheel(0, 1800)
2976
+ page.wait_for_timeout(350)
2977
+ except Exception:
2978
+ break
2979
+ page.wait_for_timeout(500)
2980
+ html = page.content()
2981
+ context.close()
2982
+ return html
2983
+ finally:
2984
+ browser.close()
2985
+ except RenderUnavailable:
2986
+ raise
2987
+ except Exception as exc:
2988
+ raise RenderUnavailable(f"Playwright rendering failed: {exc}") from exc
2989
+
2990
+
2991
+ # ---------------------------------------------------------------------------
2992
+ # Link discovery, sessions and agentic browsing
2993
+ # ---------------------------------------------------------------------------
2994
+
2995
+ _SKIP_LINK_WORDS = (
2996
+ "登录",
2997
+ "注册",
2998
+ "隐私",
2999
+ "免责声明",
3000
+ "广告",
3001
+ "login",
3002
+ "sign in",
3003
+ "sign up",
3004
+ "register",
3005
+ "privacy",
3006
+ "cookie",
3007
+ )
3008
+ _SKIP_LINK_EXTENSIONS = (".zip", ".rar", ".7z", ".exe", ".dmg", ".apk", ".mp4", ".mp3", ".iso")
3009
+
3010
+
3011
+ def _link_in_skipped_region(anchor: Tag) -> bool:
3012
+ """True for chrome (nav, forms, page header) but not an article's header.
3013
+
3014
+ A results page commonly wraps each card's title in ``<article><header>``;
3015
+ skipping every ``<header>`` silently dropped the very links a caller wants.
3016
+ """
3017
+
3018
+ inside_content = anchor.find_parent(["article", "main"]) is not None
3019
+ for parent in anchor.parents:
3020
+ name = parent.name
3021
+ if name not in _SKIP_TAGS:
3022
+ continue
3023
+ if name == "header" and inside_content:
3024
+ continue
3025
+ return True
3026
+ return False
3027
+
3028
+
3029
+ def extract_links(
3030
+ html: str,
3031
+ base_url: str,
3032
+ query: str = "",
3033
+ limit: int = 20,
3034
+ same_site: bool = True,
3035
+ ) -> list[dict[str, Any]]:
3036
+ """Rank in-page links for a follow-up visit.
3037
+
3038
+ Ordering follows the query: anchor text that covers more query tokens
3039
+ ranks first; ties keep document order (how a human scans a page).
3040
+ """
3041
+
3042
+ soup = make_soup(html)
3043
+ base_host = urllib.parse.urlsplit(base_url).netloc.lower()
3044
+ tokens = _query_tokens(query)
3045
+ candidates: list[dict[str, Any]] = []
3046
+ seen: set[str] = set()
3047
+ order = 0
3048
+
3049
+ for anchor in soup.find_all("a", href=True):
3050
+ if _link_in_skipped_region(anchor):
3051
+ continue
3052
+ url = absolute_url(_attr_text(anchor, "href"), base_url)
3053
+ if not url or url in seen:
3054
+ continue
3055
+ parts = urllib.parse.urlsplit(url)
3056
+ if parts.scheme not in _ALLOWED_SCHEMES:
3057
+ continue
3058
+ if same_site and parts.netloc.lower() != base_host:
3059
+ continue
3060
+ text = _clean_result_text(anchor.get_text(" ", strip=True))
3061
+ if len(text) < 3:
3062
+ continue
3063
+ lowered = text.lower()
3064
+ if any(word in lowered for word in _SKIP_LINK_WORDS):
3065
+ continue
3066
+ if any(parts.path.lower().endswith(ext) for ext in _SKIP_LINK_EXTENSIONS):
3067
+ continue
3068
+ score = result_coverage({"title": text, "snippet": "", "url": url}, tokens)
3069
+ candidates.append(
3070
+ {"url": url, "text": text, "score": round(score, 3), "order": order}
3071
+ )
3072
+ seen.add(url)
3073
+ order += 1
3074
+
3075
+ candidates.sort(key=lambda item: (-item["score"], item["order"]))
3076
+ return candidates[:limit]
3077
+
3078
+
3079
+ def session_path(name: str) -> Path:
3080
+ safe = re.sub(r"[^A-Za-z0-9._-]+", "-", str(name or "")).strip("-._") or "default"
3081
+ return Path.home() / ".web-forager" / "sessions" / f"{safe}.json"
3082
+
3083
+
3084
+ def load_session(name: str) -> dict[str, Any]:
3085
+ path = session_path(name)
3086
+ try:
3087
+ data = json.loads(path.read_text(encoding="utf-8"))
3088
+ except (OSError, json.JSONDecodeError):
3089
+ data = {}
3090
+ if not isinstance(data, dict):
3091
+ data = {}
3092
+ data.setdefault("visited", [])
3093
+ data.setdefault("trail", [])
3094
+ return data
3095
+
3096
+
3097
+ def save_session(name: str, data: dict[str, Any]) -> Path:
3098
+ path = session_path(name)
3099
+ path.parent.mkdir(parents=True, exist_ok=True)
3100
+ payload = {
3101
+ "visited": list(data.get("visited", []))[-1000:],
3102
+ "trail": list(data.get("trail", []))[-500:],
3103
+ "updated_at": utc_now(),
3104
+ }
3105
+ path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
3106
+ return path
3107
+
3108
+
3109
+ def surf_web(
3110
+ seed: str,
3111
+ config: Optional[FetchConfig] = None,
3112
+ depth: int = 1,
3113
+ follow: Optional[str] = None,
3114
+ max_pages: Optional[int] = None,
3115
+ engine_name: str = "auto",
3116
+ session: Optional[str] = None,
3117
+ fetcher: Optional[HttpFetcher] = None,
3118
+ ) -> dict[str, Any]:
3119
+ """Search or open a page, then follow ranked links like a human browsing.
3120
+
3121
+ ``--session`` makes the walk resumable: visited URLs are skipped on the
3122
+ next run and the trail is appended to ``~/.web-forager/sessions/<name>.json``.
3123
+ """
3124
+
3125
+ config = config or FetchConfig()
3126
+ config.validate()
3127
+ follow = follow or config.follow
3128
+ if follow not in ("same-site", "all", "none"):
3129
+ raise ValueError("follow must be same-site, all or none")
3130
+ config.follow = follow
3131
+ config.extract_links = True
3132
+ max_pages = max_pages or config.max_pages
3133
+
3134
+ own_fetcher = fetcher is None
3135
+ fetcher = fetcher or HttpFetcher(config)
3136
+ session_data = load_session(session) if session else {"visited": [], "trail": []}
3137
+ visited: set[str] = set(session_data.get("visited", []))
3138
+ trail: list[dict[str, Any]] = list(session_data.get("trail", []))
3139
+ pages: list[dict[str, Any]] = []
3140
+ search_results: list[dict[str, str]] = []
3141
+ render_fallbacks = 0
3142
+
3143
+ seed_text = str(seed or "").strip()
3144
+ if not seed_text:
3145
+ raise ValueError("surf needs a URL or a search query")
3146
+ is_url = seed_text.lower().startswith(("http://", "https://"))
3147
+ query = "" if is_url else seed_text
3148
+
3149
+ try:
3150
+ if is_url:
3151
+ seeds = [normalize_url(seed_text)]
3152
+ else:
3153
+ search_results = search(
3154
+ query, config, engine=engine_name, limit=config.num_results, fetcher=fetcher
3155
+ )
3156
+ seeds = [item["url"] for item in search_results[:max_pages]]
3157
+
3158
+ queue: list[tuple[str, str, int]] = [(url, "search" if not is_url else "seed", 0) for url in seeds]
3159
+
3160
+ while queue and len(pages) < max_pages:
3161
+ url, origin, level = queue.pop(0)
3162
+ if url in visited:
3163
+ continue
3164
+ visited.add(url)
3165
+ try:
3166
+ page = fetch_page(
3167
+ url,
3168
+ config,
3169
+ fetcher=fetcher,
3170
+ query=query,
3171
+ allow_render_fallback=(
3172
+ config.render == "auto" and render_fallbacks < 2
3173
+ ),
3174
+ )
3175
+ if page.get("render_fallback"):
3176
+ render_fallbacks += 1
3177
+ except (
3178
+ RobotsDenied,
3179
+ FetchError,
3180
+ ExtractionError,
3181
+ RenderUnavailable,
3182
+ PdfUnavailable,
3183
+ PdfExtractionError,
3184
+ ) as exc:
3185
+ pages.append({"url": url, "error": f"{exc.__class__.__name__}: {exc}"})
3186
+ trail.append(
3187
+ {
3188
+ "step": len(trail) + 1,
3189
+ "level": level,
3190
+ "from": origin,
3191
+ "url": url,
3192
+ "error": f"{exc.__class__.__name__}: {exc}",
3193
+ }
3194
+ )
3195
+ continue
3196
+
3197
+ pages.append(page)
3198
+ trail.append(
3199
+ {
3200
+ "step": len(trail) + 1,
3201
+ "level": level,
3202
+ "from": origin,
3203
+ "url": url,
3204
+ "title": page.get("title", ""),
3205
+ }
3206
+ )
3207
+
3208
+ if level < depth and follow != "none":
3209
+ for link in page.get("links", [])[: config.max_links]:
3210
+ if link["url"] in visited:
3211
+ continue
3212
+ queue.append((link["url"], url, level + 1))
3213
+
3214
+ return {
3215
+ "mode": "surf",
3216
+ "seed": seed_text,
3217
+ "query": query,
3218
+ "engine": engine_name,
3219
+ "depth": depth,
3220
+ "follow": follow,
3221
+ "session": session or "",
3222
+ "search_results": search_results,
3223
+ "trail": trail,
3224
+ "visited": sorted(visited),
3225
+ "pages": pages,
3226
+ }
3227
+ finally:
3228
+ if session:
3229
+ save_session(session, {"visited": sorted(visited), "trail": trail})
3230
+ if own_fetcher:
3231
+ fetcher.close()
3232
+
3233
+
3234
+ # ---------------------------------------------------------------------------
3235
+ # High-level pipeline
3236
+ # ---------------------------------------------------------------------------
3237
+
3238
+
3239
+ # Interstitials served instead of content. They are checked only when the
3240
+ # extracted body is short, so an article *about* verification is not flagged.
3241
+ _VERIFICATION_PATTERNS = (
3242
+ "verify you are human",
3243
+ "unusual traffic",
3244
+ "enable javascript and cookies to continue",
3245
+ "complete the security check",
3246
+ )
3247
+ _WECHAT_VERIFICATION_PATTERNS = (
3248
+ "环境异常",
3249
+ "完成验证后即可继续访问",
3250
+ "请在微信客户端打开链接",
3251
+ "访问过于频繁",
3252
+ )
3253
+ _VERIFICATION_MAX_CHARS = 1500
3254
+
3255
+
3256
+ def looks_like_verification_page(markdown: str, title: str, url: str) -> str:
3257
+ """Return the marker that identifies a verification page, or ``""``.
3258
+
3259
+ WeChat is checked by host because its interstitial is served on the article
3260
+ URL itself; the generic English markers are safe anywhere as long as the
3261
+ page carries almost no content. The caller turns a match into
3262
+ ``VerificationRequired`` so a record says what happened instead of handing
3263
+ back 65 characters of an error page as if it were the article.
3264
+ """
3265
+
3266
+ text = f"{title}\n{markdown}".strip()
3267
+ if not text or len(text) > _VERIFICATION_MAX_CHARS:
3268
+ return ""
3269
+ lowered = text.lower()
3270
+ host = (urllib.parse.urlparse(url).hostname or "").lower()
3271
+ if host == "mp.weixin.qq.com" or host.endswith(".weixin.qq.com"):
3272
+ for marker in _WECHAT_VERIFICATION_PATTERNS:
3273
+ if marker.lower() in lowered:
3274
+ return marker
3275
+ for marker in _VERIFICATION_PATTERNS:
3276
+ if marker in lowered:
3277
+ return marker
3278
+ return ""
3279
+
3280
+
3281
+ def fetch_page(
3282
+ url: str,
3283
+ config: Optional[FetchConfig] = None,
3284
+ fetcher: Optional[HttpFetcher] = None,
3285
+ query: str = "",
3286
+ allow_render_fallback: bool = True,
3287
+ ) -> dict[str, Any]:
3288
+ """Fetch one URL and return a JSON-serialisable record."""
3289
+
3290
+ config = config or FetchConfig()
3291
+ config.validate()
3292
+ url = normalize_url(url)
3293
+ own_fetcher = fetcher is None
3294
+ fetcher = fetcher or HttpFetcher(config)
3295
+ warnings: list[str] = []
3296
+
3297
+ try:
3298
+ policy = fetcher.assert_allowed(url)
3299
+ robots_checked = bool(config.respect_robots and config.robots_policy != "ignore")
3300
+ robots_overridden = bool(robots_checked and not policy.allowed(url))
3301
+ if robots_overridden:
3302
+ warnings.append(
3303
+ "robots.txt disallows this path; read because robots policy is "
3304
+ "'warn'. This override is recorded as compliance.robots_overridden."
3305
+ )
3306
+ extra_delay = policy.crawl_delay if policy and policy.crawl_delay else 0.0
3307
+
3308
+ response: Optional[HttpResponse] = None
3309
+ rendered = False
3310
+ render_fallback = False
3311
+ html = ""
3312
+
3313
+ if config.render == "always":
3314
+ fetcher.limiter.wait(url, extra_delay)
3315
+ html = render_html(url, config)
3316
+ rendered = True
3317
+ else:
3318
+ response = fetcher.fetch(url)
3319
+ html = response.text
3320
+ if config.render == "auto" and looks_dynamic(html):
3321
+ try:
3322
+ rendered_html = render_html(url, config)
3323
+ if rendered_html and len(rendered_html) > len(html):
3324
+ html = rendered_html
3325
+ rendered = True
3326
+ log(f"rendered dynamically: {url}", config)
3327
+ except RenderUnavailable as exc:
3328
+ warnings.append(f"dynamic rendering skipped: {exc}")
3329
+ except Exception as exc: # pragma: no cover - defensive
3330
+ warnings.append(f"dynamic rendering failed: {exc}")
3331
+
3332
+ # Some sites answer a plain request with a 100-byte script shell
3333
+ # that only a browser resolves. It carries no SPA markers, so the
3334
+ # heuristic above misses it and extraction would report an empty
3335
+ # page. Render it once, within the caller's budget.
3336
+ if (
3337
+ allow_render_fallback
3338
+ and config.render == "auto"
3339
+ and not rendered
3340
+ and response is not None
3341
+ and "html" in response.content_type.lower()
3342
+ and looks_like_html_shell(html)
3343
+ ):
3344
+ try:
3345
+ rendered_html = render_html(url, config)
3346
+ if rendered_html and len(rendered_html) > len(html):
3347
+ html = rendered_html
3348
+ rendered = True
3349
+ render_fallback = True
3350
+ warnings.append(
3351
+ "the server returned a minimal HTML shell; rendered it "
3352
+ "once with Playwright before extraction"
3353
+ )
3354
+ except RenderUnavailable as exc:
3355
+ warnings.append(f"shell rendering skipped: {exc}")
3356
+ except Exception as exc: # pragma: no cover - defensive
3357
+ warnings.append(f"shell rendering failed: {exc}")
3358
+
3359
+ content_type = response.content_type if response else "text/html"
3360
+ final_url = response.final_url if response else url
3361
+ status = response.status if response else 200
3362
+
3363
+ from . import documents, feeds, pdf as pdf_mod
3364
+
3365
+ is_pdf_payload = response is not None and (
3366
+ content_type in _PDF_CONTENT_TYPES or pdf_mod.is_pdf(response.content)
3367
+ )
3368
+ document_kind = ""
3369
+ feed_format = ""
3370
+
3371
+ if is_pdf_payload and response is not None:
3372
+ try:
3373
+ pdf_document = pdf_mod.extract(response.content)
3374
+ except pdf_mod.PdfUnavailable as exc:
3375
+ raise PdfUnavailable(str(exc)) from exc
3376
+ except pdf_mod.PdfError as exc:
3377
+ raise PdfExtractionError(str(exc)) from exc
3378
+ markdown = pdf_mod.to_markdown(pdf_document, url=final_url)
3379
+ metadata = pdf_mod.to_metadata(pdf_document, url=final_url)
3380
+ tables = []
3381
+ warnings.append(
3382
+ f"PDF text extracted: {pdf_document.get('page_count', 0)} page(s), "
3383
+ f"{pdf_document.get('empty_pages', 0)} empty"
3384
+ )
3385
+ elif response is not None and (
3386
+ document_kind := documents.detect(response.content, content_type, final_url)
3387
+ ):
3388
+ document = documents.extract(
3389
+ response.content, document_kind, content_type, final_url
3390
+ )
3391
+ markdown = document["markdown"]
3392
+ metadata = documents.to_metadata(document, final_url)
3393
+ tables = document.get("tables") or []
3394
+ warnings.append(f"{document_kind.upper()} extracted")
3395
+ elif response is not None and (
3396
+ content_type in _FEED_CONTENT_TYPES
3397
+ or (
3398
+ ("<rss" in html[:1000].lower() or "<feed" in html[:1000].lower())
3399
+ and feeds.is_feed(html)
3400
+ )
3401
+ ):
3402
+ feed = feeds.extract(html, limit=max(1, config.num_results))
3403
+ feed_format = feed["format"]
3404
+ markdown = feed["markdown"]
3405
+ metadata = feeds.to_metadata(feed, final_url)
3406
+ tables = []
3407
+ warnings.append(f"feed parsed: {feed['format']} with {feed['count']} item(s)")
3408
+ elif any(ct in content_type for ct in _JSON_CONTENT_TYPES) and response is not None:
3409
+ markdown = "```json\n" + response.text.strip() + "\n```"
3410
+ metadata = {
3411
+ "title": "",
3412
+ "authors": [],
3413
+ "published": "",
3414
+ "modified": "",
3415
+ "journal": "",
3416
+ "volume": "",
3417
+ "issue": "",
3418
+ "first_page": "",
3419
+ "last_page": "",
3420
+ "doi": "",
3421
+ "arxiv_id": "",
3422
+ "abstract": "",
3423
+ "keywords": [],
3424
+ "language": "",
3425
+ "site_name": "",
3426
+ "canonical_url": "",
3427
+ "og_type": "",
3428
+ }
3429
+ tables: list[dict[str, Any]] = []
3430
+ else:
3431
+ markdown = html_to_markdown(
3432
+ html,
3433
+ url=final_url,
3434
+ include_tables=config.include_tables,
3435
+ keep_links=config.keep_links,
3436
+ include_images=config.include_images,
3437
+ )
3438
+ metadata = extract_metadata(html, url=final_url)
3439
+ tables = extract_tables(html) if config.include_tables else []
3440
+
3441
+ marker = looks_like_verification_page(
3442
+ markdown, str(metadata.get("title", "")), final_url
3443
+ )
3444
+ if marker:
3445
+ hint = ""
3446
+ host = (urllib.parse.urlparse(final_url).hostname or "").lower()
3447
+ if host == "mp.weixin.qq.com" or host.endswith(".weixin.qq.com"):
3448
+ hint = (
3449
+ " WeChat serves its article body only to a browser session that "
3450
+ "has already passed verification: try the wechat-dig browser "
3451
+ "playbook with --backend cdp and a profile you have opened once."
3452
+ )
3453
+ raise VerificationRequired(
3454
+ f"{final_url} returned a verification/anti-bot page ({marker}); "
3455
+ f"HTTP mode does not solve challenges.{hint}"
3456
+ )
3457
+
3458
+ if not markdown.strip():
3459
+ raise ExtractionError(
3460
+ f"no readable content extracted from {final_url}; "
3461
+ "the page may be empty, paywalled, or require JavaScript rendering"
3462
+ )
3463
+
3464
+ if config.max_chars and len(markdown) > config.max_chars:
3465
+ markdown = markdown[: config.max_chars].rstrip() + "\n\n<!-- truncated by web-forager -->"
3466
+ warnings.append(f"markdown truncated to {config.max_chars} characters")
3467
+
3468
+ # Everything below came from a third party. Say so in the record, and
3469
+ # name instruction-shaped passages instead of letting them read as
3470
+ # part of the surrounding conversation.
3471
+ injection_flags: list[str] = []
3472
+ trust = "untrusted"
3473
+ if config.input_gate != "off":
3474
+ injection_flags = untrusted.scan(
3475
+ markdown + "\n" + str(metadata.get("title", ""))
3476
+ )
3477
+ if injection_flags:
3478
+ if config.input_gate == "strict":
3479
+ markdown = ""
3480
+ trust = "rejected"
3481
+ warnings.insert(0, untrusted.withheld(injection_flags))
3482
+ else:
3483
+ warnings.insert(0, untrusted.warning(injection_flags))
3484
+
3485
+ links = []
3486
+ if config.extract_links and not is_pdf_payload and not document_kind and not feed_format:
3487
+ links = extract_links(
3488
+ html,
3489
+ final_url,
3490
+ query=query,
3491
+ limit=config.max_links,
3492
+ same_site=config.follow == "same-site",
3493
+ )
3494
+
3495
+ chunks = chunk_markdown(markdown, config.chunk_size) if config.chunk_size else []
3496
+
3497
+ return {
3498
+ "url": url,
3499
+ "final_url": final_url,
3500
+ "status": status,
3501
+ "content_type": content_type,
3502
+ "document_kind": document_kind,
3503
+ "feed_format": feed_format,
3504
+ "cached": bool(response.from_cache) if response else False,
3505
+ "rendered": rendered,
3506
+ "render_fallback": render_fallback,
3507
+ "fetched_at": utc_now(),
3508
+ "title": metadata.get("title", ""),
3509
+ "metadata": metadata,
3510
+ "markdown": markdown,
3511
+ "chunks": chunks,
3512
+ "tables": tables,
3513
+ "links": links,
3514
+ "warnings": warnings,
3515
+ "trust": trust,
3516
+ "injection_flags": injection_flags,
3517
+ "compliance": {
3518
+ "robots_checked": robots_checked,
3519
+ "robots_policy": (
3520
+ config.robots_policy if config.respect_robots else "ignore"
3521
+ ),
3522
+ "robots_overridden": robots_overridden,
3523
+ },
3524
+ }
3525
+ finally:
3526
+ if own_fetcher:
3527
+ fetcher.close()
3528
+
3529
+
3530
+ def search_warnings(
3531
+ query: str, results: list[dict[str, Any]], engine: str = "auto"
3532
+ ) -> list[str]:
3533
+ """Warnings for a search that returned nothing or nothing relevant.
3534
+
3535
+ Both the CLI and the MCP server attach these, so an agent that receives a
3536
+ list of dictionary pages is told they are unrelated instead of reporting
3537
+ them as findings.
3538
+ """
3539
+
3540
+ warnings: list[str] = []
3541
+ tokens = _query_tokens(query)
3542
+ if not results:
3543
+ warnings.append(
3544
+ f"no results: {engine} returned nothing for {query!r}. Try another "
3545
+ "engine (--engine all), a shorter query, or a direct URL."
3546
+ )
3547
+ elif tokens and max(result_coverage(item, tokens) for item in results) <= 0:
3548
+ warnings.append(
3549
+ "none of the returned results mention the query text; the engine "
3550
+ "most likely split the query into single characters (common with "
3551
+ "quoted or CJK names). Treat these results as unrelated and try "
3552
+ "another engine or a more specific query."
3553
+ )
3554
+ if results:
3555
+ probe = " ".join(
3556
+ f"{item.get('title', '')} {item.get('snippet', '')}" for item in results[:8]
3557
+ )
3558
+ flags = untrusted.scan(probe)
3559
+ if flags:
3560
+ warnings.append(untrusted.warning(flags))
3561
+ if looks_like_person_name(query):
3562
+ warnings.append(
3563
+ "the query is a bare personal name. Searching a paper author or a "
3564
+ "public figure is fine; do not build or publish a profile of a "
3565
+ "private individual — that is out of scope for this tool."
3566
+ )
3567
+ return warnings
3568
+
3569
+
3570
+ SITE_SEARCH_TEMPLATES = (
3571
+ "/?s={query}",
3572
+ "/search?q={query}",
3573
+ "/?search={query}",
3574
+ "/search?query={query}",
3575
+ )
3576
+
3577
+
3578
+ def find_within_site(
3579
+ site: str,
3580
+ query: str,
3581
+ config: Optional[FetchConfig] = None,
3582
+ fetcher: Optional[HttpFetcher] = None,
3583
+ max_pages: Optional[int] = None,
3584
+ ) -> dict[str, Any]:
3585
+ """Search inside one site through the site's own search page.
3586
+
3587
+ The general engines are unreliable in some networks while a WordPress-style
3588
+ ``/?s=`` page answers directly. This reads that page, ranks its links by
3589
+ query coverage, and fetches the best ones — the same route a person takes
3590
+ when they already know where the content lives.
3591
+ """
3592
+
3593
+ config = config or FetchConfig()
3594
+ config.validate()
3595
+ site = (site or "").strip()
3596
+ query = (query or "").strip()
3597
+ if not site:
3598
+ raise ValueError("site must not be empty")
3599
+ if not query:
3600
+ raise ValueError("query must not be empty")
3601
+ reason = doxxing_reason(query)
3602
+ if reason:
3603
+ raise ProhibitedQuery(
3604
+ f"refused: {reason}. web-forager does not collect or assemble "
3605
+ "profiles of identifiable people; search for a topic, a paper "
3606
+ "title or an organisation instead."
3607
+ )
3608
+ site_url = normalize_url(site if "://" in site else "https://" + site)
3609
+ pages_limit = config.max_pages if max_pages is None else max(0, max_pages)
3610
+ own_fetcher = fetcher is None
3611
+ fetcher = fetcher or HttpFetcher(config)
3612
+
3613
+ attempts: list[dict[str, Any]] = []
3614
+ page: Optional[HttpResponse] = None
3615
+ search_url = ""
3616
+ try:
3617
+ for template in SITE_SEARCH_TEMPLATES:
3618
+ candidate = site_url.rstrip("/") + template.format(
3619
+ query=urllib.parse.quote_plus(query)
3620
+ )
3621
+ started = time.monotonic()
3622
+ try:
3623
+ response = fetcher.fetch(candidate)
3624
+ except (FetchError, RobotsDenied) as exc:
3625
+ attempts.append(
3626
+ {
3627
+ "url": candidate,
3628
+ "ok": False,
3629
+ "error": f"{exc.__class__.__name__}: {exc}",
3630
+ "ms": round((time.monotonic() - started) * 1000),
3631
+ }
3632
+ )
3633
+ continue
3634
+ attempts.append(
3635
+ {
3636
+ "url": candidate,
3637
+ "ok": True,
3638
+ "status": response.status,
3639
+ "ms": round((time.monotonic() - started) * 1000),
3640
+ }
3641
+ )
3642
+ if response.status == 200 and response.text.strip():
3643
+ page = response
3644
+ search_url = response.final_url or candidate
3645
+ break
3646
+ if page is None:
3647
+ detail = "; ".join(
3648
+ f"{item['url']}: {item.get('error', 'empty response')}" for item in attempts
3649
+ )
3650
+ raise FetchError(f"no readable search page on {site_url} ({detail})")
3651
+
3652
+ warnings: list[str] = []
3653
+ candidates = extract_links(
3654
+ page.text, search_url, query=query, limit=config.max_links, same_site=True
3655
+ )
3656
+ # The search page links back to itself ("skip to content", the form's
3657
+ # action); those are navigation, not results.
3658
+ search_stem = search_url.split("?")[0].rstrip("/")
3659
+ candidates = [
3660
+ item
3661
+ for item in candidates
3662
+ if item.get("url", "").split("?")[0].rstrip("/") != search_stem
3663
+ ]
3664
+ matched = [item for item in candidates if item.get("score", 0) > 0]
3665
+ if matched:
3666
+ candidates = matched
3667
+ elif candidates:
3668
+ warnings.append(
3669
+ "the site's search page carried no link whose text mentions the "
3670
+ "query; its ranked links are returned unfiltered"
3671
+ )
3672
+ else:
3673
+ warnings.append("the site's search page carried no followable links")
3674
+
3675
+ pages: list[dict[str, Any]] = []
3676
+ for item in candidates[:pages_limit]:
3677
+ try:
3678
+ pages.append(fetch_page(item["url"], config, fetcher=fetcher))
3679
+ except (RobotsDenied, FetchError, ExtractionError, RenderUnavailable) as exc:
3680
+ pages.append(
3681
+ {
3682
+ "url": item["url"],
3683
+ "error": f"{exc.__class__.__name__}: {exc}",
3684
+ "warnings": [],
3685
+ }
3686
+ )
3687
+
3688
+ payload: dict[str, Any] = {
3689
+ "mode": "site-search",
3690
+ "site": site_url,
3691
+ "query": query,
3692
+ "search_url": search_url,
3693
+ "candidates": candidates,
3694
+ "pages": pages,
3695
+ "attempts": attempts,
3696
+ }
3697
+ if warnings:
3698
+ payload["warnings"] = warnings
3699
+ return payload
3700
+ finally:
3701
+ if own_fetcher:
3702
+ fetcher.close()
3703
+
3704
+
3705
+ _FEED_LINK_TYPES = ("rss", "atom", "feed")
3706
+
3707
+
3708
+ def discover_feed(
3709
+ url: str,
3710
+ config: Optional[FetchConfig] = None,
3711
+ fetcher: Optional[HttpFetcher] = None,
3712
+ ) -> dict[str, Any]:
3713
+ """Read a feed, discovering it from a page's ``<link rel=alternate>`` first.
3714
+
3715
+ A site's own feed needs no search engine and no rendering: it is the
3716
+ cheapest way to follow a publication in a network where the engines fail.
3717
+ """
3718
+
3719
+ from . import feeds
3720
+
3721
+ config = config or FetchConfig()
3722
+ config.validate()
3723
+ url = (url or "").strip()
3724
+ if not url:
3725
+ raise ValueError("url must not be empty")
3726
+ own_fetcher = fetcher is None
3727
+ fetcher = fetcher or HttpFetcher(config)
3728
+
3729
+ try:
3730
+ response = fetcher.fetch(normalize_url(url))
3731
+ if feeds.is_feed(response.text) or response.content_type in _FEED_CONTENT_TYPES:
3732
+ feed_url = response.final_url
3733
+ feed = feeds.extract(response.text, limit=max(1, config.num_results))
3734
+ else:
3735
+ soup = make_soup(response.text)
3736
+ href = ""
3737
+ for link in soup.find_all("link"):
3738
+ rel = " ".join(str(value) for value in (link.get("rel") or [])).lower()
3739
+ kind = str(link.get("type") or "").lower()
3740
+ if "alternate" in rel and any(word in kind for word in _FEED_LINK_TYPES):
3741
+ href = absolute_url(_attr_text(link, "href"), response.final_url)
3742
+ if href:
3743
+ break
3744
+ if not href:
3745
+ raise ExtractionError(
3746
+ f"no RSS/Atom <link rel=alternate> found at {response.final_url}"
3747
+ )
3748
+ feed_response = fetcher.fetch(href)
3749
+ if not feeds.is_feed(feed_response.text):
3750
+ raise ExtractionError(
3751
+ f"{feed_response.final_url} is not an RSS/Atom feed"
3752
+ )
3753
+ feed = feeds.extract(feed_response.text, limit=max(1, config.num_results))
3754
+ feed_url = feed_response.final_url
3755
+ return {
3756
+ "mode": "feed",
3757
+ "feed_url": feed_url,
3758
+ "format": feed["format"],
3759
+ "count": feed["count"],
3760
+ "items": feed.get("items") or [],
3761
+ "markdown": feed["markdown"],
3762
+ "metadata": feeds.to_metadata(feed, feed_url),
3763
+ }
3764
+ finally:
3765
+ if own_fetcher:
3766
+ fetcher.close()
3767
+
3768
+
3769
+ def probe_engines(
3770
+ config: Optional[FetchConfig] = None,
3771
+ fetcher: Optional[HttpFetcher] = None,
3772
+ query: str = "web forager",
3773
+ ) -> list[dict[str, Any]]:
3774
+ """Ask every search engine once and report which ones answer right now.
3775
+
3776
+ A network can block one engine and allow another; the report turns "search
3777
+ is broken" into "Bing answers, Sogou shows a verification page".
3778
+ """
3779
+
3780
+ config = config or FetchConfig()
3781
+ config.validate()
3782
+ # A probe is a diagnostic, not a retry exercise: one attempt per engine
3783
+ # keeps a blocking network from turning the report into a 60-second wait.
3784
+ probe_config = dataclasses.replace(config, retries=0)
3785
+ own_fetcher = fetcher is None
3786
+ fetcher = fetcher or HttpFetcher(probe_config)
3787
+ report: list[dict[str, Any]] = []
3788
+ try:
3789
+ for name in ("360", "bing", "sogou", "weixin", "duckduckgo"):
3790
+ started = time.monotonic()
3791
+ try:
3792
+ items = _search_one(name, query, probe_config, 1, fetcher, pages=1)
3793
+ entry: dict[str, Any] = {
3794
+ "engine": name,
3795
+ "ok": bool(items),
3796
+ "results": len(items),
3797
+ "ms": round((time.monotonic() - started) * 1000),
3798
+ }
3799
+ if not items:
3800
+ entry["error"] = "returned no results (empty page or a challenge page)"
3801
+ report.append(entry)
3802
+ except Exception as exc:
3803
+ report.append(
3804
+ {
3805
+ "engine": name,
3806
+ "ok": False,
3807
+ "results": 0,
3808
+ "error": str(exc),
3809
+ "ms": round((time.monotonic() - started) * 1000),
3810
+ }
3811
+ )
3812
+ finally:
3813
+ if own_fetcher:
3814
+ fetcher.close()
3815
+ return report
3816
+
3817
+
3818
+ def fetch_by_query(
3819
+ query: str,
3820
+ config: Optional[FetchConfig] = None,
3821
+ engine: str = "auto",
3822
+ fetcher: Optional[HttpFetcher] = None,
3823
+ ) -> dict[str, Any]:
3824
+ """Search for a keyword query, then fetch and clean the top results."""
3825
+
3826
+ config = config or FetchConfig()
3827
+ config.validate()
3828
+ own_fetcher = fetcher is None
3829
+ fetcher = fetcher or HttpFetcher(config)
3830
+
3831
+ try:
3832
+ engine_report: list[dict[str, Any]] = []
3833
+ results = search(
3834
+ query,
3835
+ config,
3836
+ engine=engine,
3837
+ limit=config.num_results,
3838
+ report=engine_report,
3839
+ )
3840
+ pages: list[dict[str, Any]] = []
3841
+ skipped: list[dict[str, str]] = []
3842
+ attempts = 0
3843
+ render_fallbacks = 0
3844
+ # ``max_pages`` means readable pages, not attempts. Keep scanning past
3845
+ # robots-denied or unreadable results, with a cap so one query cannot
3846
+ # turn into an unbounded site walk.
3847
+ max_attempts = min(len(results), max(config.max_pages * 3, config.max_pages + 5))
3848
+ for result in results:
3849
+ if len(pages) >= config.max_pages or attempts >= max_attempts:
3850
+ break
3851
+ attempts += 1
3852
+ try:
3853
+ page = fetch_page(
3854
+ result["url"],
3855
+ config,
3856
+ fetcher=fetcher,
3857
+ allow_render_fallback=(
3858
+ config.render == "auto" and render_fallbacks < 2
3859
+ ),
3860
+ )
3861
+ if page.get("render_fallback"):
3862
+ render_fallbacks += 1
3863
+ pages.append(page)
3864
+ except RobotsDenied as exc:
3865
+ skipped.append(
3866
+ {"url": result["url"], "code": "robots_denied", "reason": str(exc)}
3867
+ )
3868
+ except VerificationRequired as exc:
3869
+ skipped.append(
3870
+ {
3871
+ "url": result["url"],
3872
+ "code": "verification_required",
3873
+ "reason": str(exc),
3874
+ }
3875
+ )
3876
+ except FetchError as exc:
3877
+ skipped.append(
3878
+ {"url": result["url"], "code": "fetch_error", "reason": str(exc)}
3879
+ )
3880
+ except ExtractionError as exc:
3881
+ skipped.append(
3882
+ {"url": result["url"], "code": "extraction_error", "reason": str(exc)}
3883
+ )
3884
+ except RenderUnavailable as exc:
3885
+ skipped.append(
3886
+ {"url": result["url"], "code": "render_unavailable", "reason": str(exc)}
3887
+ )
3888
+ except Exception as exc: # pragma: no cover - defensive
3889
+ skipped.append(
3890
+ {"url": result["url"], "code": "unexpected", "reason": str(exc)}
3891
+ )
3892
+ payload: dict[str, Any] = {
3893
+ "query": query,
3894
+ "engine": engine,
3895
+ "mode": "search",
3896
+ "search_results": results,
3897
+ "engine_report": engine_report,
3898
+ "pages": pages,
3899
+ "skipped": skipped,
3900
+ "pages_attempted": attempts,
3901
+ }
3902
+ warnings = search_warnings(query, results, engine=engine)
3903
+ if skipped:
3904
+ counts: dict[str, int] = {}
3905
+ for item in skipped:
3906
+ counts[item["code"]] = counts.get(item["code"], 0) + 1
3907
+ summary = ", ".join(f"{code} ({count})" for code, count in sorted(counts.items()))
3908
+ warnings.append(
3909
+ f"{len(skipped)} result(s) skipped while filling {config.max_pages} "
3910
+ f"readable page(s): {summary}"
3911
+ )
3912
+ if counts.get("robots_denied"):
3913
+ warnings.append(
3914
+ "robots.txt denied the skipped paths. Use --robots-policy warn only "
3915
+ "when you are authorised to read them; the record will mark the "
3916
+ "override. Do not silently switch to a tool that ignores robots."
3917
+ )
3918
+ if counts.get("verification_required"):
3919
+ warnings.append(
3920
+ "verification_required means the site answered with an anti-bot or "
3921
+ "verification page instead of the article. For a WeChat link, read it "
3922
+ "in browser mode: browser_run with playbook \"wechat-dig\" and that URL "
3923
+ "(CLI: node browser/run.js --playbook wechat-dig --url <URL> --backend cdp) "
3924
+ "after signing in once with browser/launch_chrome.py. Do not try to solve "
3925
+ "the challenge."
3926
+ )
3927
+ wechat_denied = [
3928
+ item
3929
+ for item in skipped
3930
+ if item["code"] == "robots_denied"
3931
+ and (
3932
+ urllib.parse.urlparse(item["url"]).hostname or ""
3933
+ ).lower()
3934
+ == "mp.weixin.qq.com"
3935
+ ]
3936
+ if wechat_denied:
3937
+ warnings.append(
3938
+ "WeChat article links cannot be read in HTTP mode: robots.txt disallows "
3939
+ "/s, and the page itself returns an environment check. Open one with "
3940
+ "browser mode instead: browser_run with playbook \"wechat-dig\" and that "
3941
+ "URL (CLI: node browser/run.js --playbook wechat-dig --url <URL> --backend cdp). "
3942
+ "Do not try to solve the challenge."
3943
+ )
3944
+ if not pages:
3945
+ warnings.append(
3946
+ "No readable page was left. Try --engine bing for a Latin query, "
3947
+ "--action find --site <host> for a site's own search, or "
3948
+ "--action feed --url <site> for its RSS/Atom feed."
3949
+ )
3950
+ if warnings:
3951
+ payload["warnings"] = warnings
3952
+ return payload
3953
+ finally:
3954
+ if own_fetcher:
3955
+ fetcher.close()
3956
+
3957
+
3958
+ def summarize_record(record: dict[str, Any]) -> dict[str, Any]:
3959
+ """Return a compact index entry for one fetched page.
3960
+
3961
+ ``results`` used to repeat every page in full, so a search response carried
3962
+ each Markdown body twice and callers with a response cap lost the tail.
3963
+ The body lives in ``pages`` once; this is the entry an agent loops over.
3964
+ """
3965
+
3966
+ if not isinstance(record, dict):
3967
+ return {"value": record}
3968
+ summary: dict[str, Any] = {
3969
+ "url": record.get("url") or record.get("final_url") or "",
3970
+ "title": record.get("title", ""),
3971
+ "status": record.get("status"),
3972
+ "content_type": record.get("content_type", ""),
3973
+ "markdown_chars": len(record.get("markdown") or ""),
3974
+ "tables": len(record.get("tables") or []),
3975
+ }
3976
+ if record.get("error"):
3977
+ summary.pop("status", None)
3978
+ summary["error"] = record["error"]
3979
+ if record.get("warnings"):
3980
+ summary["warnings"] = record["warnings"]
3981
+ return summary
3982
+
3983
+
3984
+ def redact_url(value: str) -> str:
3985
+ """Drop credentials from a URL before it is printed.
3986
+
3987
+ A proxy URL often carries ``user:password@``; a report must show the host
3988
+ and port without turning the password into plain text in a log.
3989
+ """
3990
+
3991
+ text = str(value or "").strip()
3992
+ if not text:
3993
+ return ""
3994
+ if "://" not in text:
3995
+ text = "http://" + text
3996
+ try:
3997
+ parts = urllib.parse.urlsplit(text)
3998
+ except ValueError:
3999
+ return ""
4000
+ host = parts.hostname or ""
4001
+ port = f":{parts.port}" if parts.port else ""
4002
+ scheme = parts.scheme or "http"
4003
+ return f"{scheme}://{host}{port}" if host else ""
4004
+
4005
+
4006
+ def doctor_report(config: Optional[FetchConfig] = None) -> dict[str, Any]:
4007
+ """Report which capabilities this interpreter can serve, with install hints.
4008
+
4009
+ An agent that gets a rendered-less page can run this and install what is
4010
+ missing instead of guessing why the output was thin.
4011
+ """
4012
+
4013
+ import importlib.metadata
4014
+ import importlib.util
4015
+ import platform
4016
+ import shutil
4017
+
4018
+ def version_of(distribution: str) -> str:
4019
+ try:
4020
+ return importlib.metadata.version(distribution)
4021
+ except Exception:
4022
+ return ""
4023
+
4024
+ def check(name: str, module: str, distribution: str, hint: str, required: bool) -> dict[str, Any]:
4025
+ entry: dict[str, Any] = {"name": name, "required": required}
4026
+ try:
4027
+ available = importlib.util.find_spec(module) is not None
4028
+ except (ImportError, ValueError):
4029
+ available = False
4030
+ entry["ok"] = available
4031
+ if available:
4032
+ entry["version"] = version_of(distribution)
4033
+ elif hint:
4034
+ entry["hint"] = hint
4035
+ return entry
4036
+
4037
+ checks = [
4038
+ check("requests", "requests", "requests", "pip install requests", True),
4039
+ check("beautifulsoup4", "bs4", "beautifulsoup4", "pip install beautifulsoup4", True),
4040
+ check("trafilatura", "trafilatura", "trafilatura", "pip install trafilatura", True),
4041
+ check(
4042
+ "playwright",
4043
+ "playwright",
4044
+ "playwright",
4045
+ "pip install playwright && playwright install chromium",
4046
+ False,
4047
+ ),
4048
+ check("pypdf", "pypdf", "pypdf", "pip install pypdf", False),
4049
+ check("defusedxml", "defusedxml", "defusedxml", "pip install defusedxml", False),
4050
+ ]
4051
+ node_path = shutil.which("node")
4052
+ if node_path:
4053
+ checks.append({"name": "node", "required": False, "ok": True, "path": node_path})
4054
+ else:
4055
+ checks.append(
4056
+ {
4057
+ "name": "node",
4058
+ "required": False,
4059
+ "ok": False,
4060
+ "hint": "install Node.js 18+ only if you use browser mode",
4061
+ }
4062
+ )
4063
+
4064
+ proxy_value = ""
4065
+ proxy_source = ""
4066
+ if config is not None and config.proxy:
4067
+ proxy_value = config.proxy
4068
+ proxy_source = "config"
4069
+ else:
4070
+ for key in (
4071
+ "HTTPS_PROXY",
4072
+ "https_proxy",
4073
+ "ALL_PROXY",
4074
+ "all_proxy",
4075
+ "HTTP_PROXY",
4076
+ "http_proxy",
4077
+ ):
4078
+ value = os.environ.get(key, "").strip()
4079
+ if value:
4080
+ proxy_value = value
4081
+ proxy_source = f"env:{key}"
4082
+ break
4083
+
4084
+ missing_required = [item["name"] for item in checks if item["required"] and not item["ok"]]
4085
+ return {
4086
+ "ok": not missing_required,
4087
+ "python": platform.python_version(),
4088
+ "platform": platform.platform(),
4089
+ "version": __version__,
4090
+ "missing_required": missing_required,
4091
+ "checks": checks,
4092
+ "proxy": {
4093
+ "configured": bool(proxy_value),
4094
+ "source": proxy_source,
4095
+ "url": redact_url(proxy_value),
4096
+ },
4097
+ }
4098
+
4099
+
4100
+ def build_envelope(
4101
+ action: str, payload: dict[str, Any], full_results: bool = False
4102
+ ) -> dict[str, Any]:
4103
+ """Wrap a payload with tool metadata for stable downstream parsing.
4104
+
4105
+ ``results`` is the flat list an agent loops over: fetched pages, videos or
4106
+ Bilibili items. It holds a compact index unless ``full_results`` is set,
4107
+ because repeating every page body in both ``pages`` and ``results`` doubled
4108
+ the response and pushed the tail past a caller's size cap. A surf run keeps
4109
+ its extra ``trail``/``visited`` fields.
4110
+ """
4111
+
4112
+ envelope: dict[str, Any] = {
4113
+ "tool": "web-forager",
4114
+ "version": __version__,
4115
+ "action": action,
4116
+ "generated_at": utc_now(),
4117
+ }
4118
+ envelope.update(payload)
4119
+ if "results" not in envelope:
4120
+ if "pages" in payload:
4121
+ if full_results:
4122
+ envelope["results"] = payload["pages"]
4123
+ else:
4124
+ snippets = {
4125
+ item.get("url", ""): item.get("snippet", "")
4126
+ for item in payload.get("search_results") or []
4127
+ }
4128
+ entries: list[dict[str, Any]] = []
4129
+ for page in payload["pages"]:
4130
+ entry = summarize_record(page)
4131
+ snippet = snippets.get(page.get("url") or page.get("final_url") or "")
4132
+ if snippet:
4133
+ entry["snippet"] = snippet
4134
+ entries.append(entry)
4135
+ envelope["results"] = entries
4136
+ elif "videos" in payload:
4137
+ envelope["results"] = payload["videos"]
4138
+ elif "search_results" in payload:
4139
+ envelope["results"] = payload["search_results"]
4140
+ elif "markdown" in payload or "error" in payload:
4141
+ envelope["results"] = [payload] if full_results else [summarize_record(payload)]
4142
+ else:
4143
+ envelope["results"] = [payload]
4144
+ return envelope
4145
+
4146
+
4147
+ CORE_SHARED_KEYS = (
4148
+ "metadata",
4149
+ "markdown",
4150
+ "tables",
4151
+ "warnings",
4152
+ "url",
4153
+ "final_url",
4154
+ "title",
4155
+ "captured_at",
4156
+ "fetched_at",
4157
+ "collected_at",
4158
+ "source",
4159
+ "mode",
4160
+ "schema_version",
4161
+ )
4162
+
4163
+
4164
+ def normalize_record(record: dict[str, Any]) -> dict[str, Any]:
4165
+ """Fold an HTTP record onto the core shape the browser mode also emits.
4166
+
4167
+ The two modes share four fields -- ``metadata``, ``markdown``, ``tables``
4168
+ and ``warnings`` -- and differ everywhere else. ``normalize_record`` maps
4169
+ both onto ``core-1`` so a caller can read one list of results without
4170
+ branching on the mode; everything mode-specific is kept under ``extra``
4171
+ rather than dropped. The browser side has the same function in
4172
+ ``browser/lib/record.js``.
4173
+ """
4174
+
4175
+ value = dict(record or {})
4176
+ metadata = value.get("metadata") or {}
4177
+ if not isinstance(metadata, dict):
4178
+ metadata = {}
4179
+ raw_source = value.get("source")
4180
+ browser = raw_source == "web-forager-browser" or bool(value.get("captured_at"))
4181
+ extra = {key: item for key, item in value.items() if key not in CORE_SHARED_KEYS}
4182
+ # ``clean`` records use ``source`` for the local file path, which is not the
4183
+ # producer name; keep it in extra instead of overwriting the core field.
4184
+ if raw_source and raw_source not in ("web-forager", "web-forager-browser"):
4185
+ extra["record_source"] = raw_source
4186
+ tables = value.get("tables")
4187
+ warnings = value.get("warnings")
4188
+ return {
4189
+ "schema_version": "core-1",
4190
+ "source": "web-forager-browser" if browser else "web-forager",
4191
+ "mode": "browser" if browser else "http",
4192
+ "url": value.get("url") or value.get("final_url") or "",
4193
+ "title": value.get("title") or metadata.get("title") or "",
4194
+ "collected_at": value.get("collected_at") or value.get("captured_at") or value.get("fetched_at") or "",
4195
+ "metadata": metadata,
4196
+ "markdown": value.get("markdown") or "",
4197
+ "tables": tables if isinstance(tables, list) else [],
4198
+ "warnings": warnings if isinstance(warnings, list) else [],
4199
+ "extra": extra,
4200
+ }
4201
+
4202
+
4203
+ def apply_core(envelope: dict[str, Any]) -> dict[str, Any]:
4204
+ """Replace an envelope's ``results`` list with core-1 records."""
4205
+
4206
+ output = dict(envelope)
4207
+ results = output.get("results")
4208
+ if isinstance(results, list):
4209
+ output["results"] = [normalize_record(item) for item in results if isinstance(item, dict)]
4210
+ elif isinstance(results, dict):
4211
+ output["results"] = normalize_record(results)
4212
+ elif "markdown" in output or "metadata" in output:
4213
+ # A single-record action (``clean``) keeps its record at the top level.
4214
+ record = {
4215
+ key: value
4216
+ for key, value in output.items()
4217
+ if key not in ("tool", "version", "action", "generated_at")
4218
+ }
4219
+ output["results"] = [normalize_record(record)]
4220
+ return output