web-forager 0.0.0-stage → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +16 -0
- package/.claude-plugin/plugin.json +20 -0
- package/.clinerules/web-forager.md +85 -0
- package/.codebuddy-plugin/marketplace.json +14 -0
- package/.codebuddy-plugin/plugin.json +25 -0
- package/.codex-plugin/marketplace.json +12 -0
- package/.codex-plugin/mcp.json +20 -0
- package/.codex-plugin/plugin.json +15 -0
- package/.continue/rules/web-forager.md +90 -0
- package/.cursor/rules/web-forager.mdc +80 -0
- package/.dsh-plugin/index.js +11 -0
- package/.dsh-plugin/plugin.json +5 -0
- package/.github/copilot-instructions.md +85 -0
- package/.mcp.json +12 -0
- package/.roo/rules/web-forager.md +85 -0
- package/.windsurf/rules/web-forager.md +90 -0
- package/AGENTS.md +112 -0
- package/CHANGELOG.md +454 -0
- package/LICENSE +21 -0
- package/README.md +477 -3
- package/README_zh.md +477 -0
- package/SKILL.md +351 -0
- package/THIRD_PARTY_LICENSES.txt +35 -0
- package/browser/bridges/playwright-bridge.js +375 -0
- package/browser/launch_chrome.py +144 -0
- package/browser/lib/backend.js +302 -0
- package/browser/lib/bili-api.js +79 -0
- package/browser/lib/bridge.js +83 -0
- package/browser/lib/context.js +60 -0
- package/browser/lib/doctor.js +97 -0
- package/browser/lib/errors.js +17 -0
- package/browser/lib/netguard.js +149 -0
- package/browser/lib/playbooks.js +30 -0
- package/browser/lib/record.js +109 -0
- package/browser/lib/robots.js +195 -0
- package/browser/lib/safety.js +115 -0
- package/browser/lib/selectors.js +80 -0
- package/browser/lib/session.js +116 -0
- package/browser/lib/snapshot.js +229 -0
- package/browser/lib/untrusted.js +56 -0
- package/browser/lib/workspace.js +40 -0
- package/browser/mcp_server.js +272 -0
- package/browser/package.json +40 -0
- package/browser/playbooks/bili-chain.js +58 -0
- package/browser/playbooks/bili-danmaku.js +102 -0
- package/browser/playbooks/bili-feed.js +66 -0
- package/browser/playbooks/bili-video.js +38 -0
- package/browser/playbooks/search-dig.js +105 -0
- package/browser/playbooks/shared.js +138 -0
- package/browser/playbooks/wechat-dig.js +78 -0
- package/browser/playbooks/xhs-dig.js +89 -0
- package/browser/playbooks/zhihu-dig.js +89 -0
- package/browser/run.js +432 -0
- package/commands/browse.md +53 -0
- package/commands/fetch.md +29 -0
- package/commands/map.md +24 -0
- package/commands/scholar.md +28 -0
- package/commands/surf.md +28 -0
- package/cordis.patch.yml +19 -0
- package/docs/ACKNOWLEDGEMENTS.md +15 -0
- package/docs/ACKNOWLEDGEMENTS_zh.md +15 -0
- package/docs/DEPENDENCIES.md +40 -0
- package/docs/INSTALL_COMMANDS.md +113 -0
- package/docs/INSTALL_COMMANDS_zh.md +113 -0
- package/docs/REFERENCE.md +308 -0
- package/docs/REFERENCE_zh.md +309 -0
- package/index.js +7 -0
- package/install.py +254 -0
- package/manifest.json +97 -0
- package/mcp_server.py +597 -0
- package/package.json +80 -6
- package/requirements.txt +12 -0
- package/run.py +852 -0
- package/scripts/__init__.py +1 -0
- package/scripts/bilibili.py +874 -0
- package/scripts/documents.py +437 -0
- package/scripts/engine.py +4220 -0
- package/scripts/feeds.py +180 -0
- package/scripts/pdf.py +135 -0
- package/scripts/scholar.py +354 -0
- package/scripts/sitemap.py +180 -0
- package/scripts/untrusted.py +95 -0
|
@@ -0,0 +1,4220 @@
|
|
|
1
|
+
"""web-forager — clean-room web fetching and cleaning engine.
|
|
2
|
+
|
|
3
|
+
Design notes
|
|
4
|
+
------------
|
|
5
|
+
This module is an original implementation written from scratch. It borrows no
|
|
6
|
+
source code from Crawlee, Scrapy, or any other crawling framework. Only general
|
|
7
|
+
ideas are reused (politeness delay, a robots-policy cache, an extraction
|
|
8
|
+
pipeline); every line below was written for this plugin.
|
|
9
|
+
|
|
10
|
+
Permitted third-party packages
|
|
11
|
+
------------------------------
|
|
12
|
+
requests Apache-2.0
|
|
13
|
+
beautifulsoup4 MIT
|
|
14
|
+
trafilatura Apache-2.0 (preferred extractor, optional at runtime)
|
|
15
|
+
playwright Apache-2.0 (optional, dynamic rendering only)
|
|
16
|
+
|
|
17
|
+
No GPL / AGPL dependency is used.
|
|
18
|
+
|
|
19
|
+
Compliance
|
|
20
|
+
----------
|
|
21
|
+
* robots.txt is always consulted before fetching a target page (RFC 9309
|
|
22
|
+
matching: longest rule wins, ``*`` wildcard, ``$`` end anchor, UA groups).
|
|
23
|
+
* Every request carries an explicit information-gathering User-Agent.
|
|
24
|
+
* Requests are rate-limited per host; the default delay is one second.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import base64
|
|
30
|
+
import dataclasses
|
|
31
|
+
import html as html_lib
|
|
32
|
+
import ipaddress
|
|
33
|
+
import json
|
|
34
|
+
import os
|
|
35
|
+
import re
|
|
36
|
+
import socket
|
|
37
|
+
import sys
|
|
38
|
+
import threading
|
|
39
|
+
import time
|
|
40
|
+
import urllib.parse
|
|
41
|
+
import xml.etree.ElementTree as ET
|
|
42
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
43
|
+
from dataclasses import dataclass, field
|
|
44
|
+
from datetime import datetime, timezone
|
|
45
|
+
from pathlib import Path
|
|
46
|
+
from typing import Any, Iterable, Iterator, Optional
|
|
47
|
+
|
|
48
|
+
import requests
|
|
49
|
+
from bs4 import BeautifulSoup
|
|
50
|
+
from bs4.element import NavigableString, Tag
|
|
51
|
+
|
|
52
|
+
from . import untrusted
|
|
53
|
+
|
|
54
|
+
__version__ = "1.3.0"
|
|
55
|
+
|
|
56
|
+
DEFAULT_USER_AGENT = (
|
|
57
|
+
f"WebForager/{__version__} (Information Gathering; "
|
|
58
|
+
"+https://github.com/JiamingShi-JsPhys/web-forager)"
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
DISCLAIMER = (
|
|
62
|
+
"本插件仅供学习与信息收集使用。使用者需遵守目标网站的 Terms of Service 及相关版权法律。"
|
|
63
|
+
"禁止用于恶意并发攻击。严禁收集、聚合或推断特定自然人的个人信息,或将其用于人肉搜索、骚扰、跟踪等侵害他人隐私的活动;此类用法产生的法律责任由使用者自行承担。"
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
# ---------------------------------------------------------------------------
|
|
67
|
+
# Errors
|
|
68
|
+
# ---------------------------------------------------------------------------
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class WebForagerError(Exception):
|
|
72
|
+
"""Base class for all engine errors."""
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class InvalidURLError(WebForagerError):
|
|
76
|
+
"""The supplied URL is missing, malformed, or uses a forbidden scheme."""
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class RobotsDenied(WebForagerError):
|
|
80
|
+
"""robots.txt forbids fetching this URL."""
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class FetchError(WebForagerError):
|
|
84
|
+
"""Network failure, non-success status, or unsupported content type."""
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class ExtractionError(WebForagerError):
|
|
88
|
+
"""No readable content could be extracted from the document."""
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class VerificationRequired(WebForagerError):
|
|
92
|
+
"""The site answered with a verification or anti-bot page, not content.
|
|
93
|
+
|
|
94
|
+
HTTP mode cannot pass these interstitials: doing so would mean solving a
|
|
95
|
+
challenge, which this tool does not do. Browser mode may succeed when the
|
|
96
|
+
page is readable in a real profile that has already passed verification.
|
|
97
|
+
"""
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class RenderUnavailable(WebForagerError):
|
|
101
|
+
"""Dynamic rendering was requested but Playwright is not usable."""
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class PdfUnavailable(WebForagerError):
|
|
105
|
+
"""PDF support was requested but pypdf is not usable."""
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class PdfExtractionError(WebForagerError):
|
|
109
|
+
"""The PDF could not be parsed or contained no text."""
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class ProhibitedQuery(WebForagerError):
|
|
113
|
+
"""The query asks to identify a private person and is refused."""
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def parse_xml(text: str) -> ET.Element:
|
|
117
|
+
"""Parse untrusted XML without XXE or entity-expansion risk.
|
|
118
|
+
|
|
119
|
+
``defusedxml`` is used when it happens to be installed; otherwise the
|
|
120
|
+
input is rejected outright if it declares a DOCTYPE or an ENTITY, which
|
|
121
|
+
blocks both external entities and billion-laughs expansion.
|
|
122
|
+
"""
|
|
123
|
+
|
|
124
|
+
if "<!ENTITY" in text:
|
|
125
|
+
raise FetchError("refusing XML with ENTITY declarations")
|
|
126
|
+
# An external DTD is harmless here (defusedxml forbids external entities;
|
|
127
|
+
# stdlib ElementTree never fetches the DTD), but an internal subset can
|
|
128
|
+
# declare entities, so only a bare DOCTYPE is stripped.
|
|
129
|
+
doctype = re.search(r"<!DOCTYPE[^>]*>", text, re.I | re.S)
|
|
130
|
+
if doctype:
|
|
131
|
+
if "[" in doctype.group(0):
|
|
132
|
+
raise FetchError("refusing XML with an internal DTD subset")
|
|
133
|
+
text = text.replace(doctype.group(0), "", 1)
|
|
134
|
+
try:
|
|
135
|
+
import defusedxml.ElementTree as DefusedET # type: ignore
|
|
136
|
+
except ImportError:
|
|
137
|
+
DefusedET = None # type: ignore[assignment]
|
|
138
|
+
if DefusedET is not None:
|
|
139
|
+
try:
|
|
140
|
+
return DefusedET.fromstring(text)
|
|
141
|
+
except Exception as exc:
|
|
142
|
+
raise FetchError(f"refusing unsafe XML: {exc}") from exc
|
|
143
|
+
try:
|
|
144
|
+
return ET.fromstring(text)
|
|
145
|
+
except ET.ParseError as exc:
|
|
146
|
+
raise FetchError(f"cannot parse XML: {exc}") from exc
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
# ---------------------------------------------------------------------------
|
|
150
|
+
# Configuration
|
|
151
|
+
# ---------------------------------------------------------------------------
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _allow_private_from_env() -> bool:
|
|
155
|
+
"""Local fixtures and trusted intranet tools set WEB_FORAGER_ALLOW_PRIVATE=1."""
|
|
156
|
+
|
|
157
|
+
return os.environ.get("WEB_FORAGER_ALLOW_PRIVATE", "").strip().lower() in (
|
|
158
|
+
"1",
|
|
159
|
+
"true",
|
|
160
|
+
"yes",
|
|
161
|
+
"on",
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
@dataclass
|
|
166
|
+
class FetchConfig:
|
|
167
|
+
"""Runtime knobs for one CLI invocation."""
|
|
168
|
+
|
|
169
|
+
user_agent: str = DEFAULT_USER_AGENT
|
|
170
|
+
timeout: float = 20.0
|
|
171
|
+
retries: int = 2
|
|
172
|
+
delay: float = 1.0
|
|
173
|
+
max_bytes: int = 8 * 1024 * 1024
|
|
174
|
+
max_chars: int = 20_000
|
|
175
|
+
respect_robots: bool = True
|
|
176
|
+
# respect: refuse a disallowed path; warn: read it and mark the record;
|
|
177
|
+
# ignore: do not consult robots.txt (same as respect_robots=False).
|
|
178
|
+
robots_policy: str = "respect"
|
|
179
|
+
render: str = "auto" # auto | never | always
|
|
180
|
+
include_tables: bool = True
|
|
181
|
+
keep_links: bool = False
|
|
182
|
+
include_images: bool = False
|
|
183
|
+
max_pages: int = 3
|
|
184
|
+
num_results: int = 8
|
|
185
|
+
verbose: bool = False
|
|
186
|
+
extract_links: bool = False
|
|
187
|
+
max_links: int = 20
|
|
188
|
+
follow: str = "same-site" # same-site | all | none
|
|
189
|
+
search_engine: str = "auto"
|
|
190
|
+
search_pages: int = 1
|
|
191
|
+
# Engines are different hosts, so they may run at once without breaking the
|
|
192
|
+
# per-origin delay. Set False for a strictly serial diagnostic run.
|
|
193
|
+
parallel_engines: bool = True
|
|
194
|
+
cache: bool = False
|
|
195
|
+
cache_ttl: int = 3600
|
|
196
|
+
cache_dir: str = ""
|
|
197
|
+
chunk_size: int = 0
|
|
198
|
+
wait_for: str = ""
|
|
199
|
+
scroll: int = 0
|
|
200
|
+
proxy: str = "" # http(s) proxy for every request; "" uses the environment
|
|
201
|
+
# Refuse private/loopback/link-local targets unless the caller opts in.
|
|
202
|
+
allow_private: bool = field(default_factory=_allow_private_from_env)
|
|
203
|
+
# off | mark | strict: what to do with instruction-shaped page text.
|
|
204
|
+
input_gate: str = "mark"
|
|
205
|
+
|
|
206
|
+
def validate(self) -> None:
|
|
207
|
+
if self.render not in ("auto", "never", "always"):
|
|
208
|
+
raise ValueError("render must be auto, never or always")
|
|
209
|
+
if self.robots_policy not in ("respect", "warn", "ignore"):
|
|
210
|
+
raise ValueError("robots_policy must be respect, warn or ignore")
|
|
211
|
+
if self.input_gate not in ("off", "mark", "strict"):
|
|
212
|
+
raise ValueError("input_gate must be off, mark or strict")
|
|
213
|
+
if self.follow not in ("same-site", "all", "none"):
|
|
214
|
+
raise ValueError("follow must be same-site, all or none")
|
|
215
|
+
if self.search_engine not in SEARCH_ENGINES:
|
|
216
|
+
raise ValueError(
|
|
217
|
+
"search_engine must be one of: " + ", ".join(SEARCH_ENGINES)
|
|
218
|
+
)
|
|
219
|
+
if self.timeout <= 0:
|
|
220
|
+
raise ValueError("timeout must be positive")
|
|
221
|
+
if self.delay < 0:
|
|
222
|
+
raise ValueError("delay must be >= 0")
|
|
223
|
+
if self.max_chars < 0:
|
|
224
|
+
raise ValueError("max_chars must be >= 0")
|
|
225
|
+
if self.max_pages < 0:
|
|
226
|
+
raise ValueError("max_pages must be >= 0")
|
|
227
|
+
if self.max_links < 1:
|
|
228
|
+
raise ValueError("max_links must be >= 1")
|
|
229
|
+
if self.search_pages < 1:
|
|
230
|
+
raise ValueError("search_pages must be >= 1")
|
|
231
|
+
if self.cache_ttl < 0:
|
|
232
|
+
raise ValueError("cache_ttl must be >= 0")
|
|
233
|
+
if self.chunk_size < 0:
|
|
234
|
+
raise ValueError("chunk_size must be >= 0")
|
|
235
|
+
if self.scroll < 0:
|
|
236
|
+
raise ValueError("scroll must be >= 0")
|
|
237
|
+
if self.proxy and "://" in self.proxy:
|
|
238
|
+
scheme = self.proxy.split("://", 1)[0].lower()
|
|
239
|
+
if scheme not in ("http", "https", "socks5", "socks5h"):
|
|
240
|
+
raise ValueError("proxy must be an http(s) or socks5 URL")
|
|
241
|
+
if not self.user_agent or len(self.user_agent) < 8:
|
|
242
|
+
raise ValueError("user_agent must be a descriptive string")
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def log(message: str, config: Optional[FetchConfig] = None) -> None:
|
|
246
|
+
"""Write a diagnostic line to stderr (never to stdout)."""
|
|
247
|
+
|
|
248
|
+
if config is not None and not config.verbose:
|
|
249
|
+
return
|
|
250
|
+
print(f"[web-forager] {message}", file=sys.stderr, flush=True)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def utc_now() -> str:
|
|
254
|
+
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
# ---------------------------------------------------------------------------
|
|
258
|
+
# URL helpers
|
|
259
|
+
# ---------------------------------------------------------------------------
|
|
260
|
+
|
|
261
|
+
_ALLOWED_SCHEMES = ("http", "https")
|
|
262
|
+
_FORBIDDEN_PREFIXES = ("javascript:", "data:", "file:", "mailto:", "tel:", "ftp:")
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def normalize_url(url: str, base: Optional[str] = None) -> str:
|
|
266
|
+
"""Return a canonical absolute http(s) URL or raise InvalidURLError.
|
|
267
|
+
|
|
268
|
+
Fragments are dropped, whitespace trimmed, and scheme-less input such as
|
|
269
|
+
``example.com/page`` is promoted to https. ``base`` resolves relative
|
|
270
|
+
links found while parsing a page.
|
|
271
|
+
"""
|
|
272
|
+
|
|
273
|
+
if url is None:
|
|
274
|
+
raise InvalidURLError("URL is required")
|
|
275
|
+
raw = str(url).strip().strip("<>").strip()
|
|
276
|
+
if not raw:
|
|
277
|
+
raise InvalidURLError("URL is empty")
|
|
278
|
+
lowered = raw.lower()
|
|
279
|
+
if lowered.startswith(_FORBIDDEN_PREFIXES):
|
|
280
|
+
raise InvalidURLError(f"unsupported URL scheme: {raw.split(':', 1)[0]}")
|
|
281
|
+
|
|
282
|
+
if base:
|
|
283
|
+
raw = urllib.parse.urljoin(base, raw)
|
|
284
|
+
|
|
285
|
+
parts = urllib.parse.urlsplit(raw)
|
|
286
|
+
if not parts.scheme:
|
|
287
|
+
raw = "https://" + raw.lstrip("/")
|
|
288
|
+
parts = urllib.parse.urlsplit(raw)
|
|
289
|
+
|
|
290
|
+
scheme = parts.scheme.lower()
|
|
291
|
+
if scheme not in _ALLOWED_SCHEMES:
|
|
292
|
+
raise InvalidURLError(f"unsupported URL scheme: {scheme}")
|
|
293
|
+
if not parts.netloc:
|
|
294
|
+
raise InvalidURLError(f"URL has no host: {raw}")
|
|
295
|
+
|
|
296
|
+
# Normalise default ports and strip fragments.
|
|
297
|
+
netloc = parts.netloc
|
|
298
|
+
if (scheme == "http" and netloc.endswith(":80")) or (
|
|
299
|
+
scheme == "https" and netloc.endswith(":443")
|
|
300
|
+
):
|
|
301
|
+
netloc = netloc.rsplit(":", 1)[0]
|
|
302
|
+
|
|
303
|
+
path = parts.path or "/"
|
|
304
|
+
rebuilt = urllib.parse.urlunsplit((scheme, netloc, path, parts.query, ""))
|
|
305
|
+
return rebuilt
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def origin_of(url: str) -> str:
|
|
309
|
+
"""Return ``scheme://host[:port]`` for rate limiting and robots lookup."""
|
|
310
|
+
|
|
311
|
+
parts = urllib.parse.urlsplit(url)
|
|
312
|
+
return f"{parts.scheme.lower()}://{parts.netloc.lower()}"
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def absolute_url(href: Optional[str], base: Optional[str]) -> str:
|
|
316
|
+
"""Best-effort absolute link; returns '' when it cannot be resolved."""
|
|
317
|
+
|
|
318
|
+
if not href:
|
|
319
|
+
return ""
|
|
320
|
+
text = str(href).strip()
|
|
321
|
+
if not text:
|
|
322
|
+
return ""
|
|
323
|
+
lowered = text.lower()
|
|
324
|
+
if lowered.startswith(_FORBIDDEN_PREFIXES):
|
|
325
|
+
return ""
|
|
326
|
+
# Without a base there is nothing to resolve a relative link against.
|
|
327
|
+
if base is None and not (text.startswith("//") or re.match(r"^https?://", text, re.I)):
|
|
328
|
+
return ""
|
|
329
|
+
try:
|
|
330
|
+
return normalize_url(text, base=base)
|
|
331
|
+
except InvalidURLError:
|
|
332
|
+
return ""
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
# ---------------------------------------------------------------------------
|
|
336
|
+
# Private-address guard (SSRF)
|
|
337
|
+
# ---------------------------------------------------------------------------
|
|
338
|
+
|
|
339
|
+
# Loopback, RFC 1918, link-local, CGNAT, documentation, multicast and reserved
|
|
340
|
+
# ranges. ``ipaddress`` flags cover most of them; the explicit list keeps the
|
|
341
|
+
# same answer on every supported Python version.
|
|
342
|
+
_BLOCKED_NETWORKS = tuple(
|
|
343
|
+
ipaddress.ip_network(cidr)
|
|
344
|
+
for cidr in (
|
|
345
|
+
"0.0.0.0/8",
|
|
346
|
+
"10.0.0.0/8",
|
|
347
|
+
"100.64.0.0/10",
|
|
348
|
+
"127.0.0.0/8",
|
|
349
|
+
"169.254.0.0/16",
|
|
350
|
+
"172.16.0.0/12",
|
|
351
|
+
"192.0.0.0/24",
|
|
352
|
+
"192.0.2.0/24",
|
|
353
|
+
"192.168.0.0/16",
|
|
354
|
+
"198.18.0.0/15",
|
|
355
|
+
"198.51.100.0/24",
|
|
356
|
+
"203.0.113.0/24",
|
|
357
|
+
"240.0.0.0/4",
|
|
358
|
+
"255.255.255.255/32",
|
|
359
|
+
"::1/128",
|
|
360
|
+
"64:ff9b::/96",
|
|
361
|
+
"100::/64",
|
|
362
|
+
"2001:db8::/32",
|
|
363
|
+
"fc00::/7",
|
|
364
|
+
"fe80::/10",
|
|
365
|
+
"ff00::/8",
|
|
366
|
+
)
|
|
367
|
+
)
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
class UnsafeAddress(InvalidURLError):
|
|
371
|
+
"""The URL points at a loopback, private or link-local address."""
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def _blocked_ip(ip: Any) -> bool:
|
|
375
|
+
if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None:
|
|
376
|
+
ip = ip.ipv4_mapped
|
|
377
|
+
if any(ip in network for network in _BLOCKED_NETWORKS if network.version == ip.version):
|
|
378
|
+
return True
|
|
379
|
+
return bool(
|
|
380
|
+
ip.is_loopback
|
|
381
|
+
or ip.is_link_local
|
|
382
|
+
or ip.is_private
|
|
383
|
+
or ip.is_multicast
|
|
384
|
+
or ip.is_reserved
|
|
385
|
+
or ip.is_unspecified
|
|
386
|
+
)
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def _host_is_local_name(host: str) -> bool:
|
|
390
|
+
host = host.rstrip(".").lower()
|
|
391
|
+
return (
|
|
392
|
+
host == "localhost"
|
|
393
|
+
or host.endswith(".localhost")
|
|
394
|
+
or host.endswith(".local")
|
|
395
|
+
or host.endswith(".internal")
|
|
396
|
+
or host == "metadata.google.internal"
|
|
397
|
+
)
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def assert_public_url(url: str, *, allow_private: bool = False) -> None:
|
|
401
|
+
"""Refuse URLs that resolve to this machine, the LAN or a metadata service.
|
|
402
|
+
|
|
403
|
+
The host as written and every address DNS returns are checked, so decimal
|
|
404
|
+
IPv4 spellings and IPv6-mapped IPv4 are caught as well. The guard runs
|
|
405
|
+
before each request and on every redirect hop. A host that does not resolve
|
|
406
|
+
is left to the HTTP client, which fails on its own.
|
|
407
|
+
|
|
408
|
+
``allow_private`` exists for local fixtures and trusted intranet tools; it
|
|
409
|
+
is off by default and the CLI exposes it as ``--allow-private``.
|
|
410
|
+
"""
|
|
411
|
+
|
|
412
|
+
if allow_private:
|
|
413
|
+
return
|
|
414
|
+
parts = urllib.parse.urlsplit(url)
|
|
415
|
+
host = parts.hostname or ""
|
|
416
|
+
if not host:
|
|
417
|
+
raise UnsafeAddress(f"URL has no host: {url}")
|
|
418
|
+
if _host_is_local_name(host):
|
|
419
|
+
raise UnsafeAddress(f"refusing a local host: {host}")
|
|
420
|
+
try:
|
|
421
|
+
port = parts.port or (443 if parts.scheme.lower() == "https" else 80)
|
|
422
|
+
except ValueError as exc:
|
|
423
|
+
raise UnsafeAddress(f"invalid port in {url}") from exc
|
|
424
|
+
|
|
425
|
+
try:
|
|
426
|
+
literal = ipaddress.ip_address(host.strip("[]").split("%", 1)[0])
|
|
427
|
+
except ValueError:
|
|
428
|
+
literal = None
|
|
429
|
+
if literal is not None:
|
|
430
|
+
if _blocked_ip(literal):
|
|
431
|
+
raise UnsafeAddress(f"refusing a private address: {host}")
|
|
432
|
+
return
|
|
433
|
+
|
|
434
|
+
try:
|
|
435
|
+
infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
|
|
436
|
+
except OSError:
|
|
437
|
+
return # unresolvable: the HTTP client reports the failure
|
|
438
|
+
for info in infos:
|
|
439
|
+
address = str(info[4][0])
|
|
440
|
+
try:
|
|
441
|
+
ip = ipaddress.ip_address(address.split("%", 1)[0])
|
|
442
|
+
except ValueError:
|
|
443
|
+
continue
|
|
444
|
+
if _blocked_ip(ip):
|
|
445
|
+
raise UnsafeAddress(f"refusing {host}: it resolves to {address}")
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def _guarded_request(
|
|
449
|
+
getter: Any,
|
|
450
|
+
url: str,
|
|
451
|
+
*,
|
|
452
|
+
allow_private: bool = False,
|
|
453
|
+
max_redirects: int = 5,
|
|
454
|
+
**kwargs: Any,
|
|
455
|
+
) -> Any:
|
|
456
|
+
"""GET ``url`` and follow redirects one hop at a time, checking each target.
|
|
457
|
+
|
|
458
|
+
``requests`` follows redirects internally, which would let a public URL
|
|
459
|
+
bounce the client to ``127.0.0.1`` or a metadata endpoint without the guard
|
|
460
|
+
ever seeing it. Every hop is therefore resolved and checked here.
|
|
461
|
+
"""
|
|
462
|
+
|
|
463
|
+
current = url
|
|
464
|
+
for _ in range(max_redirects + 1):
|
|
465
|
+
assert_public_url(current, allow_private=allow_private)
|
|
466
|
+
response = getter(current, allow_redirects=False, **kwargs)
|
|
467
|
+
location = response.headers.get("Location") if response.headers else None
|
|
468
|
+
if response.status_code in (301, 302, 303, 307, 308) and location:
|
|
469
|
+
current = normalize_url(urllib.parse.urljoin(current, location))
|
|
470
|
+
response.close()
|
|
471
|
+
continue
|
|
472
|
+
return response
|
|
473
|
+
raise FetchError(f"too many redirects for {url}")
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
# ---------------------------------------------------------------------------
|
|
477
|
+
# robots.txt (RFC 9309 compatible subset, written from scratch)
|
|
478
|
+
# ---------------------------------------------------------------------------
|
|
479
|
+
|
|
480
|
+
_ROBOTS_CHARSET_RE = re.compile(rb"charset\s*=\s*[\"']?([A-Za-z0-9._:-]+)", re.I)
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
@dataclass(frozen=True)
|
|
484
|
+
class RobotsRule:
|
|
485
|
+
allow: bool
|
|
486
|
+
pattern: str
|
|
487
|
+
regex: re.Pattern
|
|
488
|
+
specificity: int
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
class RobotsPolicy:
|
|
492
|
+
"""An immutable parsed robots.txt policy for one user agent."""
|
|
493
|
+
|
|
494
|
+
def __init__(
|
|
495
|
+
self,
|
|
496
|
+
rules: Iterable[RobotsRule] = (),
|
|
497
|
+
crawl_delay: Optional[float] = None,
|
|
498
|
+
*,
|
|
499
|
+
allow_all: bool = False,
|
|
500
|
+
deny_all: bool = False,
|
|
501
|
+
source: str = "",
|
|
502
|
+
note: str = "",
|
|
503
|
+
) -> None:
|
|
504
|
+
self.rules = tuple(rules)
|
|
505
|
+
self.crawl_delay = crawl_delay
|
|
506
|
+
self.allow_all = allow_all
|
|
507
|
+
self.deny_all = deny_all
|
|
508
|
+
self.source = source
|
|
509
|
+
self.note = note
|
|
510
|
+
|
|
511
|
+
@classmethod
|
|
512
|
+
def allowing_all(cls, source: str = "", note: str = "") -> "RobotsPolicy":
|
|
513
|
+
return cls(allow_all=True, source=source, note=note)
|
|
514
|
+
|
|
515
|
+
@classmethod
|
|
516
|
+
def denying_all(cls, source: str = "", note: str = "") -> "RobotsPolicy":
|
|
517
|
+
return cls(deny_all=True, source=source, note=note)
|
|
518
|
+
|
|
519
|
+
def allowed(self, url: str) -> bool:
|
|
520
|
+
if self.deny_all:
|
|
521
|
+
return False
|
|
522
|
+
if self.allow_all:
|
|
523
|
+
return True
|
|
524
|
+
if not self.rules:
|
|
525
|
+
return True
|
|
526
|
+
|
|
527
|
+
path = _robots_path(url)
|
|
528
|
+
winner: Optional[RobotsRule] = None
|
|
529
|
+
for rule in self.rules:
|
|
530
|
+
if not rule.regex.match(path):
|
|
531
|
+
continue
|
|
532
|
+
if winner is None or rule.specificity > winner.specificity:
|
|
533
|
+
winner = rule
|
|
534
|
+
elif rule.specificity == winner.specificity and rule.allow and not winner.allow:
|
|
535
|
+
# RFC 9309: ties are resolved in favour of Allow.
|
|
536
|
+
winner = rule
|
|
537
|
+
return True if winner is None else winner.allow
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
def _robots_path(url: str) -> str:
|
|
541
|
+
parts = urllib.parse.urlsplit(url)
|
|
542
|
+
path = parts.path or "/"
|
|
543
|
+
if parts.query:
|
|
544
|
+
path += "?" + parts.query
|
|
545
|
+
# robots.txt patterns are conventionally unencoded; compare unencoded text.
|
|
546
|
+
try:
|
|
547
|
+
path = urllib.parse.unquote(path)
|
|
548
|
+
except Exception:
|
|
549
|
+
pass
|
|
550
|
+
return path
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
def _compile_rule(pattern: str) -> re.Pattern:
|
|
554
|
+
"""Translate a robots.txt path pattern into a regular expression."""
|
|
555
|
+
|
|
556
|
+
anchored = pattern.endswith("$")
|
|
557
|
+
core = pattern[:-1] if anchored else pattern
|
|
558
|
+
if not core.startswith(("/", "*")):
|
|
559
|
+
core = "/" + core
|
|
560
|
+
escaped = re.escape(core).replace(r"\*", ".*")
|
|
561
|
+
return re.compile("^" + escaped + ("$" if anchored else ""), re.S)
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def _parse_robots_groups(text: str) -> list[dict[str, Any]]:
|
|
565
|
+
"""Split robots.txt into groups of {agents, rules, delay}."""
|
|
566
|
+
|
|
567
|
+
groups: list[dict[str, Any]] = []
|
|
568
|
+
current: Optional[dict[str, Any]] = None
|
|
569
|
+
|
|
570
|
+
for raw_line in text.splitlines():
|
|
571
|
+
line = raw_line.split("#", 1)[0].strip()
|
|
572
|
+
if not line or ":" not in line:
|
|
573
|
+
continue
|
|
574
|
+
key, _, value = line.partition(":")
|
|
575
|
+
key = key.strip().lower()
|
|
576
|
+
value = value.strip()
|
|
577
|
+
|
|
578
|
+
if key == "user-agent":
|
|
579
|
+
# A user-agent line after rules starts a new group.
|
|
580
|
+
if current is None or current["rules"]:
|
|
581
|
+
current = {"agents": [], "rules": [], "delay": None}
|
|
582
|
+
groups.append(current)
|
|
583
|
+
current["agents"].append(value.lower())
|
|
584
|
+
elif key in ("allow", "disallow"):
|
|
585
|
+
if current is None:
|
|
586
|
+
continue
|
|
587
|
+
if key == "disallow" and value == "":
|
|
588
|
+
# An empty Disallow means "no restriction"; ignore it.
|
|
589
|
+
continue
|
|
590
|
+
current["rules"].append((key == "allow", value))
|
|
591
|
+
elif key == "crawl-delay" and current is not None:
|
|
592
|
+
try:
|
|
593
|
+
delay = float(value)
|
|
594
|
+
except ValueError:
|
|
595
|
+
continue
|
|
596
|
+
if delay >= 0:
|
|
597
|
+
current["delay"] = delay
|
|
598
|
+
|
|
599
|
+
return groups
|
|
600
|
+
|
|
601
|
+
|
|
602
|
+
def _agent_match_score(agent_token: str, user_agent: str) -> int:
|
|
603
|
+
"""Return a match score for a robots.txt user-agent token.
|
|
604
|
+
|
|
605
|
+
``*`` scores 0; a product token found in the request UA scores its length.
|
|
606
|
+
``-1`` means no match.
|
|
607
|
+
"""
|
|
608
|
+
|
|
609
|
+
token = agent_token.strip().lower()
|
|
610
|
+
if not token:
|
|
611
|
+
return -1
|
|
612
|
+
if token == "*":
|
|
613
|
+
return 0
|
|
614
|
+
token = token.split("/", 1)[0].strip()
|
|
615
|
+
if token and token in user_agent.lower():
|
|
616
|
+
return len(token)
|
|
617
|
+
return -1
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
def parse_robots(text: str, user_agent: str, source: str = "") -> RobotsPolicy:
|
|
621
|
+
"""Parse robots.txt text and return the policy for ``user_agent``."""
|
|
622
|
+
|
|
623
|
+
groups = _parse_robots_groups(text)
|
|
624
|
+
best_score = -1
|
|
625
|
+
selected: list[dict[str, Any]] = []
|
|
626
|
+
|
|
627
|
+
for group in groups:
|
|
628
|
+
score = max((_agent_match_score(a, user_agent) for a in group["agents"]), default=-1)
|
|
629
|
+
if score < 0:
|
|
630
|
+
continue
|
|
631
|
+
if score > best_score:
|
|
632
|
+
best_score = score
|
|
633
|
+
selected = [group]
|
|
634
|
+
elif score == best_score:
|
|
635
|
+
selected.append(group)
|
|
636
|
+
|
|
637
|
+
if not selected:
|
|
638
|
+
return RobotsPolicy.allowing_all(source=source, note="no matching user-agent group")
|
|
639
|
+
|
|
640
|
+
rules: list[RobotsRule] = []
|
|
641
|
+
delay: Optional[float] = None
|
|
642
|
+
for group in selected:
|
|
643
|
+
if group["delay"] is not None:
|
|
644
|
+
delay = group["delay"] if delay is None else max(delay, group["delay"])
|
|
645
|
+
for allow, pattern in group["rules"]:
|
|
646
|
+
rules.append(
|
|
647
|
+
RobotsRule(
|
|
648
|
+
allow=allow,
|
|
649
|
+
pattern=pattern,
|
|
650
|
+
regex=_compile_rule(pattern),
|
|
651
|
+
specificity=len(pattern.replace("*", "")),
|
|
652
|
+
)
|
|
653
|
+
)
|
|
654
|
+
|
|
655
|
+
return RobotsPolicy(rules=rules, crawl_delay=delay, source=source)
|
|
656
|
+
|
|
657
|
+
|
|
658
|
+
class RobotsCache:
|
|
659
|
+
"""Fetch robots.txt once per origin and cache the parsed policy."""
|
|
660
|
+
|
|
661
|
+
def __init__(self, config: FetchConfig, ttl: float = 3600.0) -> None:
|
|
662
|
+
self.config = config
|
|
663
|
+
self.ttl = ttl
|
|
664
|
+
self._cache: dict[str, tuple[float, RobotsPolicy]] = {}
|
|
665
|
+
self._lock = threading.Lock()
|
|
666
|
+
|
|
667
|
+
def policy_for(self, url: str) -> RobotsPolicy:
|
|
668
|
+
origin = origin_of(url)
|
|
669
|
+
now = time.monotonic()
|
|
670
|
+
with self._lock:
|
|
671
|
+
hit = self._cache.get(origin)
|
|
672
|
+
if hit and hit[0] > now:
|
|
673
|
+
return hit[1]
|
|
674
|
+
|
|
675
|
+
policy = self._download(origin)
|
|
676
|
+
with self._lock:
|
|
677
|
+
self._cache[origin] = (now + self.ttl, policy)
|
|
678
|
+
return policy
|
|
679
|
+
|
|
680
|
+
def _download(self, origin: str) -> RobotsPolicy:
|
|
681
|
+
robots_url = origin + "/robots.txt"
|
|
682
|
+
headers = {
|
|
683
|
+
"User-Agent": self.config.user_agent,
|
|
684
|
+
"Accept": "text/plain,*/*;q=0.5",
|
|
685
|
+
}
|
|
686
|
+
try:
|
|
687
|
+
response = _guarded_request(
|
|
688
|
+
requests.get,
|
|
689
|
+
robots_url,
|
|
690
|
+
allow_private=self.config.allow_private,
|
|
691
|
+
headers=headers,
|
|
692
|
+
timeout=min(max(self.config.timeout, 1.0), 10.0),
|
|
693
|
+
)
|
|
694
|
+
except requests.RequestException as exc:
|
|
695
|
+
# RFC 9309 §2.3.1: an unreachable robots.txt is treated as a
|
|
696
|
+
# server error, so we take the conservative route and deny.
|
|
697
|
+
return RobotsPolicy.denying_all(
|
|
698
|
+
source=robots_url,
|
|
699
|
+
note=f"robots.txt unreachable ({exc.__class__.__name__}); denied conservatively",
|
|
700
|
+
)
|
|
701
|
+
|
|
702
|
+
status = response.status_code
|
|
703
|
+
if status == 200:
|
|
704
|
+
raw = response.content[:512_000]
|
|
705
|
+
text = _decode_robots_bytes(raw, response)
|
|
706
|
+
return parse_robots(text, self.config.user_agent, source=robots_url)
|
|
707
|
+
if status in (401, 403):
|
|
708
|
+
return RobotsPolicy.denying_all(source=robots_url, note=f"HTTP {status} on robots.txt")
|
|
709
|
+
if 500 <= status < 600:
|
|
710
|
+
return RobotsPolicy.denying_all(
|
|
711
|
+
source=robots_url,
|
|
712
|
+
note=(
|
|
713
|
+
f"HTTP {status} on robots.txt; denied per RFC 9309. The site's "
|
|
714
|
+
"robots.txt is broken, so no page on it can be read. Contact the "
|
|
715
|
+
"site owner, or use --ignore-robots only for resources you are "
|
|
716
|
+
"authorised to access."
|
|
717
|
+
),
|
|
718
|
+
)
|
|
719
|
+
# 404/410 and other 4xx mean "no restrictions".
|
|
720
|
+
return RobotsPolicy.allowing_all(source=robots_url, note=f"HTTP {status} on robots.txt")
|
|
721
|
+
|
|
722
|
+
|
|
723
|
+
def _decode_robots_bytes(raw: bytes, response: Any) -> str:
|
|
724
|
+
encoding = None
|
|
725
|
+
try:
|
|
726
|
+
match = _ROBOTS_CHARSET_RE.search(raw[:512])
|
|
727
|
+
if match:
|
|
728
|
+
encoding = match.group(1).decode("ascii", "ignore")
|
|
729
|
+
except Exception:
|
|
730
|
+
encoding = None
|
|
731
|
+
if not encoding:
|
|
732
|
+
content_type = ""
|
|
733
|
+
try:
|
|
734
|
+
content_type = response.headers.get("Content-Type", "")
|
|
735
|
+
except Exception:
|
|
736
|
+
content_type = ""
|
|
737
|
+
match = re.search(r"charset=([A-Za-z0-9._:-]+)", content_type, re.I)
|
|
738
|
+
if match:
|
|
739
|
+
encoding = match.group(1)
|
|
740
|
+
if not encoding:
|
|
741
|
+
encoding = getattr(response, "encoding", None) or "utf-8"
|
|
742
|
+
try:
|
|
743
|
+
return raw.decode(encoding, errors="replace")
|
|
744
|
+
except (LookupError, TypeError):
|
|
745
|
+
return raw.decode("utf-8", errors="replace")
|
|
746
|
+
|
|
747
|
+
|
|
748
|
+
# ---------------------------------------------------------------------------
|
|
749
|
+
# Politeness
|
|
750
|
+
# ---------------------------------------------------------------------------
|
|
751
|
+
|
|
752
|
+
|
|
753
|
+
class RateLimiter:
|
|
754
|
+
"""Per-origin politeness delay, capped so robots.txt cannot stall a run."""
|
|
755
|
+
|
|
756
|
+
MAX_EXTRA_DELAY = 30.0
|
|
757
|
+
|
|
758
|
+
def __init__(self, delay: float) -> None:
|
|
759
|
+
self.delay = max(0.0, float(delay))
|
|
760
|
+
self._last: dict[str, float] = {}
|
|
761
|
+
self._lock = threading.Lock()
|
|
762
|
+
|
|
763
|
+
def wait(self, url: str, extra_delay: float = 0.0) -> None:
|
|
764
|
+
target = max(self.delay, min(extra_delay, self.MAX_EXTRA_DELAY))
|
|
765
|
+
if target <= 0:
|
|
766
|
+
return
|
|
767
|
+
origin = origin_of(url)
|
|
768
|
+
with self._lock:
|
|
769
|
+
now = time.monotonic()
|
|
770
|
+
last = self._last.get(origin, 0.0)
|
|
771
|
+
pause = max(0.0, last + target - now)
|
|
772
|
+
self._last[origin] = now + pause
|
|
773
|
+
if pause > 0:
|
|
774
|
+
time.sleep(pause)
|
|
775
|
+
|
|
776
|
+
|
|
777
|
+
# ---------------------------------------------------------------------------
|
|
778
|
+
# HTTP fetching
|
|
779
|
+
# ---------------------------------------------------------------------------
|
|
780
|
+
|
|
781
|
+
_HTML_CONTENT_TYPES = (
|
|
782
|
+
"text/html",
|
|
783
|
+
"application/xhtml+xml",
|
|
784
|
+
"application/xml",
|
|
785
|
+
"text/xml",
|
|
786
|
+
"application/rss+xml",
|
|
787
|
+
"application/atom+xml",
|
|
788
|
+
)
|
|
789
|
+
_TEXT_CONTENT_TYPES = ("text/plain", "text/markdown", "text/csv")
|
|
790
|
+
_JSON_CONTENT_TYPES = ("application/json", "application/ld+json")
|
|
791
|
+
_PDF_CONTENT_TYPES = ("application/pdf",)
|
|
792
|
+
_DOCUMENT_CONTENT_TYPES = (
|
|
793
|
+
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
794
|
+
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
795
|
+
"application/epub+zip",
|
|
796
|
+
)
|
|
797
|
+
_DOCUMENT_EXTENSIONS = {
|
|
798
|
+
".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
799
|
+
".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
800
|
+
".epub": "application/epub+zip",
|
|
801
|
+
".csv": "text/csv",
|
|
802
|
+
}
|
|
803
|
+
_FEED_CONTENT_TYPES = ("application/rss+xml", "application/atom+xml", "application/xml", "text/xml")
|
|
804
|
+
|
|
805
|
+
|
|
806
|
+
@dataclass
|
|
807
|
+
class HttpResponse:
|
|
808
|
+
url: str
|
|
809
|
+
final_url: str
|
|
810
|
+
status: int
|
|
811
|
+
headers: dict[str, str]
|
|
812
|
+
content: bytes
|
|
813
|
+
text: str
|
|
814
|
+
encoding: str
|
|
815
|
+
content_type: str
|
|
816
|
+
from_cache: bool = False
|
|
817
|
+
|
|
818
|
+
|
|
819
|
+
def _cache_root(config: FetchConfig) -> Path:
|
|
820
|
+
if config.cache_dir:
|
|
821
|
+
return Path(config.cache_dir).expanduser()
|
|
822
|
+
return Path.home() / ".web-forager" / "cache"
|
|
823
|
+
|
|
824
|
+
|
|
825
|
+
def _cache_key(url: str) -> str:
|
|
826
|
+
import hashlib
|
|
827
|
+
|
|
828
|
+
return hashlib.sha256(url.encode("utf-8")).hexdigest()
|
|
829
|
+
|
|
830
|
+
|
|
831
|
+
def decode_js_redirect(text: str) -> str:
|
|
832
|
+
"""Read the target URL out of a search engine's ``/link`` interstitial.
|
|
833
|
+
|
|
834
|
+
Two shapes exist. One writes the address literally
|
|
835
|
+
(``window.location.replace('https://...')``). The other — used by
|
|
836
|
+
weixin.sogou.com — assembles it from ``url += '...'`` chunks so that the
|
|
837
|
+
address never appears in the HTML as a whole; concatenating the chunks
|
|
838
|
+
recovers it without running any JavaScript.
|
|
839
|
+
"""
|
|
840
|
+
|
|
841
|
+
if not text:
|
|
842
|
+
return ""
|
|
843
|
+
chunks = re.findall(r"""url\s*\+=\s*['"]([^'"]*)['"]""", text)
|
|
844
|
+
if chunks:
|
|
845
|
+
joined = "".join(chunks)
|
|
846
|
+
if joined.startswith("http"):
|
|
847
|
+
return joined
|
|
848
|
+
match = re.search(r"""window\.location\.replace\(\s*['"]([^'"]+)""", text)
|
|
849
|
+
if not match:
|
|
850
|
+
match = re.search(r"""URL\s*=\s*['"]([^'"]+)""", text)
|
|
851
|
+
return match.group(1) if match else ""
|
|
852
|
+
|
|
853
|
+
|
|
854
|
+
class HttpFetcher:
|
|
855
|
+
"""A polite, robots-aware HTTP client built on requests."""
|
|
856
|
+
|
|
857
|
+
def __init__(self, config: FetchConfig) -> None:
|
|
858
|
+
self.config = config
|
|
859
|
+
self.session = requests.Session()
|
|
860
|
+
self.session.max_redirects = 5
|
|
861
|
+
self.session.headers.update(
|
|
862
|
+
{
|
|
863
|
+
"User-Agent": config.user_agent,
|
|
864
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,"
|
|
865
|
+
"text/plain;q=0.8,application/json;q=0.7,*/*;q=0.5",
|
|
866
|
+
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
|
867
|
+
"Accept-Encoding": "gzip, deflate",
|
|
868
|
+
}
|
|
869
|
+
)
|
|
870
|
+
if config.proxy:
|
|
871
|
+
# An explicit proxy wins over the environment; requests itself still
|
|
872
|
+
# honours HTTPS_PROXY / ALL_PROXY when this is empty.
|
|
873
|
+
self.session.proxies.update({"http": config.proxy, "https": config.proxy})
|
|
874
|
+
self.robots = RobotsCache(config)
|
|
875
|
+
self.limiter = RateLimiter(config.delay)
|
|
876
|
+
|
|
877
|
+
def close(self) -> None:
|
|
878
|
+
try:
|
|
879
|
+
self.session.close()
|
|
880
|
+
except Exception:
|
|
881
|
+
pass
|
|
882
|
+
|
|
883
|
+
# -- robots ---------------------------------------------------------
|
|
884
|
+
|
|
885
|
+
def assert_allowed(self, url: str) -> RobotsPolicy:
|
|
886
|
+
"""Raise RobotsDenied when policy is ``respect`` and robots forbids it.
|
|
887
|
+
|
|
888
|
+
``warn`` returns the denying policy instead of raising, so the caller
|
|
889
|
+
can still read the page and must record ``robots_overridden: true``.
|
|
890
|
+
"""
|
|
891
|
+
|
|
892
|
+
if not self.config.respect_robots or self.config.robots_policy == "ignore":
|
|
893
|
+
return RobotsPolicy.allowing_all(note="robots checking disabled by policy")
|
|
894
|
+
policy = self.robots.policy_for(url)
|
|
895
|
+
if not policy.allowed(url):
|
|
896
|
+
if self.config.robots_policy == "warn":
|
|
897
|
+
return policy
|
|
898
|
+
detail = policy.note or policy.source or "robots.txt"
|
|
899
|
+
raise RobotsDenied(f"robots.txt disallows {url} ({detail})")
|
|
900
|
+
return policy
|
|
901
|
+
|
|
902
|
+
# -- main entry -----------------------------------------------------
|
|
903
|
+
|
|
904
|
+
def fetch(
|
|
905
|
+
self, url: str, *, check_robots: bool = True, use_cache: bool = True
|
|
906
|
+
) -> HttpResponse:
|
|
907
|
+
"""Fetch one URL.
|
|
908
|
+
|
|
909
|
+
``use_cache=False`` skips both lookup and write. A bootstrap request
|
|
910
|
+
whose only job is to set cookies must not be answered from the cache:
|
|
911
|
+
a cached anti-bot page would silently supply no cookie at all.
|
|
912
|
+
"""
|
|
913
|
+
|
|
914
|
+
url = normalize_url(url)
|
|
915
|
+
assert_public_url(url, allow_private=self.config.allow_private)
|
|
916
|
+
policy: Optional[RobotsPolicy] = None
|
|
917
|
+
if check_robots:
|
|
918
|
+
policy = self.assert_allowed(url)
|
|
919
|
+
|
|
920
|
+
cached = self._read_cache(url) if use_cache else None
|
|
921
|
+
if cached is not None:
|
|
922
|
+
return cached
|
|
923
|
+
|
|
924
|
+
extra = policy.crawl_delay if policy and policy.crawl_delay else 0.0
|
|
925
|
+
self.limiter.wait(url, extra)
|
|
926
|
+
|
|
927
|
+
last_error: Optional[Exception] = None
|
|
928
|
+
for attempt in range(self.config.retries + 1):
|
|
929
|
+
try:
|
|
930
|
+
response = _guarded_request(
|
|
931
|
+
self.session.get,
|
|
932
|
+
url,
|
|
933
|
+
allow_private=self.config.allow_private,
|
|
934
|
+
max_redirects=self.session.max_redirects,
|
|
935
|
+
timeout=self.config.timeout,
|
|
936
|
+
stream=True,
|
|
937
|
+
)
|
|
938
|
+
except requests.RequestException as exc:
|
|
939
|
+
last_error = exc
|
|
940
|
+
if attempt >= self.config.retries:
|
|
941
|
+
break
|
|
942
|
+
time.sleep(min(0.5 * (2**attempt), 4.0))
|
|
943
|
+
continue
|
|
944
|
+
|
|
945
|
+
status = response.status_code
|
|
946
|
+
if status in (429, 500, 502, 503, 504) and attempt < self.config.retries:
|
|
947
|
+
retry_after = _retry_after_seconds(response.headers.get("Retry-After"))
|
|
948
|
+
response.close()
|
|
949
|
+
time.sleep(retry_after if retry_after is not None else min(1.0 * (2**attempt), 8.0))
|
|
950
|
+
continue
|
|
951
|
+
|
|
952
|
+
if status >= 400:
|
|
953
|
+
response.close()
|
|
954
|
+
raise FetchError(f"HTTP {status} for {url}")
|
|
955
|
+
|
|
956
|
+
result = self._read_response(response, url)
|
|
957
|
+
if use_cache:
|
|
958
|
+
self._write_cache(url, result)
|
|
959
|
+
return result
|
|
960
|
+
|
|
961
|
+
raise FetchError(f"request failed for {url}: {last_error}") from last_error
|
|
962
|
+
|
|
963
|
+
# -- on-disk response cache ----------------------------------------
|
|
964
|
+
|
|
965
|
+
def _cache_paths(self, url: str) -> tuple[Path, Path]:
|
|
966
|
+
root = _cache_root(self.config)
|
|
967
|
+
key = _cache_key(url)
|
|
968
|
+
return root / f"{key}.json", root / f"{key}.bin"
|
|
969
|
+
|
|
970
|
+
def _read_cache(self, url: str) -> Optional[HttpResponse]:
|
|
971
|
+
if not self.config.cache or self.config.cache_ttl <= 0:
|
|
972
|
+
return None
|
|
973
|
+
meta_path, body_path = self._cache_paths(url)
|
|
974
|
+
try:
|
|
975
|
+
if not meta_path.is_file() or not body_path.is_file():
|
|
976
|
+
return None
|
|
977
|
+
age = time.time() - meta_path.stat().st_mtime
|
|
978
|
+
if age > self.config.cache_ttl:
|
|
979
|
+
return None
|
|
980
|
+
meta = json.loads(meta_path.read_text(encoding="utf-8"))
|
|
981
|
+
body = body_path.read_bytes()
|
|
982
|
+
except (OSError, json.JSONDecodeError):
|
|
983
|
+
return None
|
|
984
|
+
content_type = str(meta.get("content_type") or "text/html")
|
|
985
|
+
encoding = str(meta.get("encoding") or "utf-8")
|
|
986
|
+
text = body.decode(encoding, errors="replace")
|
|
987
|
+
log(f"cache hit: {url}", self.config)
|
|
988
|
+
return HttpResponse(
|
|
989
|
+
url=url,
|
|
990
|
+
final_url=str(meta.get("final_url") or url),
|
|
991
|
+
status=int(meta.get("status") or 200),
|
|
992
|
+
headers={str(k): str(v) for k, v in (meta.get("headers") or {}).items()},
|
|
993
|
+
content=body,
|
|
994
|
+
text=text,
|
|
995
|
+
encoding=encoding,
|
|
996
|
+
content_type=content_type,
|
|
997
|
+
from_cache=True,
|
|
998
|
+
)
|
|
999
|
+
|
|
1000
|
+
def _write_cache(self, url: str, response: HttpResponse) -> None:
|
|
1001
|
+
if not self.config.cache or response.status != 200:
|
|
1002
|
+
return
|
|
1003
|
+
lowered = {str(key).lower() for key in response.headers}
|
|
1004
|
+
if lowered & {"set-cookie", "set-cookie2"}:
|
|
1005
|
+
# A response that plants a session cookie is not a neutral copy of a
|
|
1006
|
+
# public page; keep it (and the credential) out of the on-disk cache.
|
|
1007
|
+
log("cache skipped: response sets cookies", self.config)
|
|
1008
|
+
return
|
|
1009
|
+
safe_headers = {
|
|
1010
|
+
key: value
|
|
1011
|
+
for key, value in response.headers.items()
|
|
1012
|
+
if str(key).lower() not in ("authorization", "proxy-authorization")
|
|
1013
|
+
}
|
|
1014
|
+
meta_path, body_path = self._cache_paths(url)
|
|
1015
|
+
try:
|
|
1016
|
+
meta_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1017
|
+
body_path.write_bytes(response.content)
|
|
1018
|
+
meta_path.write_text(
|
|
1019
|
+
json.dumps(
|
|
1020
|
+
{
|
|
1021
|
+
"url": url,
|
|
1022
|
+
"final_url": response.final_url,
|
|
1023
|
+
"status": response.status,
|
|
1024
|
+
"headers": safe_headers,
|
|
1025
|
+
"content_type": response.content_type,
|
|
1026
|
+
"encoding": response.encoding,
|
|
1027
|
+
},
|
|
1028
|
+
ensure_ascii=False,
|
|
1029
|
+
),
|
|
1030
|
+
encoding="utf-8",
|
|
1031
|
+
)
|
|
1032
|
+
except OSError as exc: # a cache failure must never fail the fetch
|
|
1033
|
+
log(f"cache write skipped ({exc})", self.config)
|
|
1034
|
+
|
|
1035
|
+
def _read_response(self, response: requests.Response, requested_url: str) -> HttpResponse:
|
|
1036
|
+
content_type = (response.headers.get("Content-Type") or "").split(";", 1)[0].strip().lower()
|
|
1037
|
+
path = requested_url.lower().split("?", 1)[0]
|
|
1038
|
+
if content_type == "application/octet-stream":
|
|
1039
|
+
for extension, mapped in _DOCUMENT_EXTENSIONS.items():
|
|
1040
|
+
if path.endswith(extension):
|
|
1041
|
+
content_type = mapped
|
|
1042
|
+
break
|
|
1043
|
+
else:
|
|
1044
|
+
if path.endswith(".pdf"):
|
|
1045
|
+
content_type = "application/pdf"
|
|
1046
|
+
if content_type and not _is_supported_content_type(content_type):
|
|
1047
|
+
response.close()
|
|
1048
|
+
raise FetchError(
|
|
1049
|
+
f"unsupported content type '{content_type}' for {requested_url}; "
|
|
1050
|
+
"web-forager handles HTML, XML/feeds, plain text, CSV, JSON, PDF "
|
|
1051
|
+
"and DOCX/XLSX/EPUB only"
|
|
1052
|
+
)
|
|
1053
|
+
|
|
1054
|
+
chunks: list[bytes] = []
|
|
1055
|
+
total = 0
|
|
1056
|
+
truncated = False
|
|
1057
|
+
try:
|
|
1058
|
+
for chunk in response.iter_content(chunk_size=65_536):
|
|
1059
|
+
if not chunk:
|
|
1060
|
+
continue
|
|
1061
|
+
chunks.append(chunk)
|
|
1062
|
+
total += len(chunk)
|
|
1063
|
+
if total >= self.config.max_bytes:
|
|
1064
|
+
truncated = True
|
|
1065
|
+
break
|
|
1066
|
+
finally:
|
|
1067
|
+
response.close()
|
|
1068
|
+
|
|
1069
|
+
raw = b"".join(chunks)[: self.config.max_bytes]
|
|
1070
|
+
if raw[:5] == b"%PDF-":
|
|
1071
|
+
content_type = "application/pdf"
|
|
1072
|
+
if content_type in ("application/octet-stream", "binary/octet-stream"):
|
|
1073
|
+
if raw[:4] == b"PK\x03\x04":
|
|
1074
|
+
pass # DOCX/XLSX/EPUB is detected after reading
|
|
1075
|
+
elif not _looks_like_text(raw):
|
|
1076
|
+
response.close()
|
|
1077
|
+
raise FetchError(
|
|
1078
|
+
f"unsupported binary payload for {requested_url}; "
|
|
1079
|
+
"web-forager handles text, HTML, XML/feeds, JSON, PDF and "
|
|
1080
|
+
"DOCX/XLSX/EPUB only"
|
|
1081
|
+
)
|
|
1082
|
+
encoding = _detect_encoding(raw, content_type, response.encoding)
|
|
1083
|
+
text = raw.decode(encoding, errors="replace")
|
|
1084
|
+
if truncated:
|
|
1085
|
+
text += "\n<!-- web-forager: response truncated at max_bytes -->"
|
|
1086
|
+
|
|
1087
|
+
headers = {k.lower(): v for k, v in response.headers.items()}
|
|
1088
|
+
return HttpResponse(
|
|
1089
|
+
url=requested_url,
|
|
1090
|
+
final_url=response.url,
|
|
1091
|
+
status=response.status_code,
|
|
1092
|
+
headers=headers,
|
|
1093
|
+
content=raw,
|
|
1094
|
+
text=text,
|
|
1095
|
+
encoding=encoding,
|
|
1096
|
+
content_type=content_type or "text/html",
|
|
1097
|
+
)
|
|
1098
|
+
|
|
1099
|
+
def resolve(self, url: str) -> str:
|
|
1100
|
+
"""Follow redirects cheaply and decode Sogou's JS ``/link`` hop.
|
|
1101
|
+
|
|
1102
|
+
Used only for search-result URLs; the returned URL is still passed
|
|
1103
|
+
through robots.txt before its content is fetched.
|
|
1104
|
+
"""
|
|
1105
|
+
|
|
1106
|
+
url = normalize_url(url)
|
|
1107
|
+
self.limiter.wait(url)
|
|
1108
|
+
response: Optional[requests.Response] = None
|
|
1109
|
+
try:
|
|
1110
|
+
response = _guarded_request(
|
|
1111
|
+
self.session.get,
|
|
1112
|
+
url,
|
|
1113
|
+
allow_private=self.config.allow_private,
|
|
1114
|
+
max_redirects=self.session.max_redirects,
|
|
1115
|
+
timeout=self.config.timeout,
|
|
1116
|
+
stream=True,
|
|
1117
|
+
)
|
|
1118
|
+
final = response.url
|
|
1119
|
+
text = ""
|
|
1120
|
+
if response.status_code == 200:
|
|
1121
|
+
try:
|
|
1122
|
+
text = response.content[:4096].decode("utf-8", "ignore")
|
|
1123
|
+
except Exception:
|
|
1124
|
+
text = ""
|
|
1125
|
+
is_js_redirect = bool(
|
|
1126
|
+
text and ("sogou.com/link" in final or "window.location.replace" in text)
|
|
1127
|
+
)
|
|
1128
|
+
if is_js_redirect:
|
|
1129
|
+
final = decode_js_redirect(text) or final
|
|
1130
|
+
return final
|
|
1131
|
+
except (requests.RequestException, InvalidURLError):
|
|
1132
|
+
return url
|
|
1133
|
+
finally:
|
|
1134
|
+
if response is not None:
|
|
1135
|
+
try:
|
|
1136
|
+
response.close()
|
|
1137
|
+
except Exception:
|
|
1138
|
+
pass
|
|
1139
|
+
|
|
1140
|
+
|
|
1141
|
+
def _is_supported_content_type(content_type: str) -> bool:
|
|
1142
|
+
if (
|
|
1143
|
+
content_type in _JSON_CONTENT_TYPES
|
|
1144
|
+
or content_type in _PDF_CONTENT_TYPES
|
|
1145
|
+
or content_type in _DOCUMENT_CONTENT_TYPES
|
|
1146
|
+
or content_type in ("application/octet-stream", "binary/octet-stream")
|
|
1147
|
+
):
|
|
1148
|
+
return True
|
|
1149
|
+
return any(content_type.startswith(prefix) for prefix in _HTML_CONTENT_TYPES + _TEXT_CONTENT_TYPES)
|
|
1150
|
+
|
|
1151
|
+
|
|
1152
|
+
def _looks_like_text(raw: bytes) -> bool:
|
|
1153
|
+
sample = raw[:4096]
|
|
1154
|
+
if not sample:
|
|
1155
|
+
return True
|
|
1156
|
+
if b"\x00" in sample:
|
|
1157
|
+
return False
|
|
1158
|
+
controls = sum(1 for byte in sample if byte < 9 or 13 < byte < 32)
|
|
1159
|
+
return controls / len(sample) < 0.02
|
|
1160
|
+
|
|
1161
|
+
|
|
1162
|
+
def _retry_after_seconds(value: Optional[str]) -> Optional[float]:
|
|
1163
|
+
if not value:
|
|
1164
|
+
return None
|
|
1165
|
+
try:
|
|
1166
|
+
return max(0.0, min(float(value), 10.0))
|
|
1167
|
+
except (TypeError, ValueError):
|
|
1168
|
+
return None
|
|
1169
|
+
|
|
1170
|
+
|
|
1171
|
+
_META_CHARSET_RE = re.compile(
|
|
1172
|
+
rb"""<meta[^>]+charset\s*=\s*["']?\s*([A-Za-z0-9._:-]+)""", re.I
|
|
1173
|
+
)
|
|
1174
|
+
|
|
1175
|
+
|
|
1176
|
+
def _detect_encoding(raw: bytes, content_type: str, header_encoding: Optional[str]) -> str:
|
|
1177
|
+
# 1. An explicit charset in Content-Type always wins.
|
|
1178
|
+
match = re.search(r"charset=([A-Za-z0-9._:-]+)", content_type or "", re.I)
|
|
1179
|
+
if match:
|
|
1180
|
+
return match.group(1)
|
|
1181
|
+
# 2. A <meta charset> declaration beats requests' RFC 2616 fallback of
|
|
1182
|
+
# ISO-8859-1 for text/* responses without a charset parameter.
|
|
1183
|
+
match = _META_CHARSET_RE.search(raw[:4096])
|
|
1184
|
+
if match:
|
|
1185
|
+
try:
|
|
1186
|
+
return match.group(1).decode("ascii", "ignore")
|
|
1187
|
+
except Exception:
|
|
1188
|
+
pass
|
|
1189
|
+
# 3. Validate a UTF-8 sample, then common CJK encodings.
|
|
1190
|
+
sample = raw[:4096]
|
|
1191
|
+
try:
|
|
1192
|
+
sample.decode("utf-8")
|
|
1193
|
+
return "utf-8"
|
|
1194
|
+
except UnicodeDecodeError:
|
|
1195
|
+
pass
|
|
1196
|
+
for candidate in ("gb18030", "big5"):
|
|
1197
|
+
try:
|
|
1198
|
+
sample.decode(candidate)
|
|
1199
|
+
return candidate
|
|
1200
|
+
except (UnicodeDecodeError, LookupError):
|
|
1201
|
+
continue
|
|
1202
|
+
if header_encoding and header_encoding.lower() not in ("iso-8859-1", "latin-1"):
|
|
1203
|
+
return header_encoding
|
|
1204
|
+
return "utf-8"
|
|
1205
|
+
|
|
1206
|
+
|
|
1207
|
+
def decode_bytes(data: bytes, content_type: str = "") -> str:
|
|
1208
|
+
"""Decode a local HTML/text file using the same heuristics as the fetcher."""
|
|
1209
|
+
|
|
1210
|
+
return data.decode(_detect_encoding(data, content_type, None), errors="replace")
|
|
1211
|
+
|
|
1212
|
+
|
|
1213
|
+
# ---------------------------------------------------------------------------
|
|
1214
|
+
# Search back-ends (public result pages; parsed from scratch)
|
|
1215
|
+
# ---------------------------------------------------------------------------
|
|
1216
|
+
|
|
1217
|
+
_BING_RSS = "https://www.bing.com/search?q={query}&format=rss&count={count}&first={first}"
|
|
1218
|
+
_BING_HTML = (
|
|
1219
|
+
"https://www.bing.com/search?q={query}&count={count}&first={first}"
|
|
1220
|
+
"&setlang=zh-hans&mkt=zh-CN"
|
|
1221
|
+
)
|
|
1222
|
+
_DDG_HTML = "https://html.duckduckgo.com/html/?q={query}"
|
|
1223
|
+
_SO_360 = "https://www.so.com/s?q={query}&pn={page}"
|
|
1224
|
+
_SOGOU = "https://www.sogou.com/web?query={query}&page={page}"
|
|
1225
|
+
_WEIXIN = "https://weixin.sogou.com/weixin?type=2&query={query}&page={page}"
|
|
1226
|
+
|
|
1227
|
+
SEARCH_ENGINES = ("auto", "all", "360", "bing", "sogou", "weixin", "duckduckgo")
|
|
1228
|
+
|
|
1229
|
+
_360_INTERNAL_HOSTS = (
|
|
1230
|
+
"so.com",
|
|
1231
|
+
"360kan.com",
|
|
1232
|
+
"hao.360.com",
|
|
1233
|
+
"e.so.com",
|
|
1234
|
+
"image.so.com",
|
|
1235
|
+
"tv.360kan.com",
|
|
1236
|
+
)
|
|
1237
|
+
_SOGOU_BLOCK_MARKERS = ("antispider", "安全验证", "人机验证", "验证码")
|
|
1238
|
+
|
|
1239
|
+
|
|
1240
|
+
_CJK_RE = re.compile(r"[㐀-䶿一-鿿豈--ヿ가-]")
|
|
1241
|
+
_USEFUL_TITLE_RE = re.compile(r"[\w㐀-䶿一-鿿]")
|
|
1242
|
+
_MOJIBAKE_RE = re.compile(r"[�\u0000-\u0008\u000b\u000c\u000e-\u001f]")
|
|
1243
|
+
|
|
1244
|
+
|
|
1245
|
+
def looks_garbled(text: str, threshold: float = 0.2) -> bool:
|
|
1246
|
+
"""True when text is mostly replacement or control characters.
|
|
1247
|
+
|
|
1248
|
+
This catches an engine handing back an anti-bot page instead of results.
|
|
1249
|
+
Real mojibake (UTF-8 read as Latin-1) is not caught here: it looks like
|
|
1250
|
+
ordinary punctuation to a character-class check.
|
|
1251
|
+
"""
|
|
1252
|
+
|
|
1253
|
+
value = str(text or "")
|
|
1254
|
+
if not value:
|
|
1255
|
+
return False
|
|
1256
|
+
return len(_MOJIBAKE_RE.findall(value)) / len(value) >= threshold
|
|
1257
|
+
|
|
1258
|
+
|
|
1259
|
+
_CJK_CHAR_RE = re.compile(r"[㐀-䶿一-鿿]")
|
|
1260
|
+
|
|
1261
|
+
_CHARACTER_PAGE_MARKERS = (
|
|
1262
|
+
"百科",
|
|
1263
|
+
"字典",
|
|
1264
|
+
"词典",
|
|
1265
|
+
"拼音",
|
|
1266
|
+
"笔顺",
|
|
1267
|
+
"笔画",
|
|
1268
|
+
"部首",
|
|
1269
|
+
"读音",
|
|
1270
|
+
"组词",
|
|
1271
|
+
"汉字",
|
|
1272
|
+
"汉语",
|
|
1273
|
+
"释义",
|
|
1274
|
+
"字义",
|
|
1275
|
+
"意思",
|
|
1276
|
+
"解释",
|
|
1277
|
+
)
|
|
1278
|
+
|
|
1279
|
+
|
|
1280
|
+
def looks_like_character_page(title: str, query: str) -> bool:
|
|
1281
|
+
"""True for a dictionary entry that shares only one character with the query.
|
|
1282
|
+
|
|
1283
|
+
Engines that split a Chinese phrase into characters rank the dictionary
|
|
1284
|
+
page of its first character — ``反(汉语汉字)_百度百科`` for a query about
|
|
1285
|
+
反逆向工程. That page shares a single character with the query and is never
|
|
1286
|
+
the one the caller asked for, so it is dropped instead of padding the list.
|
|
1287
|
+
"""
|
|
1288
|
+
|
|
1289
|
+
query_chars = set(_CJK_CHAR_RE.findall(query or ""))
|
|
1290
|
+
if len(query_chars) < 2:
|
|
1291
|
+
return False
|
|
1292
|
+
title_text = title or ""
|
|
1293
|
+
title_chars = set(_CJK_CHAR_RE.findall(title_text))
|
|
1294
|
+
if len(title_chars & query_chars) != 1:
|
|
1295
|
+
return False
|
|
1296
|
+
return any(marker in title_text for marker in _CHARACTER_PAGE_MARKERS)
|
|
1297
|
+
|
|
1298
|
+
|
|
1299
|
+
_DICTIONARY_MARKERS = (
|
|
1300
|
+
"词典",
|
|
1301
|
+
"翻译",
|
|
1302
|
+
"音标",
|
|
1303
|
+
"是什么意思",
|
|
1304
|
+
"英语单词",
|
|
1305
|
+
"汉语词典",
|
|
1306
|
+
"dictionary",
|
|
1307
|
+
"translation",
|
|
1308
|
+
)
|
|
1309
|
+
|
|
1310
|
+
|
|
1311
|
+
def looks_like_dictionary_page(result: dict[str, str], query: str) -> bool:
|
|
1312
|
+
"""True for a word page that matches only one token of a longer query.
|
|
1313
|
+
|
|
1314
|
+
Bing occasionally answers a Latin phrase with the dictionary entry of its
|
|
1315
|
+
first word (``reinforcement`` for ``reinforcement learning survey``). The
|
|
1316
|
+
page is not wrong, but it is not an answer to the query; dropping it lets
|
|
1317
|
+
the next result through instead of filling the list with word pages.
|
|
1318
|
+
"""
|
|
1319
|
+
|
|
1320
|
+
tokens = _query_tokens(query)
|
|
1321
|
+
if len(tokens) < 2:
|
|
1322
|
+
return False
|
|
1323
|
+
title = (result.get("title") or "").lower()
|
|
1324
|
+
if not any(marker in title for marker in _DICTIONARY_MARKERS):
|
|
1325
|
+
return False
|
|
1326
|
+
return result_coverage(result, tokens) <= 1 / len(tokens)
|
|
1327
|
+
|
|
1328
|
+
|
|
1329
|
+
_DOXXING_PHRASES = (
|
|
1330
|
+
"人肉搜索",
|
|
1331
|
+
"人肉",
|
|
1332
|
+
"开盒",
|
|
1333
|
+
"社工库",
|
|
1334
|
+
"查户籍",
|
|
1335
|
+
"户籍信息",
|
|
1336
|
+
"开房记录",
|
|
1337
|
+
"查开房",
|
|
1338
|
+
"家庭住址",
|
|
1339
|
+
"具体住址",
|
|
1340
|
+
"亲属关系",
|
|
1341
|
+
"查身份证",
|
|
1342
|
+
"查手机号",
|
|
1343
|
+
"查住址",
|
|
1344
|
+
"查家人",
|
|
1345
|
+
"查个人资料",
|
|
1346
|
+
"定位某人",
|
|
1347
|
+
)
|
|
1348
|
+
_DOXXING_LATIN = ("doxxing", "doxx", "doxed", "doxing", "dox ")
|
|
1349
|
+
_SENSITIVE_ATTRIBUTES = (
|
|
1350
|
+
"手机号",
|
|
1351
|
+
"手机号码",
|
|
1352
|
+
"电话号码",
|
|
1353
|
+
"住址",
|
|
1354
|
+
"身份证",
|
|
1355
|
+
"照片",
|
|
1356
|
+
"微信号",
|
|
1357
|
+
"学号",
|
|
1358
|
+
"工号",
|
|
1359
|
+
"行踪",
|
|
1360
|
+
"轨迹",
|
|
1361
|
+
)
|
|
1362
|
+
_PERSON_REFERENCES = (
|
|
1363
|
+
"这个人",
|
|
1364
|
+
"此人",
|
|
1365
|
+
"该人",
|
|
1366
|
+
"某人",
|
|
1367
|
+
"对方",
|
|
1368
|
+
"本人",
|
|
1369
|
+
"他的",
|
|
1370
|
+
"她的",
|
|
1371
|
+
"家人",
|
|
1372
|
+
"家属",
|
|
1373
|
+
)
|
|
1374
|
+
_LATIN_PERSON_RE = re.compile(r"\b[A-Z][a-z]+(?: [A-Z][a-z]+){1,2}\b")
|
|
1375
|
+
_LATIN_SENSITIVE = ("phone number", "home address", "email address", "social security", "id number")
|
|
1376
|
+
_PERSON_NAME_RE = re.compile(r"^[一-鿿]{2,3}$")
|
|
1377
|
+
_LATIN_NAME_RE = re.compile(r"^[A-Z][a-z]+(?: [A-Z][a-z]+){1,2}$")
|
|
1378
|
+
|
|
1379
|
+
|
|
1380
|
+
def doxxing_reason(query: str) -> str:
|
|
1381
|
+
"""Return a reason when a query asks to identify a private person, else "".
|
|
1382
|
+
|
|
1383
|
+
The check refuses *intent*, not names: an explicit doxxing phrase, a
|
|
1384
|
+
sensitive attribute asked about a specific person, or a Latin personal name
|
|
1385
|
+
combined with a contact field. A plain name on its own is not refused,
|
|
1386
|
+
because it cannot be told apart from a paper author or a public figure; it
|
|
1387
|
+
draws a warning instead.
|
|
1388
|
+
"""
|
|
1389
|
+
|
|
1390
|
+
text = (query or "").lower()
|
|
1391
|
+
for phrase in _DOXXING_PHRASES:
|
|
1392
|
+
if phrase in text:
|
|
1393
|
+
return (
|
|
1394
|
+
"the query asks for information about a person in a way that "
|
|
1395
|
+
"amounts to doxxing or profiling"
|
|
1396
|
+
)
|
|
1397
|
+
for word in _DOXXING_LATIN:
|
|
1398
|
+
if word in text:
|
|
1399
|
+
return "the query names doxxing as its purpose"
|
|
1400
|
+
if any(word in text for word in _LATIN_SENSITIVE) and _LATIN_PERSON_RE.search(query or ""):
|
|
1401
|
+
return "the query asks for contact details about a named individual"
|
|
1402
|
+
if any(word in text for word in _SENSITIVE_ATTRIBUTES):
|
|
1403
|
+
if any(ref in text for ref in _PERSON_REFERENCES):
|
|
1404
|
+
return "the query asks for a sensitive attribute of a specific person"
|
|
1405
|
+
# A short Chinese token beside an attribute word is a name: 王五 身份证.
|
|
1406
|
+
for token in re.split(r"[\s,,;;|/]+", query or ""):
|
|
1407
|
+
if not _PERSON_NAME_RE.match(token):
|
|
1408
|
+
continue
|
|
1409
|
+
if any(token in word or word in token for word in _SENSITIVE_ATTRIBUTES):
|
|
1410
|
+
continue
|
|
1411
|
+
return "the query pairs a person's name with a sensitive attribute"
|
|
1412
|
+
return ""
|
|
1413
|
+
|
|
1414
|
+
|
|
1415
|
+
def looks_like_person_name(query: str) -> bool:
|
|
1416
|
+
"""True for a bare personal name, which draws a warning but is not refused."""
|
|
1417
|
+
|
|
1418
|
+
text = (query or "").strip()
|
|
1419
|
+
if not text:
|
|
1420
|
+
return False
|
|
1421
|
+
return bool(_PERSON_NAME_RE.match(text) or _LATIN_NAME_RE.match(text))
|
|
1422
|
+
|
|
1423
|
+
|
|
1424
|
+
def prefers_chinese_engines(query: str) -> bool:
|
|
1425
|
+
"""True when a query is mostly CJK text.
|
|
1426
|
+
|
|
1427
|
+
Such queries rank better on 360 and Sogou; Bing's tokeniser treats quoted
|
|
1428
|
+
Chinese phrases as single characters, which buries the precise hits.
|
|
1429
|
+
"""
|
|
1430
|
+
|
|
1431
|
+
text = str(query or "")
|
|
1432
|
+
if not text:
|
|
1433
|
+
return False
|
|
1434
|
+
cjk = len(_CJK_RE.findall(text))
|
|
1435
|
+
if not cjk:
|
|
1436
|
+
return False
|
|
1437
|
+
return cjk >= len(re.findall(r"[A-Za-z]", text))
|
|
1438
|
+
|
|
1439
|
+
|
|
1440
|
+
def search(
|
|
1441
|
+
query: str,
|
|
1442
|
+
config: FetchConfig,
|
|
1443
|
+
engine: str = "auto",
|
|
1444
|
+
limit: Optional[int] = None,
|
|
1445
|
+
fetcher: Optional[HttpFetcher] = None,
|
|
1446
|
+
report: Optional[list[dict[str, Any]]] = None,
|
|
1447
|
+
) -> list[dict[str, str]]:
|
|
1448
|
+
"""Return ``[{title, url, snippet, source}]`` for a keyword query.
|
|
1449
|
+
|
|
1450
|
+
``engine`` is one of ``auto`` (360 + Bing, and 360 + Sogou + the WeChat
|
|
1451
|
+
index for a mostly-CJK query), ``all`` (every engine), ``360``, ``bing``,
|
|
1452
|
+
``sogou``, ``weixin`` or ``duckduckgo``. ``auto`` asks one more engine when
|
|
1453
|
+
the answer holds fewer than three distinct results.
|
|
1454
|
+
|
|
1455
|
+
Pass ``report`` to collect one entry per engine tried — name, whether it
|
|
1456
|
+
answered, how many results and how long it took — so a caller in a hostile
|
|
1457
|
+
network can see which backend is actually reachable.
|
|
1458
|
+
|
|
1459
|
+
Search endpoints are query interfaces, not target pages. robots.txt
|
|
1460
|
+
therefore applies to the *result pages* we later fetch, not to the query
|
|
1461
|
+
request itself. Queries are still rate-limited per host.
|
|
1462
|
+
"""
|
|
1463
|
+
|
|
1464
|
+
query = (query or "").strip()
|
|
1465
|
+
if not query:
|
|
1466
|
+
raise ValueError("query must not be empty")
|
|
1467
|
+
reason = doxxing_reason(query)
|
|
1468
|
+
if reason:
|
|
1469
|
+
raise ProhibitedQuery(
|
|
1470
|
+
f"refused: {reason}. web-forager does not collect or assemble "
|
|
1471
|
+
"profiles of identifiable people; search for a topic, a paper "
|
|
1472
|
+
"title or an organisation instead."
|
|
1473
|
+
)
|
|
1474
|
+
limit = limit or config.num_results or 8
|
|
1475
|
+
engine = (engine or "auto").lower()
|
|
1476
|
+
if engine not in SEARCH_ENGINES:
|
|
1477
|
+
raise ValueError(f"unsupported search engine: {engine}")
|
|
1478
|
+
|
|
1479
|
+
fallback_engines: list[str] = []
|
|
1480
|
+
if engine == "auto":
|
|
1481
|
+
# Bing tokenises Chinese phrases into single characters: a query about
|
|
1482
|
+
# 反逆向工程 comes back as the dictionary page of 反. For a CJK query,
|
|
1483
|
+
# ask 360, Sogou and 搜狗微信 and stop there — the article index is the
|
|
1484
|
+
# one channel that still carries a Chinese person's or institution's
|
|
1485
|
+
# name. Bing stays available through --engine bing.
|
|
1486
|
+
if prefers_chinese_engines(query):
|
|
1487
|
+
engines = ["360", "sogou", "weixin"]
|
|
1488
|
+
fallback_engines = []
|
|
1489
|
+
else:
|
|
1490
|
+
engines = ["360", "bing"]
|
|
1491
|
+
# A Latin query answered with a handful of results (or none) is
|
|
1492
|
+
# often a narrow topic the first two engines do not index; Sogou
|
|
1493
|
+
# widens the net without being asked for every query.
|
|
1494
|
+
fallback_engines = ["sogou"]
|
|
1495
|
+
elif engine == "all":
|
|
1496
|
+
engines = ["360", "bing", "sogou", "weixin"]
|
|
1497
|
+
else:
|
|
1498
|
+
engines = [engine]
|
|
1499
|
+
|
|
1500
|
+
own_fetcher = fetcher is None
|
|
1501
|
+
fetcher = fetcher or HttpFetcher(config)
|
|
1502
|
+
parallel = own_fetcher and config.parallel_engines
|
|
1503
|
+
groups: list[tuple[str, list[dict[str, str]]]] = []
|
|
1504
|
+
errors: list[str] = []
|
|
1505
|
+
|
|
1506
|
+
def run_engine(
|
|
1507
|
+
name: str, fallback: bool, qtext: str = query
|
|
1508
|
+
) -> tuple[dict[str, Any], Optional[list[dict[str, str]]]]:
|
|
1509
|
+
started = time.monotonic()
|
|
1510
|
+
entry: dict[str, Any] = {"engine": name}
|
|
1511
|
+
if fallback:
|
|
1512
|
+
entry["fallback"] = True
|
|
1513
|
+
if qtext != query:
|
|
1514
|
+
entry["query_variant"] = qtext
|
|
1515
|
+
# A requests.Session is not safe to share across engine threads. Each
|
|
1516
|
+
# engine gets its own fetcher; the disk cache and per-host limiters are
|
|
1517
|
+
# still shared through the filesystem and the host split.
|
|
1518
|
+
local = HttpFetcher(config) if parallel else fetcher
|
|
1519
|
+
try:
|
|
1520
|
+
items = _search_one(
|
|
1521
|
+
name, qtext, config, limit, local, pages=config.search_pages
|
|
1522
|
+
)
|
|
1523
|
+
# An engine that split a Chinese phrase into characters returns
|
|
1524
|
+
# the dictionary page of the first one; drop it before it can
|
|
1525
|
+
# pad the list or inflate the per-engine count.
|
|
1526
|
+
items = [
|
|
1527
|
+
item
|
|
1528
|
+
for item in items
|
|
1529
|
+
if not looks_like_character_page(item.get("title", ""), query)
|
|
1530
|
+
and not looks_like_dictionary_page(item, query)
|
|
1531
|
+
]
|
|
1532
|
+
entry.update(
|
|
1533
|
+
ok=True,
|
|
1534
|
+
results=len(items),
|
|
1535
|
+
ms=round((time.monotonic() - started) * 1000),
|
|
1536
|
+
)
|
|
1537
|
+
return entry, items
|
|
1538
|
+
except Exception as exc: # one engine failing must not kill the query
|
|
1539
|
+
errors.append(f"{name}: {exc}")
|
|
1540
|
+
entry.update(
|
|
1541
|
+
ok=False,
|
|
1542
|
+
results=0,
|
|
1543
|
+
error=str(exc),
|
|
1544
|
+
ms=round((time.monotonic() - started) * 1000),
|
|
1545
|
+
)
|
|
1546
|
+
return entry, None
|
|
1547
|
+
finally:
|
|
1548
|
+
if parallel:
|
|
1549
|
+
local.close()
|
|
1550
|
+
|
|
1551
|
+
def collect(names: list[str], fallback: bool = False, qtext: str = query) -> None:
|
|
1552
|
+
if parallel and len(names) > 1:
|
|
1553
|
+
with ThreadPoolExecutor(max_workers=len(names)) as pool:
|
|
1554
|
+
outcomes = list(pool.map(lambda n: run_engine(n, fallback, qtext), names))
|
|
1555
|
+
else:
|
|
1556
|
+
outcomes = [run_engine(name, fallback, qtext) for name in names]
|
|
1557
|
+
for entry, items in outcomes:
|
|
1558
|
+
if items is not None:
|
|
1559
|
+
groups.append((entry["engine"], items))
|
|
1560
|
+
if report is not None:
|
|
1561
|
+
report.append(entry)
|
|
1562
|
+
|
|
1563
|
+
try:
|
|
1564
|
+
collect(engines)
|
|
1565
|
+
if fallback_engines and (not groups or _thin_results(groups, limit)):
|
|
1566
|
+
collect(fallback_engines, fallback=True)
|
|
1567
|
+
# A thin first pass is often a formatting mismatch rather than a
|
|
1568
|
+
# missing topic. One simplified variant costs two or three more
|
|
1569
|
+
# requests and is skipped entirely when the first pass was healthy, or
|
|
1570
|
+
# when the caller pinned a single engine on purpose.
|
|
1571
|
+
if groups and engine in ("auto", "all") and _thin_results(groups, limit):
|
|
1572
|
+
for variant in query_variants(query)[1:]:
|
|
1573
|
+
collect(engines, fallback=True, qtext=variant)
|
|
1574
|
+
finally:
|
|
1575
|
+
if own_fetcher:
|
|
1576
|
+
fetcher.close()
|
|
1577
|
+
|
|
1578
|
+
if not groups:
|
|
1579
|
+
raise FetchError("all search engines failed: " + "; ".join(errors) or "no engine available")
|
|
1580
|
+
|
|
1581
|
+
if len(groups) == 1:
|
|
1582
|
+
merged = groups[0][1]
|
|
1583
|
+
else:
|
|
1584
|
+
merged = merge_search_results(groups, query, limit=limit)
|
|
1585
|
+
return diversify_results(
|
|
1586
|
+
dedupe_results(merged, limit=max(limit * 3, limit)), limit=limit
|
|
1587
|
+
)
|
|
1588
|
+
|
|
1589
|
+
|
|
1590
|
+
def _thin_results(groups: list[tuple[str, list[dict[str, str]]]], limit: int) -> bool:
|
|
1591
|
+
"""True when the engines answered with fewer than a useful number of hits.
|
|
1592
|
+
|
|
1593
|
+
Three is the floor: below it a caller almost always has to run a second
|
|
1594
|
+
query by hand. The check counts distinct URLs, not raw rows, because one
|
|
1595
|
+
engine repeats the same result across pages.
|
|
1596
|
+
"""
|
|
1597
|
+
|
|
1598
|
+
items = dedupe_results(
|
|
1599
|
+
[item for _, group in groups for item in group], limit=max(limit, 3)
|
|
1600
|
+
)
|
|
1601
|
+
return len(items) < min(limit, 3)
|
|
1602
|
+
|
|
1603
|
+
|
|
1604
|
+
def is_360_blocked(html: str) -> bool:
|
|
1605
|
+
"""True when so.com answered with a verification page instead of results.
|
|
1606
|
+
|
|
1607
|
+
The page is small and has no result list; a real result page always carries
|
|
1608
|
+
``res-list``. Only used to report the engine honestly, never to retry past
|
|
1609
|
+
the check.
|
|
1610
|
+
"""
|
|
1611
|
+
|
|
1612
|
+
lowered = (html or "").lower()
|
|
1613
|
+
if not lowered or "res-list" in lowered or len(html) > 60000:
|
|
1614
|
+
return False
|
|
1615
|
+
if any(marker in html for marker in ("安全验证", "人机验证", "请输入验证码", "访问异常", "滑动验证")):
|
|
1616
|
+
return True
|
|
1617
|
+
return "验证" in html and len(html) < 12000
|
|
1618
|
+
|
|
1619
|
+
|
|
1620
|
+
def _search_one(
|
|
1621
|
+
name: str,
|
|
1622
|
+
query: str,
|
|
1623
|
+
config: FetchConfig,
|
|
1624
|
+
limit: int,
|
|
1625
|
+
fetcher: HttpFetcher,
|
|
1626
|
+
pages: int = 1,
|
|
1627
|
+
) -> list[dict[str, str]]:
|
|
1628
|
+
quoted = urllib.parse.quote_plus(query)
|
|
1629
|
+
pages = max(1, pages)
|
|
1630
|
+
collected: list[dict[str, str]] = []
|
|
1631
|
+
|
|
1632
|
+
for page in range(1, pages + 1):
|
|
1633
|
+
if name == "duckduckgo" and page > 1:
|
|
1634
|
+
break
|
|
1635
|
+
if name == "360":
|
|
1636
|
+
response = fetcher.fetch(
|
|
1637
|
+
_SO_360.format(query=quoted, page=page), check_robots=False
|
|
1638
|
+
)
|
|
1639
|
+
if is_360_blocked(response.text):
|
|
1640
|
+
raise FetchError("360 returned a verification page; wait and retry later")
|
|
1641
|
+
found = parse_360_html(response.text, limit=limit)
|
|
1642
|
+
elif name == "sogou":
|
|
1643
|
+
response = fetcher.fetch(
|
|
1644
|
+
_SOGOU.format(query=quoted, page=page), check_robots=False
|
|
1645
|
+
)
|
|
1646
|
+
if is_sogou_blocked(response.text):
|
|
1647
|
+
raise FetchError("Sogou returned a verification page (antispider)")
|
|
1648
|
+
parsed = parse_sogou_html(
|
|
1649
|
+
response.text, base="https://www.sogou.com/", limit=limit
|
|
1650
|
+
)
|
|
1651
|
+
found = resolve_sogou_results(parsed, fetcher, limit=limit)
|
|
1652
|
+
elif name == "weixin":
|
|
1653
|
+
response = fetcher.fetch(
|
|
1654
|
+
_WEIXIN.format(query=quoted, page=page), check_robots=False
|
|
1655
|
+
)
|
|
1656
|
+
if is_sogou_blocked(response.text):
|
|
1657
|
+
raise FetchError("weixin.sogou.com returned a verification page (antispider)")
|
|
1658
|
+
parsed = parse_weixin_html(
|
|
1659
|
+
response.text, base="https://weixin.sogou.com/", limit=limit
|
|
1660
|
+
)
|
|
1661
|
+
found = resolve_sogou_results(parsed, fetcher, limit=limit)
|
|
1662
|
+
elif name == "duckduckgo":
|
|
1663
|
+
response = fetcher.fetch(_DDG_HTML.format(query=quoted), check_robots=False)
|
|
1664
|
+
found = parse_ddg_html(response.text, limit=limit)
|
|
1665
|
+
else:
|
|
1666
|
+
first = 1 + (page - 1) * 10
|
|
1667
|
+
# Bing: RSS is cheaper and gives direct URLs; HTML is the fallback.
|
|
1668
|
+
try:
|
|
1669
|
+
response = fetcher.fetch(
|
|
1670
|
+
_BING_RSS.format(query=quoted, count=limit, first=first),
|
|
1671
|
+
check_robots=False,
|
|
1672
|
+
)
|
|
1673
|
+
found = parse_bing_rss(response.text, limit=limit)
|
|
1674
|
+
except FetchError:
|
|
1675
|
+
found = []
|
|
1676
|
+
if not found:
|
|
1677
|
+
response = fetcher.fetch(
|
|
1678
|
+
_BING_HTML.format(query=quoted, count=limit, first=first),
|
|
1679
|
+
check_robots=False,
|
|
1680
|
+
)
|
|
1681
|
+
found = parse_bing_html(response.text, limit=limit)
|
|
1682
|
+
|
|
1683
|
+
if not found:
|
|
1684
|
+
break
|
|
1685
|
+
collected.extend(found)
|
|
1686
|
+
collected = dedupe_results(collected, limit=limit * pages)
|
|
1687
|
+
if len(collected) >= limit * pages:
|
|
1688
|
+
break
|
|
1689
|
+
|
|
1690
|
+
return collected[: limit * pages]
|
|
1691
|
+
|
|
1692
|
+
|
|
1693
|
+
def _query_tokens(query: str) -> list[str]:
|
|
1694
|
+
return [token.lower() for token in re.split(r"[\s,,;;|]+", query.strip()) if token]
|
|
1695
|
+
|
|
1696
|
+
|
|
1697
|
+
def result_coverage(result: dict[str, str], tokens: list[str]) -> float:
|
|
1698
|
+
"""Fraction of query tokens present in title/snippet/url (0..1)."""
|
|
1699
|
+
|
|
1700
|
+
if not tokens:
|
|
1701
|
+
return 0.0
|
|
1702
|
+
haystack = " ".join(
|
|
1703
|
+
(result.get("title", ""), result.get("snippet", ""), result.get("url", ""))
|
|
1704
|
+
).lower()
|
|
1705
|
+
return sum(1 for token in tokens if token in haystack) / len(tokens)
|
|
1706
|
+
|
|
1707
|
+
|
|
1708
|
+
_HIGH_VALUE_SUFFIXES = (
|
|
1709
|
+
"docs.python.org",
|
|
1710
|
+
"developer.mozilla.org",
|
|
1711
|
+
"wikipedia.org",
|
|
1712
|
+
"arxiv.org",
|
|
1713
|
+
"github.com",
|
|
1714
|
+
"stackoverflow.com",
|
|
1715
|
+
"nature.com",
|
|
1716
|
+
"science.org",
|
|
1717
|
+
"who.int",
|
|
1718
|
+
"worldbank.org",
|
|
1719
|
+
"gov.cn",
|
|
1720
|
+
"edu.cn",
|
|
1721
|
+
"ac.uk",
|
|
1722
|
+
"edu",
|
|
1723
|
+
)
|
|
1724
|
+
_LOW_VALUE_SUFFIXES = (
|
|
1725
|
+
"doc88.com",
|
|
1726
|
+
"docin.com",
|
|
1727
|
+
"wenku.baidu.com",
|
|
1728
|
+
"360doc.com",
|
|
1729
|
+
"book118.com",
|
|
1730
|
+
)
|
|
1731
|
+
|
|
1732
|
+
|
|
1733
|
+
def _host_prior(host: str) -> float:
|
|
1734
|
+
"""A small prior for domains that usually answer a query well.
|
|
1735
|
+
|
|
1736
|
+
It only breaks ties and nudges; query coverage carries most of the score,
|
|
1737
|
+
so a content farm with the exact answer still outranks an off-topic paper.
|
|
1738
|
+
"""
|
|
1739
|
+
|
|
1740
|
+
host = (host or "").lower()
|
|
1741
|
+
for suffix in _HIGH_VALUE_SUFFIXES:
|
|
1742
|
+
if host == suffix or host.endswith("." + suffix):
|
|
1743
|
+
return 0.08
|
|
1744
|
+
for suffix in _LOW_VALUE_SUFFIXES:
|
|
1745
|
+
if host == suffix or host.endswith("." + suffix):
|
|
1746
|
+
return -0.08
|
|
1747
|
+
return 0.0
|
|
1748
|
+
|
|
1749
|
+
|
|
1750
|
+
def _freshness_bonus(text: str) -> float:
|
|
1751
|
+
"""Prefer results whose snippet carries a recent year."""
|
|
1752
|
+
|
|
1753
|
+
years = [int(match) for match in re.findall(r"\b(20\d{2})\b", text or "")]
|
|
1754
|
+
if not years:
|
|
1755
|
+
return 0.0
|
|
1756
|
+
newest = max(years)
|
|
1757
|
+
current = datetime.now(timezone.utc).year
|
|
1758
|
+
if newest >= current:
|
|
1759
|
+
return 0.06
|
|
1760
|
+
if newest == current - 1:
|
|
1761
|
+
return 0.03
|
|
1762
|
+
return 0.0
|
|
1763
|
+
|
|
1764
|
+
|
|
1765
|
+
def query_variants(query: str) -> list[str]:
|
|
1766
|
+
"""Return the original query and at most one simplified variant.
|
|
1767
|
+
|
|
1768
|
+
Only used when the first pass came back thin. It strips quotes and common
|
|
1769
|
+
English stopwords, and joins a spaced CJK query, because result pages treat
|
|
1770
|
+
those two forms differently.
|
|
1771
|
+
"""
|
|
1772
|
+
|
|
1773
|
+
base = (query or "").strip()
|
|
1774
|
+
if not base:
|
|
1775
|
+
return []
|
|
1776
|
+
variants = [base]
|
|
1777
|
+
simplified = re.sub(r"[\"'“”‘’]", " ", base)
|
|
1778
|
+
simplified = re.sub(r"[\s,,;;|、]+", " ", simplified).strip()
|
|
1779
|
+
tokens = [
|
|
1780
|
+
token
|
|
1781
|
+
for token in simplified.split()
|
|
1782
|
+
if token.lower() not in _QUERY_STOPWORDS
|
|
1783
|
+
]
|
|
1784
|
+
candidate = " ".join(tokens).strip()
|
|
1785
|
+
if candidate and candidate.lower() != base.lower():
|
|
1786
|
+
variants.append(candidate)
|
|
1787
|
+
if re.search(r"[㐀-鿿]", base) and " " in base:
|
|
1788
|
+
joined = re.sub(r"\s+", "", base)
|
|
1789
|
+
if joined and joined not in variants:
|
|
1790
|
+
variants.append(joined)
|
|
1791
|
+
return variants[:2]
|
|
1792
|
+
|
|
1793
|
+
|
|
1794
|
+
_QUERY_STOPWORDS = {
|
|
1795
|
+
"a",
|
|
1796
|
+
"an",
|
|
1797
|
+
"the",
|
|
1798
|
+
"of",
|
|
1799
|
+
"for",
|
|
1800
|
+
"to",
|
|
1801
|
+
"in",
|
|
1802
|
+
"on",
|
|
1803
|
+
"and",
|
|
1804
|
+
"or",
|
|
1805
|
+
"with",
|
|
1806
|
+
"about",
|
|
1807
|
+
"please",
|
|
1808
|
+
"如何",
|
|
1809
|
+
"怎么",
|
|
1810
|
+
"什么",
|
|
1811
|
+
"怎样",
|
|
1812
|
+
}
|
|
1813
|
+
|
|
1814
|
+
|
|
1815
|
+
def diversify_results(
|
|
1816
|
+
results: list[dict[str, str]], limit: int, per_host: int = 2
|
|
1817
|
+
) -> list[dict[str, str]]:
|
|
1818
|
+
"""Keep at most ``per_host`` results per host, then fill from the overflow.
|
|
1819
|
+
|
|
1820
|
+
A single host that dominates the list (WeChat, one content farm) no longer
|
|
1821
|
+
crowds out every other source; if too few hosts answered, the overflow is
|
|
1822
|
+
appended so the caller still receives ``limit`` results.
|
|
1823
|
+
"""
|
|
1824
|
+
|
|
1825
|
+
chosen: list[dict[str, str]] = []
|
|
1826
|
+
overflow: list[dict[str, str]] = []
|
|
1827
|
+
counts: dict[str, int] = {}
|
|
1828
|
+
for item in results:
|
|
1829
|
+
host = (urllib.parse.urlsplit(item.get("url", "")).netloc or "").lower()
|
|
1830
|
+
if counts.get(host, 0) < per_host:
|
|
1831
|
+
counts[host] = counts.get(host, 0) + 1
|
|
1832
|
+
chosen.append(item)
|
|
1833
|
+
else:
|
|
1834
|
+
overflow.append(item)
|
|
1835
|
+
return (chosen + overflow)[:limit]
|
|
1836
|
+
|
|
1837
|
+
|
|
1838
|
+
def merge_search_results(
|
|
1839
|
+
groups: list[tuple[str, list[dict[str, str]]]],
|
|
1840
|
+
query: str,
|
|
1841
|
+
limit: int = 8,
|
|
1842
|
+
) -> list[dict[str, str]]:
|
|
1843
|
+
"""Merge several engines by coverage, title match, domain prior and freshness.
|
|
1844
|
+
|
|
1845
|
+
Engine order only breaks exact ties, so two engines that return the same
|
|
1846
|
+
page still produce one stable ordering.
|
|
1847
|
+
"""
|
|
1848
|
+
|
|
1849
|
+
tokens = _query_tokens(query)
|
|
1850
|
+
phrase = " ".join((query or "").lower().split())
|
|
1851
|
+
scored: list[tuple[float, int, int, dict[str, str]]] = []
|
|
1852
|
+
for group_index, (name, items) in enumerate(groups):
|
|
1853
|
+
for rank, item in enumerate(items):
|
|
1854
|
+
candidate = dict(item)
|
|
1855
|
+
candidate.setdefault("source", name)
|
|
1856
|
+
coverage = result_coverage(candidate, tokens)
|
|
1857
|
+
title = (candidate.get("title") or "").lower()
|
|
1858
|
+
title_hits = (
|
|
1859
|
+
sum(1 for token in tokens if token in title) / len(tokens)
|
|
1860
|
+
if tokens
|
|
1861
|
+
else 0.0
|
|
1862
|
+
)
|
|
1863
|
+
exact = 0.1 if phrase and phrase in title else 0.0
|
|
1864
|
+
host = urllib.parse.urlsplit(candidate.get("url", "")).netloc
|
|
1865
|
+
score = (
|
|
1866
|
+
coverage * 0.55
|
|
1867
|
+
+ title_hits * 0.2
|
|
1868
|
+
+ exact
|
|
1869
|
+
+ _host_prior(host)
|
|
1870
|
+
+ _freshness_bonus(
|
|
1871
|
+
f"{candidate.get('title', '')} {candidate.get('snippet', '')}"
|
|
1872
|
+
)
|
|
1873
|
+
)
|
|
1874
|
+
scored.append((score, group_index, rank, candidate))
|
|
1875
|
+
scored.sort(key=lambda entry: (-entry[0], entry[1], entry[2]))
|
|
1876
|
+
return [entry[3] for entry in scored[: max(limit * 3, limit)]]
|
|
1877
|
+
|
|
1878
|
+
|
|
1879
|
+
|
|
1880
|
+
def parse_360_html(html: str, limit: int = 8) -> list[dict[str, str]]:
|
|
1881
|
+
"""Parse 360 Search organic results (``li.res-list``)."""
|
|
1882
|
+
|
|
1883
|
+
soup = make_soup(html)
|
|
1884
|
+
results: list[dict[str, str]] = []
|
|
1885
|
+
seen: set[str] = set()
|
|
1886
|
+
for item in soup.select("li.res-list"):
|
|
1887
|
+
anchor = item.select_one("h3 a") or item.select_one("a[data-mdurl]")
|
|
1888
|
+
if anchor is None:
|
|
1889
|
+
continue
|
|
1890
|
+
href = _attr_text(anchor, "data-mdurl") or _attr_text(anchor, "href")
|
|
1891
|
+
url = absolute_url(href, base="https://www.so.com/")
|
|
1892
|
+
if not url:
|
|
1893
|
+
continue
|
|
1894
|
+
host = urllib.parse.urlsplit(url).netloc.lower()
|
|
1895
|
+
if any(host == internal or host.endswith("." + internal) for internal in _360_INTERNAL_HOSTS):
|
|
1896
|
+
continue
|
|
1897
|
+
heading = item.select_one("h3")
|
|
1898
|
+
title = _clean_result_text(heading.get_text(" ", strip=True) if heading else "")
|
|
1899
|
+
if not title:
|
|
1900
|
+
continue
|
|
1901
|
+
snippet = ""
|
|
1902
|
+
for selector in (".res-desc", ".res-rich", ".res-comm-con", ".g-ellipsis2", ".mh-title"):
|
|
1903
|
+
node = item.select_one(selector)
|
|
1904
|
+
if node is not None:
|
|
1905
|
+
snippet = _clean_result_text(node.get_text(" ", strip=True))
|
|
1906
|
+
if snippet:
|
|
1907
|
+
break
|
|
1908
|
+
if url in seen:
|
|
1909
|
+
continue
|
|
1910
|
+
seen.add(url)
|
|
1911
|
+
results.append({"title": title, "url": url, "snippet": snippet, "source": "360"})
|
|
1912
|
+
if len(results) >= limit:
|
|
1913
|
+
break
|
|
1914
|
+
return results
|
|
1915
|
+
|
|
1916
|
+
|
|
1917
|
+
def is_sogou_blocked(html: str) -> bool:
|
|
1918
|
+
lowered = (html or "").lower()
|
|
1919
|
+
return "antispider" in lowered or (
|
|
1920
|
+
any(marker in html for marker in _SOGOU_BLOCK_MARKERS) and "vrwrap" not in lowered
|
|
1921
|
+
)
|
|
1922
|
+
|
|
1923
|
+
|
|
1924
|
+
def parse_sogou_html(
|
|
1925
|
+
html: str, base: str = "https://www.sogou.com/", limit: int = 8
|
|
1926
|
+
) -> list[dict[str, str]]:
|
|
1927
|
+
"""Parse Sogou organic results (``div.vrwrap``)."""
|
|
1928
|
+
|
|
1929
|
+
soup = make_soup(html)
|
|
1930
|
+
results: list[dict[str, str]] = []
|
|
1931
|
+
seen: set[str] = set()
|
|
1932
|
+
for item in soup.select("div.vrwrap"):
|
|
1933
|
+
classes = " ".join(str(value) for value in (item.get("class") or []))
|
|
1934
|
+
if "hintBox" in classes:
|
|
1935
|
+
continue
|
|
1936
|
+
anchor = item.select_one("h3 a")
|
|
1937
|
+
if anchor is None:
|
|
1938
|
+
continue
|
|
1939
|
+
href = _attr_text(anchor, "href")
|
|
1940
|
+
url = absolute_url(href, base=base)
|
|
1941
|
+
if not url:
|
|
1942
|
+
continue
|
|
1943
|
+
heading = item.select_one("h3")
|
|
1944
|
+
title = _clean_result_text(heading.get_text(" ", strip=True) if heading else "")
|
|
1945
|
+
if not title:
|
|
1946
|
+
continue
|
|
1947
|
+
snippet = ""
|
|
1948
|
+
for selector in (".str_info", ".space-txt", ".fz-mid", ".text-layout", ".str-text"):
|
|
1949
|
+
node = item.select_one(selector)
|
|
1950
|
+
if node is not None:
|
|
1951
|
+
snippet = _clean_result_text(node.get_text(" ", strip=True))
|
|
1952
|
+
if snippet:
|
|
1953
|
+
break
|
|
1954
|
+
if url in seen:
|
|
1955
|
+
continue
|
|
1956
|
+
seen.add(url)
|
|
1957
|
+
results.append({"title": title, "url": url, "snippet": snippet, "source": "sogou"})
|
|
1958
|
+
if len(results) >= limit:
|
|
1959
|
+
break
|
|
1960
|
+
return results
|
|
1961
|
+
|
|
1962
|
+
|
|
1963
|
+
def _join_cjk(text: str) -> str:
|
|
1964
|
+
"""Remove the spaces a highlight tag leaves inside a Chinese sentence.
|
|
1965
|
+
|
|
1966
|
+
``示例<em>名字</em>解析`` reads as "示例 名字 解析" once tags are replaced with
|
|
1967
|
+
spaces; Chinese text has no word spaces, so joining across CJK characters
|
|
1968
|
+
restores the original phrase without touching spaces between Latin words.
|
|
1969
|
+
"""
|
|
1970
|
+
|
|
1971
|
+
return re.sub(r"(?<=[㐀-䶿一-鿿])\s+(?=[㐀-䶿一-鿿])", "", text)
|
|
1972
|
+
|
|
1973
|
+
|
|
1974
|
+
def parse_weixin_html(
|
|
1975
|
+
html: str, base: str = "https://weixin.sogou.com/", limit: int = 8
|
|
1976
|
+
) -> list[dict[str, str]]:
|
|
1977
|
+
"""Parse 搜狗微信 article results (``ul.news-list > li``).
|
|
1978
|
+
|
|
1979
|
+
Each entry is one WeChat official-account article. The link is a Sogou
|
|
1980
|
+
``/link?url=`` wrapper that resolves to ``mp.weixin.qq.com`` (handled by
|
|
1981
|
+
:func:`resolve_sogou_results`); the account name and publish date travel
|
|
1982
|
+
with the result so a caller can tell a lab's own post from a re-post.
|
|
1983
|
+
"""
|
|
1984
|
+
|
|
1985
|
+
soup = make_soup(html)
|
|
1986
|
+
results: list[dict[str, str]] = []
|
|
1987
|
+
seen: set[str] = set()
|
|
1988
|
+
for item in soup.select("ul.news-list li"):
|
|
1989
|
+
anchor = item.select_one("h3 a")
|
|
1990
|
+
if anchor is None:
|
|
1991
|
+
continue
|
|
1992
|
+
href = _attr_text(anchor, "href")
|
|
1993
|
+
url = absolute_url(href, base=base)
|
|
1994
|
+
if not url:
|
|
1995
|
+
continue
|
|
1996
|
+
heading = item.select_one("h3")
|
|
1997
|
+
title = _join_cjk(_clean_result_text(heading.get_text(" ", strip=True) if heading else ""))
|
|
1998
|
+
if not title:
|
|
1999
|
+
continue
|
|
2000
|
+
snippet = ""
|
|
2001
|
+
node = item.select_one("p.txt-info")
|
|
2002
|
+
if node is not None:
|
|
2003
|
+
snippet = _join_cjk(_clean_result_text(node.get_text(" ", strip=True)))
|
|
2004
|
+
entry = {
|
|
2005
|
+
"title": title,
|
|
2006
|
+
"url": url,
|
|
2007
|
+
"snippet": snippet,
|
|
2008
|
+
"source": "weixin",
|
|
2009
|
+
}
|
|
2010
|
+
account = item.select_one("div.s-p span.all-time-y2") or item.select_one(
|
|
2011
|
+
"div.s-p a.account"
|
|
2012
|
+
)
|
|
2013
|
+
if account is not None:
|
|
2014
|
+
name = _clean_result_text(account.get_text(" ", strip=True))
|
|
2015
|
+
if name:
|
|
2016
|
+
entry["account"] = name
|
|
2017
|
+
stamp = re.search(r"timeConvert\('(\d+)'\)", str(item))
|
|
2018
|
+
if stamp:
|
|
2019
|
+
try:
|
|
2020
|
+
entry["date"] = time.strftime(
|
|
2021
|
+
"%Y-%m-%d", time.localtime(int(stamp.group(1)))
|
|
2022
|
+
)
|
|
2023
|
+
except (OverflowError, OSError, ValueError):
|
|
2024
|
+
pass
|
|
2025
|
+
if url in seen:
|
|
2026
|
+
continue
|
|
2027
|
+
seen.add(url)
|
|
2028
|
+
results.append(entry)
|
|
2029
|
+
if len(results) >= limit:
|
|
2030
|
+
break
|
|
2031
|
+
return results
|
|
2032
|
+
|
|
2033
|
+
|
|
2034
|
+
def resolve_sogou_results(
|
|
2035
|
+
results: list[dict[str, str]],
|
|
2036
|
+
fetcher: HttpFetcher,
|
|
2037
|
+
limit: int = 8,
|
|
2038
|
+
) -> list[dict[str, str]]:
|
|
2039
|
+
"""Resolve Sogou ``/link?url=`` wrappers to the real target URL."""
|
|
2040
|
+
|
|
2041
|
+
resolved: list[dict[str, str]] = []
|
|
2042
|
+
for item in results[:limit]:
|
|
2043
|
+
url = item.get("url", "")
|
|
2044
|
+
if "sogou.com/link" in url or "/link?url=" in url:
|
|
2045
|
+
real = fetcher.resolve(url)
|
|
2046
|
+
if real and "sogou.com/link" not in real:
|
|
2047
|
+
item = dict(item)
|
|
2048
|
+
item["url"] = real
|
|
2049
|
+
resolved.append(item)
|
|
2050
|
+
return resolved
|
|
2051
|
+
|
|
2052
|
+
|
|
2053
|
+
def dedupe_results(results: Iterable[dict[str, str]], limit: int = 8) -> list[dict[str, str]]:
|
|
2054
|
+
"""Remove duplicate URLs and unusable titles while preserving rank order."""
|
|
2055
|
+
|
|
2056
|
+
seen: set[str] = set()
|
|
2057
|
+
unique: list[dict[str, str]] = []
|
|
2058
|
+
for item in results:
|
|
2059
|
+
url = item.get("url", "")
|
|
2060
|
+
if not url or url in seen:
|
|
2061
|
+
continue
|
|
2062
|
+
title = str(item.get("title") or "")
|
|
2063
|
+
# Engines sometimes answer a query with an anti-bot interstitial and
|
|
2064
|
+
# the "title" comes back as replacement characters or bare punctuation.
|
|
2065
|
+
# An entry with no title at all is left alone: other callers feed
|
|
2066
|
+
# dedupe_results bare URLs.
|
|
2067
|
+
if title and (looks_garbled(title) or not _USEFUL_TITLE_RE.search(title)):
|
|
2068
|
+
continue
|
|
2069
|
+
seen.add(url)
|
|
2070
|
+
unique.append(item)
|
|
2071
|
+
if len(unique) >= limit:
|
|
2072
|
+
break
|
|
2073
|
+
return unique
|
|
2074
|
+
|
|
2075
|
+
|
|
2076
|
+
def _clean_result_text(value: Optional[str]) -> str:
|
|
2077
|
+
if not value:
|
|
2078
|
+
return ""
|
|
2079
|
+
text = html_lib.unescape(str(value))
|
|
2080
|
+
if "<" in text:
|
|
2081
|
+
text = BeautifulSoup(text, "html.parser").get_text(" ", strip=True)
|
|
2082
|
+
return re.sub(r"\s+", " ", text).strip()
|
|
2083
|
+
|
|
2084
|
+
|
|
2085
|
+
def _attr_text(tag: Tag, name: str, default: str = "") -> str:
|
|
2086
|
+
"""Read an HTML attribute as a plain string, tolerating list values."""
|
|
2087
|
+
|
|
2088
|
+
value: Any = tag.attrs.get(name, default)
|
|
2089
|
+
if isinstance(value, list):
|
|
2090
|
+
return " ".join(str(item) for item in value).strip()
|
|
2091
|
+
if value is None:
|
|
2092
|
+
return default
|
|
2093
|
+
return str(value).strip()
|
|
2094
|
+
|
|
2095
|
+
|
|
2096
|
+
def _int_attr(tag: Tag, name: str, default: int) -> int:
|
|
2097
|
+
"""Read an integer HTML attribute with a safe fallback."""
|
|
2098
|
+
|
|
2099
|
+
raw = _attr_text(tag, name)
|
|
2100
|
+
try:
|
|
2101
|
+
return int(raw) if raw else default
|
|
2102
|
+
except ValueError:
|
|
2103
|
+
return default
|
|
2104
|
+
|
|
2105
|
+
|
|
2106
|
+
def parse_bing_rss(xml_text: str, limit: int = 8) -> list[dict[str, str]]:
|
|
2107
|
+
"""Parse a Bing ``format=rss`` response.
|
|
2108
|
+
|
|
2109
|
+
The parser is only ever handed a metered search-engine response, but we
|
|
2110
|
+
still refuse documents that declare a DTD or custom entities. This blocks
|
|
2111
|
+
XXE / entity-expansion payloads without pulling in a non-permitted XML
|
|
2112
|
+
library.
|
|
2113
|
+
"""
|
|
2114
|
+
|
|
2115
|
+
results: list[dict[str, str]] = []
|
|
2116
|
+
lowered = xml_text[:20_000].lower()
|
|
2117
|
+
if "<!doctype" in lowered or "<!entity" in lowered:
|
|
2118
|
+
return results
|
|
2119
|
+
try:
|
|
2120
|
+
root = ET.fromstring(xml_text.encode("utf-8") if isinstance(xml_text, str) else xml_text)
|
|
2121
|
+
except ET.ParseError:
|
|
2122
|
+
return results
|
|
2123
|
+
|
|
2124
|
+
for item in root.iter():
|
|
2125
|
+
if not item.tag.lower().endswith("item"):
|
|
2126
|
+
continue
|
|
2127
|
+
title = _clean_result_text(_xml_text(item, "title"))
|
|
2128
|
+
link = (_xml_text(item, "link") or "").strip()
|
|
2129
|
+
description = _clean_result_text(_xml_text(item, "description"))
|
|
2130
|
+
if not link:
|
|
2131
|
+
continue
|
|
2132
|
+
url = absolute_url(_decode_bing_redirect(link), base=None)
|
|
2133
|
+
if not url:
|
|
2134
|
+
continue
|
|
2135
|
+
results.append({"title": title, "url": url, "snippet": description, "source": "bing"})
|
|
2136
|
+
if len(results) >= limit:
|
|
2137
|
+
break
|
|
2138
|
+
return results
|
|
2139
|
+
|
|
2140
|
+
|
|
2141
|
+
def _xml_text(node: ET.Element, name: str) -> str:
|
|
2142
|
+
for child in node:
|
|
2143
|
+
if child.tag.lower().split("}")[-1] == name.lower():
|
|
2144
|
+
return child.text or ""
|
|
2145
|
+
return ""
|
|
2146
|
+
|
|
2147
|
+
|
|
2148
|
+
def parse_bing_html(html: str, limit: int = 8) -> list[dict[str, str]]:
|
|
2149
|
+
"""Parse the HTML Bing result list."""
|
|
2150
|
+
|
|
2151
|
+
soup = make_soup(html)
|
|
2152
|
+
results: list[dict[str, str]] = []
|
|
2153
|
+
for block in soup.select("li.b_algo"):
|
|
2154
|
+
anchor = block.select_one("h2 a") or block.select_one("a")
|
|
2155
|
+
if not anchor:
|
|
2156
|
+
continue
|
|
2157
|
+
href = _attr_text(anchor, "href")
|
|
2158
|
+
url = absolute_url(_decode_bing_redirect(href), base="https://www.bing.com/")
|
|
2159
|
+
if not url:
|
|
2160
|
+
continue
|
|
2161
|
+
title = _clean_result_text(anchor.get_text(" ", strip=True))
|
|
2162
|
+
snippet_tag = block.select_one(".b_caption p") or block.select_one(".b_snippet")
|
|
2163
|
+
snippet = _clean_result_text(snippet_tag.get_text(" ", strip=True)) if snippet_tag else ""
|
|
2164
|
+
results.append({"title": title, "url": url, "snippet": snippet, "source": "bing"})
|
|
2165
|
+
if len(results) >= limit:
|
|
2166
|
+
break
|
|
2167
|
+
return results
|
|
2168
|
+
|
|
2169
|
+
|
|
2170
|
+
def parse_ddg_html(html: str, limit: int = 8) -> list[dict[str, str]]:
|
|
2171
|
+
"""Parse the DuckDuckGo HTML result page."""
|
|
2172
|
+
|
|
2173
|
+
soup = make_soup(html)
|
|
2174
|
+
results: list[dict[str, str]] = []
|
|
2175
|
+
for block in soup.select("div.result"):
|
|
2176
|
+
anchor = block.select_one("a.result__a")
|
|
2177
|
+
if not anchor:
|
|
2178
|
+
continue
|
|
2179
|
+
href = _attr_text(anchor, "href")
|
|
2180
|
+
url = absolute_url(_decode_ddg_redirect(href), base="https://duckduckgo.com/")
|
|
2181
|
+
if not url:
|
|
2182
|
+
continue
|
|
2183
|
+
title = _clean_result_text(anchor.get_text(" ", strip=True))
|
|
2184
|
+
snippet_tag = block.select_one(".result__snippet")
|
|
2185
|
+
snippet = _clean_result_text(snippet_tag.get_text(" ", strip=True)) if snippet_tag else ""
|
|
2186
|
+
results.append({"title": title, "url": url, "snippet": snippet, "source": "duckduckgo"})
|
|
2187
|
+
if len(results) >= limit:
|
|
2188
|
+
break
|
|
2189
|
+
return results
|
|
2190
|
+
|
|
2191
|
+
|
|
2192
|
+
def _decode_bing_redirect(href: str) -> str:
|
|
2193
|
+
"""Unwrap Bing's ``/ck/a?...&u=a1<base64url>`` redirect wrapper."""
|
|
2194
|
+
|
|
2195
|
+
if not href:
|
|
2196
|
+
return href
|
|
2197
|
+
parts = urllib.parse.urlsplit(href)
|
|
2198
|
+
if "bing.com" not in parts.netloc.lower() or not parts.path.startswith("/ck/"):
|
|
2199
|
+
return href
|
|
2200
|
+
query = urllib.parse.parse_qs(parts.query)
|
|
2201
|
+
value = (query.get("u") or [""])[0]
|
|
2202
|
+
if not value.startswith("a1"):
|
|
2203
|
+
return href
|
|
2204
|
+
payload = value[2:]
|
|
2205
|
+
payload += "=" * (-len(payload) % 4)
|
|
2206
|
+
try:
|
|
2207
|
+
decoded = base64.urlsafe_b64decode(payload.encode("ascii")).decode("utf-8", "ignore")
|
|
2208
|
+
except Exception:
|
|
2209
|
+
return href
|
|
2210
|
+
return decoded if decoded.startswith(("http://", "https://")) else href
|
|
2211
|
+
|
|
2212
|
+
|
|
2213
|
+
def _decode_ddg_redirect(href: str) -> str:
|
|
2214
|
+
if not href:
|
|
2215
|
+
return href
|
|
2216
|
+
parts = urllib.parse.urlsplit(href)
|
|
2217
|
+
if "duckduckgo.com" not in parts.netloc.lower():
|
|
2218
|
+
return href
|
|
2219
|
+
query = urllib.parse.parse_qs(parts.query)
|
|
2220
|
+
value = (query.get("uddg") or [""])[0]
|
|
2221
|
+
return urllib.parse.unquote(value) if value else href
|
|
2222
|
+
|
|
2223
|
+
|
|
2224
|
+
# ---------------------------------------------------------------------------
|
|
2225
|
+
# HTML inspection
|
|
2226
|
+
# ---------------------------------------------------------------------------
|
|
2227
|
+
|
|
2228
|
+
_SKIP_TAGS = {
|
|
2229
|
+
"script",
|
|
2230
|
+
"style",
|
|
2231
|
+
"noscript",
|
|
2232
|
+
"template",
|
|
2233
|
+
"svg",
|
|
2234
|
+
"canvas",
|
|
2235
|
+
"iframe",
|
|
2236
|
+
"form",
|
|
2237
|
+
"button",
|
|
2238
|
+
"input",
|
|
2239
|
+
"select",
|
|
2240
|
+
"textarea",
|
|
2241
|
+
"nav",
|
|
2242
|
+
"footer",
|
|
2243
|
+
"aside",
|
|
2244
|
+
"header",
|
|
2245
|
+
}
|
|
2246
|
+
|
|
2247
|
+
_DYNAMIC_MARKERS = (
|
|
2248
|
+
"__NEXT_DATA__",
|
|
2249
|
+
"__NUXT__",
|
|
2250
|
+
'id="root"',
|
|
2251
|
+
"id='root'",
|
|
2252
|
+
'id="app"',
|
|
2253
|
+
"id='app'",
|
|
2254
|
+
"window.__INITIAL_STATE__",
|
|
2255
|
+
"data-reactroot",
|
|
2256
|
+
)
|
|
2257
|
+
|
|
2258
|
+
|
|
2259
|
+
def make_soup(html: str) -> BeautifulSoup:
|
|
2260
|
+
try:
|
|
2261
|
+
return BeautifulSoup(html or "", "lxml")
|
|
2262
|
+
except Exception:
|
|
2263
|
+
return BeautifulSoup(html or "", "html.parser")
|
|
2264
|
+
|
|
2265
|
+
|
|
2266
|
+
def visible_text(html: str) -> str:
|
|
2267
|
+
soup = make_soup(html)
|
|
2268
|
+
for tag in soup(list(_SKIP_TAGS)):
|
|
2269
|
+
tag.decompose()
|
|
2270
|
+
return re.sub(r"\s+", " ", soup.get_text(" ", strip=True)).strip()
|
|
2271
|
+
|
|
2272
|
+
|
|
2273
|
+
def looks_like_html_shell(html: str) -> bool:
|
|
2274
|
+
"""A tiny HTML document whose body is a script, cookie check or redirect.
|
|
2275
|
+
|
|
2276
|
+
php.cn and similar sites answer a plain HTTP request with a few dozen bytes
|
|
2277
|
+
that only a browser can resolve; without this, extraction reports an empty
|
|
2278
|
+
page even though the rendered page is readable.
|
|
2279
|
+
"""
|
|
2280
|
+
|
|
2281
|
+
if not html or len(html) > 4000:
|
|
2282
|
+
return False
|
|
2283
|
+
if len(visible_text(html)) >= 120:
|
|
2284
|
+
return False
|
|
2285
|
+
lowered = html.lower()
|
|
2286
|
+
return "<script" in lowered or "location" in lowered or "window." in lowered
|
|
2287
|
+
|
|
2288
|
+
|
|
2289
|
+
def looks_dynamic(html: str) -> bool:
|
|
2290
|
+
"""Heuristic: a short visible body plus SPA markers suggests JS rendering."""
|
|
2291
|
+
|
|
2292
|
+
if not html:
|
|
2293
|
+
return False
|
|
2294
|
+
head = html.lstrip()[:1000].lower()
|
|
2295
|
+
if head.startswith("<?xml") or head.startswith("<rss") or head.startswith("<feed"):
|
|
2296
|
+
return False
|
|
2297
|
+
script_count = html.lower().count("<script")
|
|
2298
|
+
marker_hit = any(marker in html for marker in _DYNAMIC_MARKERS)
|
|
2299
|
+
text = visible_text(html)
|
|
2300
|
+
if len(text) < 400:
|
|
2301
|
+
return marker_hit or script_count >= 3
|
|
2302
|
+
return False
|
|
2303
|
+
|
|
2304
|
+
|
|
2305
|
+
# ---------------------------------------------------------------------------
|
|
2306
|
+
# Markdown conversion
|
|
2307
|
+
# ---------------------------------------------------------------------------
|
|
2308
|
+
|
|
2309
|
+
_INLINE_WS_RE = re.compile(r"[ \t\r\f\v]+")
|
|
2310
|
+
_MULTI_NEWLINE_RE = re.compile(r"\n{3,}")
|
|
2311
|
+
|
|
2312
|
+
|
|
2313
|
+
def _normalize_markdown(markdown: str) -> str:
|
|
2314
|
+
text = (markdown or "").replace("\r\n", "\n").replace("\r", "\n")
|
|
2315
|
+
text = _MULTI_NEWLINE_RE.sub("\n\n", text)
|
|
2316
|
+
return text.strip()
|
|
2317
|
+
|
|
2318
|
+
|
|
2319
|
+
def _hard_split_block(block: str, size: int) -> list[str]:
|
|
2320
|
+
pieces: list[str] = []
|
|
2321
|
+
remaining = block.strip()
|
|
2322
|
+
while len(remaining) > size:
|
|
2323
|
+
cut = remaining.rfind("\n", 0, size)
|
|
2324
|
+
if cut < size // 2:
|
|
2325
|
+
for marker in ("。", "!", "?", ". ", "! ", "? "):
|
|
2326
|
+
candidate = remaining.rfind(marker, 0, size)
|
|
2327
|
+
if candidate >= size // 2:
|
|
2328
|
+
cut = candidate + len(marker)
|
|
2329
|
+
break
|
|
2330
|
+
if cut < size // 2:
|
|
2331
|
+
cut = remaining.rfind(" ", 0, size)
|
|
2332
|
+
if cut < size // 2:
|
|
2333
|
+
cut = size
|
|
2334
|
+
pieces.append(remaining[:cut].strip())
|
|
2335
|
+
remaining = remaining[cut:].strip()
|
|
2336
|
+
if remaining:
|
|
2337
|
+
pieces.append(remaining)
|
|
2338
|
+
return [piece for piece in pieces if piece]
|
|
2339
|
+
|
|
2340
|
+
|
|
2341
|
+
def chunk_markdown(markdown: str, size: int = 4000, overlap: int = 0) -> list[dict[str, Any]]:
|
|
2342
|
+
"""Split Markdown into context-sized chunks at paragraph boundaries.
|
|
2343
|
+
|
|
2344
|
+
Each chunk records the nearest preceding heading so an agent can tell
|
|
2345
|
+
which section it is reading without re-reading the whole document.
|
|
2346
|
+
"""
|
|
2347
|
+
|
|
2348
|
+
if size <= 0:
|
|
2349
|
+
return []
|
|
2350
|
+
text = _normalize_markdown(markdown)
|
|
2351
|
+
if not text:
|
|
2352
|
+
return []
|
|
2353
|
+
blocks = re.split(r"\n{2,}", text)
|
|
2354
|
+
chunks: list[dict[str, Any]] = []
|
|
2355
|
+
current: list[str] = []
|
|
2356
|
+
current_len = 0
|
|
2357
|
+
heading = ""
|
|
2358
|
+
overlap = max(0, min(overlap, size // 2))
|
|
2359
|
+
|
|
2360
|
+
def flush() -> None:
|
|
2361
|
+
nonlocal current, current_len
|
|
2362
|
+
body = "\n\n".join(current).strip()
|
|
2363
|
+
if body:
|
|
2364
|
+
chunks.append(
|
|
2365
|
+
{
|
|
2366
|
+
"index": len(chunks) + 1,
|
|
2367
|
+
"heading": heading,
|
|
2368
|
+
"chars": len(body),
|
|
2369
|
+
"text": body,
|
|
2370
|
+
}
|
|
2371
|
+
)
|
|
2372
|
+
current = []
|
|
2373
|
+
current_len = 0
|
|
2374
|
+
|
|
2375
|
+
for block in blocks:
|
|
2376
|
+
block = block.strip()
|
|
2377
|
+
if not block:
|
|
2378
|
+
continue
|
|
2379
|
+
match = re.match(r"^(#{1,6})\s+(.+)$", block)
|
|
2380
|
+
if match and len(block) < 300:
|
|
2381
|
+
heading = match.group(2).strip()
|
|
2382
|
+
pieces = _hard_split_block(block, size) if len(block) > size else [block]
|
|
2383
|
+
for piece in pieces:
|
|
2384
|
+
if current and current_len + len(piece) + 2 > size:
|
|
2385
|
+
flush()
|
|
2386
|
+
if overlap and chunks:
|
|
2387
|
+
tail = chunks[-1]["text"][-overlap:]
|
|
2388
|
+
current = [tail]
|
|
2389
|
+
current_len = len(tail)
|
|
2390
|
+
current.append(piece)
|
|
2391
|
+
current_len += len(piece) + 2
|
|
2392
|
+
flush()
|
|
2393
|
+
return chunks
|
|
2394
|
+
|
|
2395
|
+
|
|
2396
|
+
def html_to_markdown(
|
|
2397
|
+
html: str,
|
|
2398
|
+
url: Optional[str] = None,
|
|
2399
|
+
*,
|
|
2400
|
+
include_tables: bool = True,
|
|
2401
|
+
keep_links: bool = False,
|
|
2402
|
+
include_images: bool = False,
|
|
2403
|
+
) -> str:
|
|
2404
|
+
"""Convert HTML to clean Markdown, preferring trafilatura when available."""
|
|
2405
|
+
|
|
2406
|
+
markdown = _trafilatura_markdown(
|
|
2407
|
+
html,
|
|
2408
|
+
url=url,
|
|
2409
|
+
include_tables=include_tables,
|
|
2410
|
+
keep_links=keep_links,
|
|
2411
|
+
include_images=include_images,
|
|
2412
|
+
)
|
|
2413
|
+
if markdown and markdown.strip():
|
|
2414
|
+
return _normalize_markdown(markdown)
|
|
2415
|
+
|
|
2416
|
+
return _bs4_to_markdown(
|
|
2417
|
+
html,
|
|
2418
|
+
url=url,
|
|
2419
|
+
include_tables=include_tables,
|
|
2420
|
+
keep_links=keep_links,
|
|
2421
|
+
include_images=include_images,
|
|
2422
|
+
)
|
|
2423
|
+
|
|
2424
|
+
|
|
2425
|
+
def _trafilatura_markdown(
|
|
2426
|
+
html: str,
|
|
2427
|
+
*,
|
|
2428
|
+
url: Optional[str],
|
|
2429
|
+
include_tables: bool,
|
|
2430
|
+
keep_links: bool,
|
|
2431
|
+
include_images: bool,
|
|
2432
|
+
) -> str:
|
|
2433
|
+
try:
|
|
2434
|
+
import trafilatura # type: ignore
|
|
2435
|
+
except ImportError:
|
|
2436
|
+
return ""
|
|
2437
|
+
try:
|
|
2438
|
+
return trafilatura.extract(
|
|
2439
|
+
html,
|
|
2440
|
+
url=url,
|
|
2441
|
+
output_format="markdown",
|
|
2442
|
+
include_tables=include_tables,
|
|
2443
|
+
include_links=keep_links,
|
|
2444
|
+
include_images=include_images,
|
|
2445
|
+
include_comments=False,
|
|
2446
|
+
include_formatting=True,
|
|
2447
|
+
favor_recall=True,
|
|
2448
|
+
# A global deduplication cache would silently drop sentences seen in
|
|
2449
|
+
# an earlier page during a long agent session. Duplicate removal
|
|
2450
|
+
# inside one document is already handled by trafilatura itself.
|
|
2451
|
+
deduplicate=False,
|
|
2452
|
+
) or ""
|
|
2453
|
+
except Exception:
|
|
2454
|
+
return ""
|
|
2455
|
+
|
|
2456
|
+
|
|
2457
|
+
@dataclass
|
|
2458
|
+
class _RenderContext:
|
|
2459
|
+
url: Optional[str]
|
|
2460
|
+
include_tables: bool
|
|
2461
|
+
keep_links: bool
|
|
2462
|
+
include_images: bool
|
|
2463
|
+
|
|
2464
|
+
|
|
2465
|
+
def _bs4_to_markdown(
|
|
2466
|
+
html: str,
|
|
2467
|
+
url: Optional[str],
|
|
2468
|
+
*,
|
|
2469
|
+
include_tables: bool,
|
|
2470
|
+
keep_links: bool,
|
|
2471
|
+
include_images: bool,
|
|
2472
|
+
) -> str:
|
|
2473
|
+
"""A dependency-light fallback converter used when trafilatura is absent."""
|
|
2474
|
+
|
|
2475
|
+
soup = make_soup(html)
|
|
2476
|
+
for tag in soup(list(_SKIP_TAGS)):
|
|
2477
|
+
tag.decompose()
|
|
2478
|
+
for tag in soup.find_all(attrs={"role": ["navigation", "banner", "contentinfo", "complementary"]}):
|
|
2479
|
+
tag.decompose()
|
|
2480
|
+
|
|
2481
|
+
root: Tag = soup.find("article") or soup.find("main") or soup.body or soup # type: ignore[assignment]
|
|
2482
|
+
ctx = _RenderContext(
|
|
2483
|
+
url=url,
|
|
2484
|
+
include_tables=include_tables,
|
|
2485
|
+
keep_links=keep_links,
|
|
2486
|
+
include_images=include_images,
|
|
2487
|
+
)
|
|
2488
|
+
text = _render_children(root, ctx)
|
|
2489
|
+
return _normalize_markdown(text)
|
|
2490
|
+
|
|
2491
|
+
|
|
2492
|
+
def _render_children(node: Tag, ctx: _RenderContext) -> str:
|
|
2493
|
+
return "".join(_render_node(child, ctx) for child in node.children)
|
|
2494
|
+
|
|
2495
|
+
|
|
2496
|
+
def _render_node(node: Any, ctx: _RenderContext) -> str:
|
|
2497
|
+
if isinstance(node, NavigableString):
|
|
2498
|
+
text = str(node)
|
|
2499
|
+
if not text.strip():
|
|
2500
|
+
return "" if "\n" in text else " "
|
|
2501
|
+
return _INLINE_WS_RE.sub(" ", text)
|
|
2502
|
+
|
|
2503
|
+
if not isinstance(node, Tag):
|
|
2504
|
+
return ""
|
|
2505
|
+
|
|
2506
|
+
name = (node.name or "").lower()
|
|
2507
|
+
if name in _SKIP_TAGS:
|
|
2508
|
+
return ""
|
|
2509
|
+
if name in ("h1", "h2", "h3", "h4", "h5", "h6"):
|
|
2510
|
+
heading = _inline(node, ctx).strip()
|
|
2511
|
+
if not heading:
|
|
2512
|
+
return ""
|
|
2513
|
+
return f"\n\n{'#' * int(name[1])} {heading}\n\n"
|
|
2514
|
+
if name == "p":
|
|
2515
|
+
paragraph = _inline(node, ctx).strip()
|
|
2516
|
+
return f"\n\n{paragraph}\n\n" if paragraph else ""
|
|
2517
|
+
if name == "br":
|
|
2518
|
+
return " \n"
|
|
2519
|
+
if name == "hr":
|
|
2520
|
+
return "\n\n---\n\n"
|
|
2521
|
+
if name in ("ul", "ol"):
|
|
2522
|
+
listing = _render_list(node, ctx)
|
|
2523
|
+
return f"\n\n{listing}\n\n" if listing else ""
|
|
2524
|
+
if name == "pre":
|
|
2525
|
+
code = node.get_text().strip("\n")
|
|
2526
|
+
if not code.strip():
|
|
2527
|
+
return ""
|
|
2528
|
+
return f"\n\n```\n{code}\n```\n\n"
|
|
2529
|
+
if name == "blockquote":
|
|
2530
|
+
body = _render_children(node, ctx).strip()
|
|
2531
|
+
if not body:
|
|
2532
|
+
return ""
|
|
2533
|
+
quoted = "\n".join(f"> {line}" if line else ">" for line in body.split("\n"))
|
|
2534
|
+
return f"\n\n{quoted}\n\n"
|
|
2535
|
+
if name == "table":
|
|
2536
|
+
if ctx.include_tables:
|
|
2537
|
+
table_md = _table_markdown(node)
|
|
2538
|
+
if table_md:
|
|
2539
|
+
return f"\n\n{table_md}\n\n"
|
|
2540
|
+
return f"\n\n{_render_children(node, ctx).strip()}\n\n"
|
|
2541
|
+
if name == "img":
|
|
2542
|
+
if not ctx.include_images:
|
|
2543
|
+
return ""
|
|
2544
|
+
src = absolute_url(_attr_text(node, "src"), ctx.url)
|
|
2545
|
+
if not src:
|
|
2546
|
+
return ""
|
|
2547
|
+
alt = _clean_result_text(_attr_text(node, "alt"))
|
|
2548
|
+
return f""
|
|
2549
|
+
if name == "a":
|
|
2550
|
+
return _inline(node, ctx)
|
|
2551
|
+
if name == "code":
|
|
2552
|
+
code = node.get_text().strip("\n")
|
|
2553
|
+
return f"`{code}`" if code else ""
|
|
2554
|
+
if name in ("strong", "b"):
|
|
2555
|
+
inner = _inline(node, ctx).strip()
|
|
2556
|
+
return f"**{inner}**" if inner else ""
|
|
2557
|
+
if name in ("em", "i"):
|
|
2558
|
+
inner = _inline(node, ctx).strip()
|
|
2559
|
+
return f"*{inner}*" if inner else ""
|
|
2560
|
+
return _render_children(node, ctx)
|
|
2561
|
+
|
|
2562
|
+
|
|
2563
|
+
def _inline(node: Tag, ctx: _RenderContext) -> str:
|
|
2564
|
+
parts: list[str] = []
|
|
2565
|
+
for child in node.children:
|
|
2566
|
+
if isinstance(child, NavigableString):
|
|
2567
|
+
parts.append(_INLINE_WS_RE.sub(" ", str(child)))
|
|
2568
|
+
continue
|
|
2569
|
+
if not isinstance(child, Tag):
|
|
2570
|
+
continue
|
|
2571
|
+
name = (child.name or "").lower()
|
|
2572
|
+
if name in _SKIP_TAGS:
|
|
2573
|
+
continue
|
|
2574
|
+
if name == "br":
|
|
2575
|
+
parts.append(" ")
|
|
2576
|
+
elif name == "a":
|
|
2577
|
+
label = _clean_result_text(child.get_text(" ", strip=True))
|
|
2578
|
+
href = absolute_url(_attr_text(child, "href"), ctx.url)
|
|
2579
|
+
if ctx.keep_links and href and label:
|
|
2580
|
+
parts.append(f"[{label}]({href})")
|
|
2581
|
+
else:
|
|
2582
|
+
parts.append(label)
|
|
2583
|
+
elif name in ("strong", "b"):
|
|
2584
|
+
inner = _inline(child, ctx).strip()
|
|
2585
|
+
if inner:
|
|
2586
|
+
parts.append(f"**{inner}**")
|
|
2587
|
+
elif name in ("em", "i"):
|
|
2588
|
+
inner = _inline(child, ctx).strip()
|
|
2589
|
+
if inner:
|
|
2590
|
+
parts.append(f"*{inner}*")
|
|
2591
|
+
elif name == "code":
|
|
2592
|
+
code = child.get_text().strip("\n")
|
|
2593
|
+
if code:
|
|
2594
|
+
parts.append(f"`{code}`")
|
|
2595
|
+
elif name == "img":
|
|
2596
|
+
if ctx.include_images:
|
|
2597
|
+
src = absolute_url(_attr_text(child, "src"), ctx.url)
|
|
2598
|
+
if src:
|
|
2599
|
+
parts.append(f"")
|
|
2600
|
+
else:
|
|
2601
|
+
parts.append(_inline(child, ctx))
|
|
2602
|
+
text = "".join(parts)
|
|
2603
|
+
return re.sub(r"[ \t]{2,}", " ", text)
|
|
2604
|
+
|
|
2605
|
+
|
|
2606
|
+
def _render_list(node: Tag, ctx: _RenderContext) -> str:
|
|
2607
|
+
ordered = (node.name or "").lower() == "ol"
|
|
2608
|
+
try:
|
|
2609
|
+
index = _int_attr(node, "start", 1)
|
|
2610
|
+
except (TypeError, ValueError):
|
|
2611
|
+
index = 1
|
|
2612
|
+
|
|
2613
|
+
lines: list[str] = []
|
|
2614
|
+
for item in node.find_all("li", recursive=False):
|
|
2615
|
+
body = _render_children(item, ctx).strip()
|
|
2616
|
+
body = re.sub(r"\n{2,}", "\n", body)
|
|
2617
|
+
body = body.replace("\n", "\n ")
|
|
2618
|
+
marker = f"{index}." if ordered else "-"
|
|
2619
|
+
lines.append(f"{marker} {body}".rstrip())
|
|
2620
|
+
index += 1
|
|
2621
|
+
return "\n".join(lines)
|
|
2622
|
+
|
|
2623
|
+
|
|
2624
|
+
def _escape_table_cell(value: str) -> str:
|
|
2625
|
+
return value.replace("|", r"\|").replace("\n", " ").strip()
|
|
2626
|
+
|
|
2627
|
+
|
|
2628
|
+
def _table_markdown(table: Tag) -> str:
|
|
2629
|
+
rows: list[list[str]] = []
|
|
2630
|
+
for tr in table.find_all("tr"):
|
|
2631
|
+
# Skip rows that belong to a nested table.
|
|
2632
|
+
if tr.find_parent("table") is not table:
|
|
2633
|
+
continue
|
|
2634
|
+
cells: list[str] = []
|
|
2635
|
+
for cell in tr.find_all(["th", "td"]):
|
|
2636
|
+
if cell.find_parent("tr") is not tr:
|
|
2637
|
+
continue
|
|
2638
|
+
cells.append(_clean_result_text(cell.get_text(" ", strip=True)))
|
|
2639
|
+
if cells:
|
|
2640
|
+
rows.append(cells)
|
|
2641
|
+
|
|
2642
|
+
if len(rows) < 2:
|
|
2643
|
+
return ""
|
|
2644
|
+
width = max(len(row) for row in rows)
|
|
2645
|
+
if width < 2:
|
|
2646
|
+
return ""
|
|
2647
|
+
rows = [row + [""] * (width - len(row)) for row in rows]
|
|
2648
|
+
|
|
2649
|
+
header = rows[0]
|
|
2650
|
+
body = rows[1:]
|
|
2651
|
+
lines = [
|
|
2652
|
+
"| " + " | ".join(_escape_table_cell(cell) for cell in header) + " |",
|
|
2653
|
+
"| " + " | ".join("---" for _ in range(width)) + " |",
|
|
2654
|
+
]
|
|
2655
|
+
for row in body:
|
|
2656
|
+
lines.append("| " + " | ".join(_escape_table_cell(cell) for cell in row) + " |")
|
|
2657
|
+
return "\n".join(lines)
|
|
2658
|
+
|
|
2659
|
+
|
|
2660
|
+
def extract_tables(html: str, *, max_tables: int = 20, max_columns: int = 40) -> list[dict[str, Any]]:
|
|
2661
|
+
"""Extract data tables from HTML as structured rows plus Markdown."""
|
|
2662
|
+
|
|
2663
|
+
soup = make_soup(html)
|
|
2664
|
+
tables: list[dict[str, Any]] = []
|
|
2665
|
+
for table in soup.find_all("table"):
|
|
2666
|
+
caption_tag = table.find("caption")
|
|
2667
|
+
caption = _clean_result_text(caption_tag.get_text(" ", strip=True)) if caption_tag else ""
|
|
2668
|
+
rows: list[list[str]] = []
|
|
2669
|
+
for tr in table.find_all("tr"):
|
|
2670
|
+
if tr.find_parent("table") is not table:
|
|
2671
|
+
continue
|
|
2672
|
+
cells = [
|
|
2673
|
+
_clean_result_text(cell.get_text(" ", strip=True))
|
|
2674
|
+
for cell in tr.find_all(["th", "td"])
|
|
2675
|
+
if cell.find_parent("tr") is tr
|
|
2676
|
+
]
|
|
2677
|
+
if cells:
|
|
2678
|
+
rows.append(cells[:max_columns])
|
|
2679
|
+
if len(rows) < 2:
|
|
2680
|
+
continue
|
|
2681
|
+
width = max(len(row) for row in rows)
|
|
2682
|
+
if width < 2:
|
|
2683
|
+
continue
|
|
2684
|
+
rows = [row + [""] * (width - len(row)) for row in rows]
|
|
2685
|
+
tables.append(
|
|
2686
|
+
{
|
|
2687
|
+
"caption": caption,
|
|
2688
|
+
"headers": rows[0],
|
|
2689
|
+
"rows": rows[1:],
|
|
2690
|
+
"markdown": _table_markdown(table),
|
|
2691
|
+
}
|
|
2692
|
+
)
|
|
2693
|
+
if len(tables) >= max_tables:
|
|
2694
|
+
break
|
|
2695
|
+
return tables
|
|
2696
|
+
|
|
2697
|
+
|
|
2698
|
+
# ---------------------------------------------------------------------------
|
|
2699
|
+
# Metadata extraction
|
|
2700
|
+
# ---------------------------------------------------------------------------
|
|
2701
|
+
|
|
2702
|
+
_DOI_RE = re.compile(r"\b10\.\d{4,9}/[^\s\"'<>]+", re.I)
|
|
2703
|
+
_ARXIV_RE = re.compile(r"arxiv\.org/(?:abs|pdf)/([0-9]{4}\.[0-9]{4,5}|[a-z\-]+/[0-9]{7})", re.I)
|
|
2704
|
+
|
|
2705
|
+
_ARTICLE_TYPES = {
|
|
2706
|
+
"article",
|
|
2707
|
+
"scholarlyarticle",
|
|
2708
|
+
"newsarticle",
|
|
2709
|
+
"blogposting",
|
|
2710
|
+
"techarticle",
|
|
2711
|
+
"report",
|
|
2712
|
+
"webpage",
|
|
2713
|
+
}
|
|
2714
|
+
|
|
2715
|
+
|
|
2716
|
+
def _meta_map(soup: BeautifulSoup) -> dict[str, list[str]]:
|
|
2717
|
+
mapping: dict[str, list[str]] = {}
|
|
2718
|
+
for tag in soup.find_all("meta"):
|
|
2719
|
+
key = tag.get("name") or tag.get("property") or tag.get("itemprop")
|
|
2720
|
+
value = tag.get("content")
|
|
2721
|
+
if key and value:
|
|
2722
|
+
mapping.setdefault(str(key).strip().lower(), []).append(str(value).strip())
|
|
2723
|
+
return mapping
|
|
2724
|
+
|
|
2725
|
+
|
|
2726
|
+
def _first(mapping: dict[str, list[str]], *keys: str) -> str:
|
|
2727
|
+
for key in keys:
|
|
2728
|
+
values = mapping.get(key.lower())
|
|
2729
|
+
if values:
|
|
2730
|
+
for value in values:
|
|
2731
|
+
if value:
|
|
2732
|
+
return value
|
|
2733
|
+
return ""
|
|
2734
|
+
|
|
2735
|
+
|
|
2736
|
+
def _all(mapping: dict[str, list[str]], *keys: str) -> list[str]:
|
|
2737
|
+
values: list[str] = []
|
|
2738
|
+
for key in keys:
|
|
2739
|
+
values.extend(mapping.get(key.lower(), []))
|
|
2740
|
+
return [value for value in values if value]
|
|
2741
|
+
|
|
2742
|
+
|
|
2743
|
+
def _iter_jsonld(soup: BeautifulSoup) -> Iterator[dict[str, Any]]:
|
|
2744
|
+
for tag in soup.find_all("script"):
|
|
2745
|
+
script_type = (tag.get("type") or "").lower()
|
|
2746
|
+
if "ld+json" not in script_type:
|
|
2747
|
+
continue
|
|
2748
|
+
raw = tag.string or tag.get_text()
|
|
2749
|
+
if not raw:
|
|
2750
|
+
continue
|
|
2751
|
+
try:
|
|
2752
|
+
data = json.loads(raw)
|
|
2753
|
+
except (json.JSONDecodeError, TypeError):
|
|
2754
|
+
continue
|
|
2755
|
+
yield from _flatten_jsonld(data)
|
|
2756
|
+
|
|
2757
|
+
|
|
2758
|
+
def _flatten_jsonld(data: Any) -> Iterator[dict[str, Any]]:
|
|
2759
|
+
if isinstance(data, list):
|
|
2760
|
+
for item in data:
|
|
2761
|
+
yield from _flatten_jsonld(item)
|
|
2762
|
+
elif isinstance(data, dict):
|
|
2763
|
+
if "@graph" in data:
|
|
2764
|
+
yield from _flatten_jsonld(data["@graph"])
|
|
2765
|
+
yield data
|
|
2766
|
+
|
|
2767
|
+
|
|
2768
|
+
def _jsonld_article(soup: BeautifulSoup) -> dict[str, Any]:
|
|
2769
|
+
fallback: dict[str, Any] = {}
|
|
2770
|
+
for item in _iter_jsonld(soup):
|
|
2771
|
+
raw_type = item.get("@type")
|
|
2772
|
+
types = raw_type if isinstance(raw_type, list) else [raw_type]
|
|
2773
|
+
lowered = {str(value).lower() for value in types if value}
|
|
2774
|
+
if lowered & _ARTICLE_TYPES:
|
|
2775
|
+
return item
|
|
2776
|
+
if not fallback and lowered & {"organization", "website", "product"}:
|
|
2777
|
+
fallback = item
|
|
2778
|
+
return fallback
|
|
2779
|
+
|
|
2780
|
+
|
|
2781
|
+
def _jsonld_names(value: Any) -> list[str]:
|
|
2782
|
+
if not value:
|
|
2783
|
+
return []
|
|
2784
|
+
if isinstance(value, str):
|
|
2785
|
+
return [value.strip()] if value.strip() else []
|
|
2786
|
+
if isinstance(value, dict):
|
|
2787
|
+
name = value.get("name") or value.get("legalName")
|
|
2788
|
+
return [str(name).strip()] if name else []
|
|
2789
|
+
if isinstance(value, list):
|
|
2790
|
+
names: list[str] = []
|
|
2791
|
+
for item in value:
|
|
2792
|
+
names.extend(_jsonld_names(item))
|
|
2793
|
+
return names
|
|
2794
|
+
return []
|
|
2795
|
+
|
|
2796
|
+
|
|
2797
|
+
def _jsonld_publisher(value: Any) -> str:
|
|
2798
|
+
if isinstance(value, dict):
|
|
2799
|
+
name = value.get("name")
|
|
2800
|
+
if name:
|
|
2801
|
+
return str(name).strip()
|
|
2802
|
+
parent = value.get("parentOrganization")
|
|
2803
|
+
return _jsonld_publisher(parent) if parent else ""
|
|
2804
|
+
if isinstance(value, list) and value:
|
|
2805
|
+
return _jsonld_publisher(value[0])
|
|
2806
|
+
if isinstance(value, str):
|
|
2807
|
+
return value.strip()
|
|
2808
|
+
return ""
|
|
2809
|
+
|
|
2810
|
+
|
|
2811
|
+
def _clean_doi(value: str) -> str:
|
|
2812
|
+
return value.rstrip(".,;:)]}。,;").strip()
|
|
2813
|
+
|
|
2814
|
+
|
|
2815
|
+
def extract_metadata(html: str, url: Optional[str] = None) -> dict[str, Any]:
|
|
2816
|
+
"""Extract bibliographic and social metadata without external services."""
|
|
2817
|
+
|
|
2818
|
+
soup = make_soup(html)
|
|
2819
|
+
meta = _meta_map(soup)
|
|
2820
|
+
jsonld = _jsonld_article(soup)
|
|
2821
|
+
|
|
2822
|
+
title = _first(meta, "citation_title", "og:title", "twitter:title", "dc.title")
|
|
2823
|
+
if not title:
|
|
2824
|
+
title_tag = soup.find("title")
|
|
2825
|
+
title = _clean_result_text(title_tag.get_text() if title_tag else "")
|
|
2826
|
+
if not title:
|
|
2827
|
+
title = str(jsonld.get("headline") or jsonld.get("name") or "").strip()
|
|
2828
|
+
|
|
2829
|
+
authors = _all(meta, "citation_author", "dc.creator", "dcterms.creator", "author")
|
|
2830
|
+
if not authors:
|
|
2831
|
+
authors = _jsonld_names(jsonld.get("author"))
|
|
2832
|
+
|
|
2833
|
+
published = _first(
|
|
2834
|
+
meta,
|
|
2835
|
+
"citation_publication_date",
|
|
2836
|
+
"citation_date",
|
|
2837
|
+
"article:published_time",
|
|
2838
|
+
"date",
|
|
2839
|
+
"dc.date",
|
|
2840
|
+
"dcterms.date",
|
|
2841
|
+
) or str(jsonld.get("datePublished") or "").strip()
|
|
2842
|
+
|
|
2843
|
+
modified = _first(meta, "article:modified_time", "og:updated_time") or str(
|
|
2844
|
+
jsonld.get("dateModified") or ""
|
|
2845
|
+
).strip()
|
|
2846
|
+
|
|
2847
|
+
journal = _first(meta, "citation_journal_title", "citation_conference_title")
|
|
2848
|
+
if not journal:
|
|
2849
|
+
journal = _jsonld_publisher(jsonld.get("isPartOf"))
|
|
2850
|
+
|
|
2851
|
+
doi = _first(meta, "citation_doi", "dc.identifier", "dcterms.identifier")
|
|
2852
|
+
if not doi:
|
|
2853
|
+
candidate = ""
|
|
2854
|
+
if url:
|
|
2855
|
+
match = _DOI_RE.search(url)
|
|
2856
|
+
if match:
|
|
2857
|
+
candidate = match.group(0)
|
|
2858
|
+
if not candidate:
|
|
2859
|
+
match = _DOI_RE.search(html[:500_000])
|
|
2860
|
+
if match:
|
|
2861
|
+
candidate = match.group(0)
|
|
2862
|
+
doi = candidate
|
|
2863
|
+
doi = _clean_doi(doi)
|
|
2864
|
+
|
|
2865
|
+
abstract = _first(
|
|
2866
|
+
meta, "citation_abstract", "dc.description", "og:description", "description", "twitter:description"
|
|
2867
|
+
) or str(jsonld.get("description") or "").strip()
|
|
2868
|
+
|
|
2869
|
+
keywords_raw = _all(meta, "citation_keywords", "keywords")
|
|
2870
|
+
jsonld_keywords = jsonld.get("keywords")
|
|
2871
|
+
if isinstance(jsonld_keywords, str):
|
|
2872
|
+
keywords_raw.append(jsonld_keywords)
|
|
2873
|
+
elif isinstance(jsonld_keywords, list):
|
|
2874
|
+
keywords_raw.extend(str(item) for item in jsonld_keywords)
|
|
2875
|
+
keywords: list[str] = []
|
|
2876
|
+
for chunk in keywords_raw:
|
|
2877
|
+
for word in re.split(r"[,;,;]", chunk):
|
|
2878
|
+
word = word.strip()
|
|
2879
|
+
if word and word not in keywords:
|
|
2880
|
+
keywords.append(word)
|
|
2881
|
+
|
|
2882
|
+
canonical = ""
|
|
2883
|
+
for link in soup.find_all("link"):
|
|
2884
|
+
rel = [str(value).lower() for value in (link.attrs.get("rel") or [])]
|
|
2885
|
+
if "canonical" in rel and _attr_text(link, "href"):
|
|
2886
|
+
canonical = absolute_url(_attr_text(link, "href"), url)
|
|
2887
|
+
if canonical:
|
|
2888
|
+
break
|
|
2889
|
+
canonical = canonical or _first(meta, "og:url")
|
|
2890
|
+
|
|
2891
|
+
language = ""
|
|
2892
|
+
html_tag = soup.find("html")
|
|
2893
|
+
if html_tag and _attr_text(html_tag, "lang"):
|
|
2894
|
+
language = _attr_text(html_tag, "lang")
|
|
2895
|
+
language = language or _first(meta, "content-language", "dc.language") or str(
|
|
2896
|
+
jsonld.get("inLanguage") or ""
|
|
2897
|
+
).strip()
|
|
2898
|
+
|
|
2899
|
+
arxiv_id = _first(meta, "citation_arxiv_id")
|
|
2900
|
+
if not arxiv_id:
|
|
2901
|
+
match = _ARXIV_RE.search(url or "") or _ARXIV_RE.search(html[:200_000])
|
|
2902
|
+
if match:
|
|
2903
|
+
arxiv_id = match.group(1)
|
|
2904
|
+
|
|
2905
|
+
return {
|
|
2906
|
+
"title": title,
|
|
2907
|
+
"authors": [author for author in authors if author],
|
|
2908
|
+
"published": published,
|
|
2909
|
+
"modified": modified,
|
|
2910
|
+
"journal": journal,
|
|
2911
|
+
"volume": _first(meta, "citation_volume"),
|
|
2912
|
+
"issue": _first(meta, "citation_issue"),
|
|
2913
|
+
"first_page": _first(meta, "citation_firstpage"),
|
|
2914
|
+
"last_page": _first(meta, "citation_lastpage"),
|
|
2915
|
+
"doi": doi,
|
|
2916
|
+
"arxiv_id": arxiv_id,
|
|
2917
|
+
"abstract": abstract,
|
|
2918
|
+
"keywords": keywords,
|
|
2919
|
+
"language": language,
|
|
2920
|
+
"site_name": _first(meta, "og:site_name", "application-name") or str(
|
|
2921
|
+
jsonld.get("publisher", {}).get("name")
|
|
2922
|
+
if isinstance(jsonld.get("publisher"), dict)
|
|
2923
|
+
else ""
|
|
2924
|
+
).strip(),
|
|
2925
|
+
"canonical_url": canonical,
|
|
2926
|
+
"og_type": _first(meta, "og:type"),
|
|
2927
|
+
}
|
|
2928
|
+
|
|
2929
|
+
|
|
2930
|
+
# ---------------------------------------------------------------------------
|
|
2931
|
+
# Optional dynamic rendering
|
|
2932
|
+
# ---------------------------------------------------------------------------
|
|
2933
|
+
|
|
2934
|
+
|
|
2935
|
+
def render_html(url: str, config: FetchConfig) -> str:
|
|
2936
|
+
"""Render a page with Playwright/Chromium (optional dependency)."""
|
|
2937
|
+
|
|
2938
|
+
assert_public_url(url, allow_private=config.allow_private)
|
|
2939
|
+
try:
|
|
2940
|
+
from playwright.sync_api import sync_playwright # type: ignore
|
|
2941
|
+
except ImportError as exc:
|
|
2942
|
+
raise RenderUnavailable(
|
|
2943
|
+
"Playwright is not installed; dynamic rendering is disabled. "
|
|
2944
|
+
"Enable it with: pip install playwright && playwright install chromium"
|
|
2945
|
+
) from exc
|
|
2946
|
+
except Exception as exc:
|
|
2947
|
+
raise RenderUnavailable(
|
|
2948
|
+
f"Playwright could not be imported ({exc}). Reinstall it with: "
|
|
2949
|
+
"pip install playwright && playwright install chromium"
|
|
2950
|
+
) from exc
|
|
2951
|
+
|
|
2952
|
+
timeout_ms = max(5_000, int(config.timeout * 1000))
|
|
2953
|
+
try:
|
|
2954
|
+
with sync_playwright() as playwright:
|
|
2955
|
+
browser = playwright.chromium.launch(headless=True)
|
|
2956
|
+
try:
|
|
2957
|
+
context = browser.new_context(user_agent=config.user_agent)
|
|
2958
|
+
page = context.new_page()
|
|
2959
|
+
page.set_default_timeout(timeout_ms)
|
|
2960
|
+
page.goto(url, wait_until="domcontentloaded", timeout=timeout_ms)
|
|
2961
|
+
# A JS or meta redirect can land on a private address; check
|
|
2962
|
+
# where the document actually ended up before reading it.
|
|
2963
|
+
assert_public_url(page.url, allow_private=config.allow_private)
|
|
2964
|
+
try:
|
|
2965
|
+
page.wait_for_load_state("networkidle", timeout=min(5_000, timeout_ms))
|
|
2966
|
+
except Exception:
|
|
2967
|
+
pass
|
|
2968
|
+
if config.wait_for:
|
|
2969
|
+
try:
|
|
2970
|
+
page.wait_for_selector(config.wait_for, timeout=min(10_000, timeout_ms))
|
|
2971
|
+
except Exception:
|
|
2972
|
+
pass
|
|
2973
|
+
for _ in range(config.scroll):
|
|
2974
|
+
try:
|
|
2975
|
+
page.mouse.wheel(0, 1800)
|
|
2976
|
+
page.wait_for_timeout(350)
|
|
2977
|
+
except Exception:
|
|
2978
|
+
break
|
|
2979
|
+
page.wait_for_timeout(500)
|
|
2980
|
+
html = page.content()
|
|
2981
|
+
context.close()
|
|
2982
|
+
return html
|
|
2983
|
+
finally:
|
|
2984
|
+
browser.close()
|
|
2985
|
+
except RenderUnavailable:
|
|
2986
|
+
raise
|
|
2987
|
+
except Exception as exc:
|
|
2988
|
+
raise RenderUnavailable(f"Playwright rendering failed: {exc}") from exc
|
|
2989
|
+
|
|
2990
|
+
|
|
2991
|
+
# ---------------------------------------------------------------------------
|
|
2992
|
+
# Link discovery, sessions and agentic browsing
|
|
2993
|
+
# ---------------------------------------------------------------------------
|
|
2994
|
+
|
|
2995
|
+
_SKIP_LINK_WORDS = (
|
|
2996
|
+
"登录",
|
|
2997
|
+
"注册",
|
|
2998
|
+
"隐私",
|
|
2999
|
+
"免责声明",
|
|
3000
|
+
"广告",
|
|
3001
|
+
"login",
|
|
3002
|
+
"sign in",
|
|
3003
|
+
"sign up",
|
|
3004
|
+
"register",
|
|
3005
|
+
"privacy",
|
|
3006
|
+
"cookie",
|
|
3007
|
+
)
|
|
3008
|
+
_SKIP_LINK_EXTENSIONS = (".zip", ".rar", ".7z", ".exe", ".dmg", ".apk", ".mp4", ".mp3", ".iso")
|
|
3009
|
+
|
|
3010
|
+
|
|
3011
|
+
def _link_in_skipped_region(anchor: Tag) -> bool:
|
|
3012
|
+
"""True for chrome (nav, forms, page header) but not an article's header.
|
|
3013
|
+
|
|
3014
|
+
A results page commonly wraps each card's title in ``<article><header>``;
|
|
3015
|
+
skipping every ``<header>`` silently dropped the very links a caller wants.
|
|
3016
|
+
"""
|
|
3017
|
+
|
|
3018
|
+
inside_content = anchor.find_parent(["article", "main"]) is not None
|
|
3019
|
+
for parent in anchor.parents:
|
|
3020
|
+
name = parent.name
|
|
3021
|
+
if name not in _SKIP_TAGS:
|
|
3022
|
+
continue
|
|
3023
|
+
if name == "header" and inside_content:
|
|
3024
|
+
continue
|
|
3025
|
+
return True
|
|
3026
|
+
return False
|
|
3027
|
+
|
|
3028
|
+
|
|
3029
|
+
def extract_links(
|
|
3030
|
+
html: str,
|
|
3031
|
+
base_url: str,
|
|
3032
|
+
query: str = "",
|
|
3033
|
+
limit: int = 20,
|
|
3034
|
+
same_site: bool = True,
|
|
3035
|
+
) -> list[dict[str, Any]]:
|
|
3036
|
+
"""Rank in-page links for a follow-up visit.
|
|
3037
|
+
|
|
3038
|
+
Ordering follows the query: anchor text that covers more query tokens
|
|
3039
|
+
ranks first; ties keep document order (how a human scans a page).
|
|
3040
|
+
"""
|
|
3041
|
+
|
|
3042
|
+
soup = make_soup(html)
|
|
3043
|
+
base_host = urllib.parse.urlsplit(base_url).netloc.lower()
|
|
3044
|
+
tokens = _query_tokens(query)
|
|
3045
|
+
candidates: list[dict[str, Any]] = []
|
|
3046
|
+
seen: set[str] = set()
|
|
3047
|
+
order = 0
|
|
3048
|
+
|
|
3049
|
+
for anchor in soup.find_all("a", href=True):
|
|
3050
|
+
if _link_in_skipped_region(anchor):
|
|
3051
|
+
continue
|
|
3052
|
+
url = absolute_url(_attr_text(anchor, "href"), base_url)
|
|
3053
|
+
if not url or url in seen:
|
|
3054
|
+
continue
|
|
3055
|
+
parts = urllib.parse.urlsplit(url)
|
|
3056
|
+
if parts.scheme not in _ALLOWED_SCHEMES:
|
|
3057
|
+
continue
|
|
3058
|
+
if same_site and parts.netloc.lower() != base_host:
|
|
3059
|
+
continue
|
|
3060
|
+
text = _clean_result_text(anchor.get_text(" ", strip=True))
|
|
3061
|
+
if len(text) < 3:
|
|
3062
|
+
continue
|
|
3063
|
+
lowered = text.lower()
|
|
3064
|
+
if any(word in lowered for word in _SKIP_LINK_WORDS):
|
|
3065
|
+
continue
|
|
3066
|
+
if any(parts.path.lower().endswith(ext) for ext in _SKIP_LINK_EXTENSIONS):
|
|
3067
|
+
continue
|
|
3068
|
+
score = result_coverage({"title": text, "snippet": "", "url": url}, tokens)
|
|
3069
|
+
candidates.append(
|
|
3070
|
+
{"url": url, "text": text, "score": round(score, 3), "order": order}
|
|
3071
|
+
)
|
|
3072
|
+
seen.add(url)
|
|
3073
|
+
order += 1
|
|
3074
|
+
|
|
3075
|
+
candidates.sort(key=lambda item: (-item["score"], item["order"]))
|
|
3076
|
+
return candidates[:limit]
|
|
3077
|
+
|
|
3078
|
+
|
|
3079
|
+
def session_path(name: str) -> Path:
|
|
3080
|
+
safe = re.sub(r"[^A-Za-z0-9._-]+", "-", str(name or "")).strip("-._") or "default"
|
|
3081
|
+
return Path.home() / ".web-forager" / "sessions" / f"{safe}.json"
|
|
3082
|
+
|
|
3083
|
+
|
|
3084
|
+
def load_session(name: str) -> dict[str, Any]:
|
|
3085
|
+
path = session_path(name)
|
|
3086
|
+
try:
|
|
3087
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
3088
|
+
except (OSError, json.JSONDecodeError):
|
|
3089
|
+
data = {}
|
|
3090
|
+
if not isinstance(data, dict):
|
|
3091
|
+
data = {}
|
|
3092
|
+
data.setdefault("visited", [])
|
|
3093
|
+
data.setdefault("trail", [])
|
|
3094
|
+
return data
|
|
3095
|
+
|
|
3096
|
+
|
|
3097
|
+
def save_session(name: str, data: dict[str, Any]) -> Path:
|
|
3098
|
+
path = session_path(name)
|
|
3099
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
3100
|
+
payload = {
|
|
3101
|
+
"visited": list(data.get("visited", []))[-1000:],
|
|
3102
|
+
"trail": list(data.get("trail", []))[-500:],
|
|
3103
|
+
"updated_at": utc_now(),
|
|
3104
|
+
}
|
|
3105
|
+
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
3106
|
+
return path
|
|
3107
|
+
|
|
3108
|
+
|
|
3109
|
+
def surf_web(
|
|
3110
|
+
seed: str,
|
|
3111
|
+
config: Optional[FetchConfig] = None,
|
|
3112
|
+
depth: int = 1,
|
|
3113
|
+
follow: Optional[str] = None,
|
|
3114
|
+
max_pages: Optional[int] = None,
|
|
3115
|
+
engine_name: str = "auto",
|
|
3116
|
+
session: Optional[str] = None,
|
|
3117
|
+
fetcher: Optional[HttpFetcher] = None,
|
|
3118
|
+
) -> dict[str, Any]:
|
|
3119
|
+
"""Search or open a page, then follow ranked links like a human browsing.
|
|
3120
|
+
|
|
3121
|
+
``--session`` makes the walk resumable: visited URLs are skipped on the
|
|
3122
|
+
next run and the trail is appended to ``~/.web-forager/sessions/<name>.json``.
|
|
3123
|
+
"""
|
|
3124
|
+
|
|
3125
|
+
config = config or FetchConfig()
|
|
3126
|
+
config.validate()
|
|
3127
|
+
follow = follow or config.follow
|
|
3128
|
+
if follow not in ("same-site", "all", "none"):
|
|
3129
|
+
raise ValueError("follow must be same-site, all or none")
|
|
3130
|
+
config.follow = follow
|
|
3131
|
+
config.extract_links = True
|
|
3132
|
+
max_pages = max_pages or config.max_pages
|
|
3133
|
+
|
|
3134
|
+
own_fetcher = fetcher is None
|
|
3135
|
+
fetcher = fetcher or HttpFetcher(config)
|
|
3136
|
+
session_data = load_session(session) if session else {"visited": [], "trail": []}
|
|
3137
|
+
visited: set[str] = set(session_data.get("visited", []))
|
|
3138
|
+
trail: list[dict[str, Any]] = list(session_data.get("trail", []))
|
|
3139
|
+
pages: list[dict[str, Any]] = []
|
|
3140
|
+
search_results: list[dict[str, str]] = []
|
|
3141
|
+
render_fallbacks = 0
|
|
3142
|
+
|
|
3143
|
+
seed_text = str(seed or "").strip()
|
|
3144
|
+
if not seed_text:
|
|
3145
|
+
raise ValueError("surf needs a URL or a search query")
|
|
3146
|
+
is_url = seed_text.lower().startswith(("http://", "https://"))
|
|
3147
|
+
query = "" if is_url else seed_text
|
|
3148
|
+
|
|
3149
|
+
try:
|
|
3150
|
+
if is_url:
|
|
3151
|
+
seeds = [normalize_url(seed_text)]
|
|
3152
|
+
else:
|
|
3153
|
+
search_results = search(
|
|
3154
|
+
query, config, engine=engine_name, limit=config.num_results, fetcher=fetcher
|
|
3155
|
+
)
|
|
3156
|
+
seeds = [item["url"] for item in search_results[:max_pages]]
|
|
3157
|
+
|
|
3158
|
+
queue: list[tuple[str, str, int]] = [(url, "search" if not is_url else "seed", 0) for url in seeds]
|
|
3159
|
+
|
|
3160
|
+
while queue and len(pages) < max_pages:
|
|
3161
|
+
url, origin, level = queue.pop(0)
|
|
3162
|
+
if url in visited:
|
|
3163
|
+
continue
|
|
3164
|
+
visited.add(url)
|
|
3165
|
+
try:
|
|
3166
|
+
page = fetch_page(
|
|
3167
|
+
url,
|
|
3168
|
+
config,
|
|
3169
|
+
fetcher=fetcher,
|
|
3170
|
+
query=query,
|
|
3171
|
+
allow_render_fallback=(
|
|
3172
|
+
config.render == "auto" and render_fallbacks < 2
|
|
3173
|
+
),
|
|
3174
|
+
)
|
|
3175
|
+
if page.get("render_fallback"):
|
|
3176
|
+
render_fallbacks += 1
|
|
3177
|
+
except (
|
|
3178
|
+
RobotsDenied,
|
|
3179
|
+
FetchError,
|
|
3180
|
+
ExtractionError,
|
|
3181
|
+
RenderUnavailable,
|
|
3182
|
+
PdfUnavailable,
|
|
3183
|
+
PdfExtractionError,
|
|
3184
|
+
) as exc:
|
|
3185
|
+
pages.append({"url": url, "error": f"{exc.__class__.__name__}: {exc}"})
|
|
3186
|
+
trail.append(
|
|
3187
|
+
{
|
|
3188
|
+
"step": len(trail) + 1,
|
|
3189
|
+
"level": level,
|
|
3190
|
+
"from": origin,
|
|
3191
|
+
"url": url,
|
|
3192
|
+
"error": f"{exc.__class__.__name__}: {exc}",
|
|
3193
|
+
}
|
|
3194
|
+
)
|
|
3195
|
+
continue
|
|
3196
|
+
|
|
3197
|
+
pages.append(page)
|
|
3198
|
+
trail.append(
|
|
3199
|
+
{
|
|
3200
|
+
"step": len(trail) + 1,
|
|
3201
|
+
"level": level,
|
|
3202
|
+
"from": origin,
|
|
3203
|
+
"url": url,
|
|
3204
|
+
"title": page.get("title", ""),
|
|
3205
|
+
}
|
|
3206
|
+
)
|
|
3207
|
+
|
|
3208
|
+
if level < depth and follow != "none":
|
|
3209
|
+
for link in page.get("links", [])[: config.max_links]:
|
|
3210
|
+
if link["url"] in visited:
|
|
3211
|
+
continue
|
|
3212
|
+
queue.append((link["url"], url, level + 1))
|
|
3213
|
+
|
|
3214
|
+
return {
|
|
3215
|
+
"mode": "surf",
|
|
3216
|
+
"seed": seed_text,
|
|
3217
|
+
"query": query,
|
|
3218
|
+
"engine": engine_name,
|
|
3219
|
+
"depth": depth,
|
|
3220
|
+
"follow": follow,
|
|
3221
|
+
"session": session or "",
|
|
3222
|
+
"search_results": search_results,
|
|
3223
|
+
"trail": trail,
|
|
3224
|
+
"visited": sorted(visited),
|
|
3225
|
+
"pages": pages,
|
|
3226
|
+
}
|
|
3227
|
+
finally:
|
|
3228
|
+
if session:
|
|
3229
|
+
save_session(session, {"visited": sorted(visited), "trail": trail})
|
|
3230
|
+
if own_fetcher:
|
|
3231
|
+
fetcher.close()
|
|
3232
|
+
|
|
3233
|
+
|
|
3234
|
+
# ---------------------------------------------------------------------------
|
|
3235
|
+
# High-level pipeline
|
|
3236
|
+
# ---------------------------------------------------------------------------
|
|
3237
|
+
|
|
3238
|
+
|
|
3239
|
+
# Interstitials served instead of content. They are checked only when the
|
|
3240
|
+
# extracted body is short, so an article *about* verification is not flagged.
|
|
3241
|
+
_VERIFICATION_PATTERNS = (
|
|
3242
|
+
"verify you are human",
|
|
3243
|
+
"unusual traffic",
|
|
3244
|
+
"enable javascript and cookies to continue",
|
|
3245
|
+
"complete the security check",
|
|
3246
|
+
)
|
|
3247
|
+
_WECHAT_VERIFICATION_PATTERNS = (
|
|
3248
|
+
"环境异常",
|
|
3249
|
+
"完成验证后即可继续访问",
|
|
3250
|
+
"请在微信客户端打开链接",
|
|
3251
|
+
"访问过于频繁",
|
|
3252
|
+
)
|
|
3253
|
+
_VERIFICATION_MAX_CHARS = 1500
|
|
3254
|
+
|
|
3255
|
+
|
|
3256
|
+
def looks_like_verification_page(markdown: str, title: str, url: str) -> str:
|
|
3257
|
+
"""Return the marker that identifies a verification page, or ``""``.
|
|
3258
|
+
|
|
3259
|
+
WeChat is checked by host because its interstitial is served on the article
|
|
3260
|
+
URL itself; the generic English markers are safe anywhere as long as the
|
|
3261
|
+
page carries almost no content. The caller turns a match into
|
|
3262
|
+
``VerificationRequired`` so a record says what happened instead of handing
|
|
3263
|
+
back 65 characters of an error page as if it were the article.
|
|
3264
|
+
"""
|
|
3265
|
+
|
|
3266
|
+
text = f"{title}\n{markdown}".strip()
|
|
3267
|
+
if not text or len(text) > _VERIFICATION_MAX_CHARS:
|
|
3268
|
+
return ""
|
|
3269
|
+
lowered = text.lower()
|
|
3270
|
+
host = (urllib.parse.urlparse(url).hostname or "").lower()
|
|
3271
|
+
if host == "mp.weixin.qq.com" or host.endswith(".weixin.qq.com"):
|
|
3272
|
+
for marker in _WECHAT_VERIFICATION_PATTERNS:
|
|
3273
|
+
if marker.lower() in lowered:
|
|
3274
|
+
return marker
|
|
3275
|
+
for marker in _VERIFICATION_PATTERNS:
|
|
3276
|
+
if marker in lowered:
|
|
3277
|
+
return marker
|
|
3278
|
+
return ""
|
|
3279
|
+
|
|
3280
|
+
|
|
3281
|
+
def fetch_page(
|
|
3282
|
+
url: str,
|
|
3283
|
+
config: Optional[FetchConfig] = None,
|
|
3284
|
+
fetcher: Optional[HttpFetcher] = None,
|
|
3285
|
+
query: str = "",
|
|
3286
|
+
allow_render_fallback: bool = True,
|
|
3287
|
+
) -> dict[str, Any]:
|
|
3288
|
+
"""Fetch one URL and return a JSON-serialisable record."""
|
|
3289
|
+
|
|
3290
|
+
config = config or FetchConfig()
|
|
3291
|
+
config.validate()
|
|
3292
|
+
url = normalize_url(url)
|
|
3293
|
+
own_fetcher = fetcher is None
|
|
3294
|
+
fetcher = fetcher or HttpFetcher(config)
|
|
3295
|
+
warnings: list[str] = []
|
|
3296
|
+
|
|
3297
|
+
try:
|
|
3298
|
+
policy = fetcher.assert_allowed(url)
|
|
3299
|
+
robots_checked = bool(config.respect_robots and config.robots_policy != "ignore")
|
|
3300
|
+
robots_overridden = bool(robots_checked and not policy.allowed(url))
|
|
3301
|
+
if robots_overridden:
|
|
3302
|
+
warnings.append(
|
|
3303
|
+
"robots.txt disallows this path; read because robots policy is "
|
|
3304
|
+
"'warn'. This override is recorded as compliance.robots_overridden."
|
|
3305
|
+
)
|
|
3306
|
+
extra_delay = policy.crawl_delay if policy and policy.crawl_delay else 0.0
|
|
3307
|
+
|
|
3308
|
+
response: Optional[HttpResponse] = None
|
|
3309
|
+
rendered = False
|
|
3310
|
+
render_fallback = False
|
|
3311
|
+
html = ""
|
|
3312
|
+
|
|
3313
|
+
if config.render == "always":
|
|
3314
|
+
fetcher.limiter.wait(url, extra_delay)
|
|
3315
|
+
html = render_html(url, config)
|
|
3316
|
+
rendered = True
|
|
3317
|
+
else:
|
|
3318
|
+
response = fetcher.fetch(url)
|
|
3319
|
+
html = response.text
|
|
3320
|
+
if config.render == "auto" and looks_dynamic(html):
|
|
3321
|
+
try:
|
|
3322
|
+
rendered_html = render_html(url, config)
|
|
3323
|
+
if rendered_html and len(rendered_html) > len(html):
|
|
3324
|
+
html = rendered_html
|
|
3325
|
+
rendered = True
|
|
3326
|
+
log(f"rendered dynamically: {url}", config)
|
|
3327
|
+
except RenderUnavailable as exc:
|
|
3328
|
+
warnings.append(f"dynamic rendering skipped: {exc}")
|
|
3329
|
+
except Exception as exc: # pragma: no cover - defensive
|
|
3330
|
+
warnings.append(f"dynamic rendering failed: {exc}")
|
|
3331
|
+
|
|
3332
|
+
# Some sites answer a plain request with a 100-byte script shell
|
|
3333
|
+
# that only a browser resolves. It carries no SPA markers, so the
|
|
3334
|
+
# heuristic above misses it and extraction would report an empty
|
|
3335
|
+
# page. Render it once, within the caller's budget.
|
|
3336
|
+
if (
|
|
3337
|
+
allow_render_fallback
|
|
3338
|
+
and config.render == "auto"
|
|
3339
|
+
and not rendered
|
|
3340
|
+
and response is not None
|
|
3341
|
+
and "html" in response.content_type.lower()
|
|
3342
|
+
and looks_like_html_shell(html)
|
|
3343
|
+
):
|
|
3344
|
+
try:
|
|
3345
|
+
rendered_html = render_html(url, config)
|
|
3346
|
+
if rendered_html and len(rendered_html) > len(html):
|
|
3347
|
+
html = rendered_html
|
|
3348
|
+
rendered = True
|
|
3349
|
+
render_fallback = True
|
|
3350
|
+
warnings.append(
|
|
3351
|
+
"the server returned a minimal HTML shell; rendered it "
|
|
3352
|
+
"once with Playwright before extraction"
|
|
3353
|
+
)
|
|
3354
|
+
except RenderUnavailable as exc:
|
|
3355
|
+
warnings.append(f"shell rendering skipped: {exc}")
|
|
3356
|
+
except Exception as exc: # pragma: no cover - defensive
|
|
3357
|
+
warnings.append(f"shell rendering failed: {exc}")
|
|
3358
|
+
|
|
3359
|
+
content_type = response.content_type if response else "text/html"
|
|
3360
|
+
final_url = response.final_url if response else url
|
|
3361
|
+
status = response.status if response else 200
|
|
3362
|
+
|
|
3363
|
+
from . import documents, feeds, pdf as pdf_mod
|
|
3364
|
+
|
|
3365
|
+
is_pdf_payload = response is not None and (
|
|
3366
|
+
content_type in _PDF_CONTENT_TYPES or pdf_mod.is_pdf(response.content)
|
|
3367
|
+
)
|
|
3368
|
+
document_kind = ""
|
|
3369
|
+
feed_format = ""
|
|
3370
|
+
|
|
3371
|
+
if is_pdf_payload and response is not None:
|
|
3372
|
+
try:
|
|
3373
|
+
pdf_document = pdf_mod.extract(response.content)
|
|
3374
|
+
except pdf_mod.PdfUnavailable as exc:
|
|
3375
|
+
raise PdfUnavailable(str(exc)) from exc
|
|
3376
|
+
except pdf_mod.PdfError as exc:
|
|
3377
|
+
raise PdfExtractionError(str(exc)) from exc
|
|
3378
|
+
markdown = pdf_mod.to_markdown(pdf_document, url=final_url)
|
|
3379
|
+
metadata = pdf_mod.to_metadata(pdf_document, url=final_url)
|
|
3380
|
+
tables = []
|
|
3381
|
+
warnings.append(
|
|
3382
|
+
f"PDF text extracted: {pdf_document.get('page_count', 0)} page(s), "
|
|
3383
|
+
f"{pdf_document.get('empty_pages', 0)} empty"
|
|
3384
|
+
)
|
|
3385
|
+
elif response is not None and (
|
|
3386
|
+
document_kind := documents.detect(response.content, content_type, final_url)
|
|
3387
|
+
):
|
|
3388
|
+
document = documents.extract(
|
|
3389
|
+
response.content, document_kind, content_type, final_url
|
|
3390
|
+
)
|
|
3391
|
+
markdown = document["markdown"]
|
|
3392
|
+
metadata = documents.to_metadata(document, final_url)
|
|
3393
|
+
tables = document.get("tables") or []
|
|
3394
|
+
warnings.append(f"{document_kind.upper()} extracted")
|
|
3395
|
+
elif response is not None and (
|
|
3396
|
+
content_type in _FEED_CONTENT_TYPES
|
|
3397
|
+
or (
|
|
3398
|
+
("<rss" in html[:1000].lower() or "<feed" in html[:1000].lower())
|
|
3399
|
+
and feeds.is_feed(html)
|
|
3400
|
+
)
|
|
3401
|
+
):
|
|
3402
|
+
feed = feeds.extract(html, limit=max(1, config.num_results))
|
|
3403
|
+
feed_format = feed["format"]
|
|
3404
|
+
markdown = feed["markdown"]
|
|
3405
|
+
metadata = feeds.to_metadata(feed, final_url)
|
|
3406
|
+
tables = []
|
|
3407
|
+
warnings.append(f"feed parsed: {feed['format']} with {feed['count']} item(s)")
|
|
3408
|
+
elif any(ct in content_type for ct in _JSON_CONTENT_TYPES) and response is not None:
|
|
3409
|
+
markdown = "```json\n" + response.text.strip() + "\n```"
|
|
3410
|
+
metadata = {
|
|
3411
|
+
"title": "",
|
|
3412
|
+
"authors": [],
|
|
3413
|
+
"published": "",
|
|
3414
|
+
"modified": "",
|
|
3415
|
+
"journal": "",
|
|
3416
|
+
"volume": "",
|
|
3417
|
+
"issue": "",
|
|
3418
|
+
"first_page": "",
|
|
3419
|
+
"last_page": "",
|
|
3420
|
+
"doi": "",
|
|
3421
|
+
"arxiv_id": "",
|
|
3422
|
+
"abstract": "",
|
|
3423
|
+
"keywords": [],
|
|
3424
|
+
"language": "",
|
|
3425
|
+
"site_name": "",
|
|
3426
|
+
"canonical_url": "",
|
|
3427
|
+
"og_type": "",
|
|
3428
|
+
}
|
|
3429
|
+
tables: list[dict[str, Any]] = []
|
|
3430
|
+
else:
|
|
3431
|
+
markdown = html_to_markdown(
|
|
3432
|
+
html,
|
|
3433
|
+
url=final_url,
|
|
3434
|
+
include_tables=config.include_tables,
|
|
3435
|
+
keep_links=config.keep_links,
|
|
3436
|
+
include_images=config.include_images,
|
|
3437
|
+
)
|
|
3438
|
+
metadata = extract_metadata(html, url=final_url)
|
|
3439
|
+
tables = extract_tables(html) if config.include_tables else []
|
|
3440
|
+
|
|
3441
|
+
marker = looks_like_verification_page(
|
|
3442
|
+
markdown, str(metadata.get("title", "")), final_url
|
|
3443
|
+
)
|
|
3444
|
+
if marker:
|
|
3445
|
+
hint = ""
|
|
3446
|
+
host = (urllib.parse.urlparse(final_url).hostname or "").lower()
|
|
3447
|
+
if host == "mp.weixin.qq.com" or host.endswith(".weixin.qq.com"):
|
|
3448
|
+
hint = (
|
|
3449
|
+
" WeChat serves its article body only to a browser session that "
|
|
3450
|
+
"has already passed verification: try the wechat-dig browser "
|
|
3451
|
+
"playbook with --backend cdp and a profile you have opened once."
|
|
3452
|
+
)
|
|
3453
|
+
raise VerificationRequired(
|
|
3454
|
+
f"{final_url} returned a verification/anti-bot page ({marker}); "
|
|
3455
|
+
f"HTTP mode does not solve challenges.{hint}"
|
|
3456
|
+
)
|
|
3457
|
+
|
|
3458
|
+
if not markdown.strip():
|
|
3459
|
+
raise ExtractionError(
|
|
3460
|
+
f"no readable content extracted from {final_url}; "
|
|
3461
|
+
"the page may be empty, paywalled, or require JavaScript rendering"
|
|
3462
|
+
)
|
|
3463
|
+
|
|
3464
|
+
if config.max_chars and len(markdown) > config.max_chars:
|
|
3465
|
+
markdown = markdown[: config.max_chars].rstrip() + "\n\n<!-- truncated by web-forager -->"
|
|
3466
|
+
warnings.append(f"markdown truncated to {config.max_chars} characters")
|
|
3467
|
+
|
|
3468
|
+
# Everything below came from a third party. Say so in the record, and
|
|
3469
|
+
# name instruction-shaped passages instead of letting them read as
|
|
3470
|
+
# part of the surrounding conversation.
|
|
3471
|
+
injection_flags: list[str] = []
|
|
3472
|
+
trust = "untrusted"
|
|
3473
|
+
if config.input_gate != "off":
|
|
3474
|
+
injection_flags = untrusted.scan(
|
|
3475
|
+
markdown + "\n" + str(metadata.get("title", ""))
|
|
3476
|
+
)
|
|
3477
|
+
if injection_flags:
|
|
3478
|
+
if config.input_gate == "strict":
|
|
3479
|
+
markdown = ""
|
|
3480
|
+
trust = "rejected"
|
|
3481
|
+
warnings.insert(0, untrusted.withheld(injection_flags))
|
|
3482
|
+
else:
|
|
3483
|
+
warnings.insert(0, untrusted.warning(injection_flags))
|
|
3484
|
+
|
|
3485
|
+
links = []
|
|
3486
|
+
if config.extract_links and not is_pdf_payload and not document_kind and not feed_format:
|
|
3487
|
+
links = extract_links(
|
|
3488
|
+
html,
|
|
3489
|
+
final_url,
|
|
3490
|
+
query=query,
|
|
3491
|
+
limit=config.max_links,
|
|
3492
|
+
same_site=config.follow == "same-site",
|
|
3493
|
+
)
|
|
3494
|
+
|
|
3495
|
+
chunks = chunk_markdown(markdown, config.chunk_size) if config.chunk_size else []
|
|
3496
|
+
|
|
3497
|
+
return {
|
|
3498
|
+
"url": url,
|
|
3499
|
+
"final_url": final_url,
|
|
3500
|
+
"status": status,
|
|
3501
|
+
"content_type": content_type,
|
|
3502
|
+
"document_kind": document_kind,
|
|
3503
|
+
"feed_format": feed_format,
|
|
3504
|
+
"cached": bool(response.from_cache) if response else False,
|
|
3505
|
+
"rendered": rendered,
|
|
3506
|
+
"render_fallback": render_fallback,
|
|
3507
|
+
"fetched_at": utc_now(),
|
|
3508
|
+
"title": metadata.get("title", ""),
|
|
3509
|
+
"metadata": metadata,
|
|
3510
|
+
"markdown": markdown,
|
|
3511
|
+
"chunks": chunks,
|
|
3512
|
+
"tables": tables,
|
|
3513
|
+
"links": links,
|
|
3514
|
+
"warnings": warnings,
|
|
3515
|
+
"trust": trust,
|
|
3516
|
+
"injection_flags": injection_flags,
|
|
3517
|
+
"compliance": {
|
|
3518
|
+
"robots_checked": robots_checked,
|
|
3519
|
+
"robots_policy": (
|
|
3520
|
+
config.robots_policy if config.respect_robots else "ignore"
|
|
3521
|
+
),
|
|
3522
|
+
"robots_overridden": robots_overridden,
|
|
3523
|
+
},
|
|
3524
|
+
}
|
|
3525
|
+
finally:
|
|
3526
|
+
if own_fetcher:
|
|
3527
|
+
fetcher.close()
|
|
3528
|
+
|
|
3529
|
+
|
|
3530
|
+
def search_warnings(
|
|
3531
|
+
query: str, results: list[dict[str, Any]], engine: str = "auto"
|
|
3532
|
+
) -> list[str]:
|
|
3533
|
+
"""Warnings for a search that returned nothing or nothing relevant.
|
|
3534
|
+
|
|
3535
|
+
Both the CLI and the MCP server attach these, so an agent that receives a
|
|
3536
|
+
list of dictionary pages is told they are unrelated instead of reporting
|
|
3537
|
+
them as findings.
|
|
3538
|
+
"""
|
|
3539
|
+
|
|
3540
|
+
warnings: list[str] = []
|
|
3541
|
+
tokens = _query_tokens(query)
|
|
3542
|
+
if not results:
|
|
3543
|
+
warnings.append(
|
|
3544
|
+
f"no results: {engine} returned nothing for {query!r}. Try another "
|
|
3545
|
+
"engine (--engine all), a shorter query, or a direct URL."
|
|
3546
|
+
)
|
|
3547
|
+
elif tokens and max(result_coverage(item, tokens) for item in results) <= 0:
|
|
3548
|
+
warnings.append(
|
|
3549
|
+
"none of the returned results mention the query text; the engine "
|
|
3550
|
+
"most likely split the query into single characters (common with "
|
|
3551
|
+
"quoted or CJK names). Treat these results as unrelated and try "
|
|
3552
|
+
"another engine or a more specific query."
|
|
3553
|
+
)
|
|
3554
|
+
if results:
|
|
3555
|
+
probe = " ".join(
|
|
3556
|
+
f"{item.get('title', '')} {item.get('snippet', '')}" for item in results[:8]
|
|
3557
|
+
)
|
|
3558
|
+
flags = untrusted.scan(probe)
|
|
3559
|
+
if flags:
|
|
3560
|
+
warnings.append(untrusted.warning(flags))
|
|
3561
|
+
if looks_like_person_name(query):
|
|
3562
|
+
warnings.append(
|
|
3563
|
+
"the query is a bare personal name. Searching a paper author or a "
|
|
3564
|
+
"public figure is fine; do not build or publish a profile of a "
|
|
3565
|
+
"private individual — that is out of scope for this tool."
|
|
3566
|
+
)
|
|
3567
|
+
return warnings
|
|
3568
|
+
|
|
3569
|
+
|
|
3570
|
+
SITE_SEARCH_TEMPLATES = (
|
|
3571
|
+
"/?s={query}",
|
|
3572
|
+
"/search?q={query}",
|
|
3573
|
+
"/?search={query}",
|
|
3574
|
+
"/search?query={query}",
|
|
3575
|
+
)
|
|
3576
|
+
|
|
3577
|
+
|
|
3578
|
+
def find_within_site(
|
|
3579
|
+
site: str,
|
|
3580
|
+
query: str,
|
|
3581
|
+
config: Optional[FetchConfig] = None,
|
|
3582
|
+
fetcher: Optional[HttpFetcher] = None,
|
|
3583
|
+
max_pages: Optional[int] = None,
|
|
3584
|
+
) -> dict[str, Any]:
|
|
3585
|
+
"""Search inside one site through the site's own search page.
|
|
3586
|
+
|
|
3587
|
+
The general engines are unreliable in some networks while a WordPress-style
|
|
3588
|
+
``/?s=`` page answers directly. This reads that page, ranks its links by
|
|
3589
|
+
query coverage, and fetches the best ones — the same route a person takes
|
|
3590
|
+
when they already know where the content lives.
|
|
3591
|
+
"""
|
|
3592
|
+
|
|
3593
|
+
config = config or FetchConfig()
|
|
3594
|
+
config.validate()
|
|
3595
|
+
site = (site or "").strip()
|
|
3596
|
+
query = (query or "").strip()
|
|
3597
|
+
if not site:
|
|
3598
|
+
raise ValueError("site must not be empty")
|
|
3599
|
+
if not query:
|
|
3600
|
+
raise ValueError("query must not be empty")
|
|
3601
|
+
reason = doxxing_reason(query)
|
|
3602
|
+
if reason:
|
|
3603
|
+
raise ProhibitedQuery(
|
|
3604
|
+
f"refused: {reason}. web-forager does not collect or assemble "
|
|
3605
|
+
"profiles of identifiable people; search for a topic, a paper "
|
|
3606
|
+
"title or an organisation instead."
|
|
3607
|
+
)
|
|
3608
|
+
site_url = normalize_url(site if "://" in site else "https://" + site)
|
|
3609
|
+
pages_limit = config.max_pages if max_pages is None else max(0, max_pages)
|
|
3610
|
+
own_fetcher = fetcher is None
|
|
3611
|
+
fetcher = fetcher or HttpFetcher(config)
|
|
3612
|
+
|
|
3613
|
+
attempts: list[dict[str, Any]] = []
|
|
3614
|
+
page: Optional[HttpResponse] = None
|
|
3615
|
+
search_url = ""
|
|
3616
|
+
try:
|
|
3617
|
+
for template in SITE_SEARCH_TEMPLATES:
|
|
3618
|
+
candidate = site_url.rstrip("/") + template.format(
|
|
3619
|
+
query=urllib.parse.quote_plus(query)
|
|
3620
|
+
)
|
|
3621
|
+
started = time.monotonic()
|
|
3622
|
+
try:
|
|
3623
|
+
response = fetcher.fetch(candidate)
|
|
3624
|
+
except (FetchError, RobotsDenied) as exc:
|
|
3625
|
+
attempts.append(
|
|
3626
|
+
{
|
|
3627
|
+
"url": candidate,
|
|
3628
|
+
"ok": False,
|
|
3629
|
+
"error": f"{exc.__class__.__name__}: {exc}",
|
|
3630
|
+
"ms": round((time.monotonic() - started) * 1000),
|
|
3631
|
+
}
|
|
3632
|
+
)
|
|
3633
|
+
continue
|
|
3634
|
+
attempts.append(
|
|
3635
|
+
{
|
|
3636
|
+
"url": candidate,
|
|
3637
|
+
"ok": True,
|
|
3638
|
+
"status": response.status,
|
|
3639
|
+
"ms": round((time.monotonic() - started) * 1000),
|
|
3640
|
+
}
|
|
3641
|
+
)
|
|
3642
|
+
if response.status == 200 and response.text.strip():
|
|
3643
|
+
page = response
|
|
3644
|
+
search_url = response.final_url or candidate
|
|
3645
|
+
break
|
|
3646
|
+
if page is None:
|
|
3647
|
+
detail = "; ".join(
|
|
3648
|
+
f"{item['url']}: {item.get('error', 'empty response')}" for item in attempts
|
|
3649
|
+
)
|
|
3650
|
+
raise FetchError(f"no readable search page on {site_url} ({detail})")
|
|
3651
|
+
|
|
3652
|
+
warnings: list[str] = []
|
|
3653
|
+
candidates = extract_links(
|
|
3654
|
+
page.text, search_url, query=query, limit=config.max_links, same_site=True
|
|
3655
|
+
)
|
|
3656
|
+
# The search page links back to itself ("skip to content", the form's
|
|
3657
|
+
# action); those are navigation, not results.
|
|
3658
|
+
search_stem = search_url.split("?")[0].rstrip("/")
|
|
3659
|
+
candidates = [
|
|
3660
|
+
item
|
|
3661
|
+
for item in candidates
|
|
3662
|
+
if item.get("url", "").split("?")[0].rstrip("/") != search_stem
|
|
3663
|
+
]
|
|
3664
|
+
matched = [item for item in candidates if item.get("score", 0) > 0]
|
|
3665
|
+
if matched:
|
|
3666
|
+
candidates = matched
|
|
3667
|
+
elif candidates:
|
|
3668
|
+
warnings.append(
|
|
3669
|
+
"the site's search page carried no link whose text mentions the "
|
|
3670
|
+
"query; its ranked links are returned unfiltered"
|
|
3671
|
+
)
|
|
3672
|
+
else:
|
|
3673
|
+
warnings.append("the site's search page carried no followable links")
|
|
3674
|
+
|
|
3675
|
+
pages: list[dict[str, Any]] = []
|
|
3676
|
+
for item in candidates[:pages_limit]:
|
|
3677
|
+
try:
|
|
3678
|
+
pages.append(fetch_page(item["url"], config, fetcher=fetcher))
|
|
3679
|
+
except (RobotsDenied, FetchError, ExtractionError, RenderUnavailable) as exc:
|
|
3680
|
+
pages.append(
|
|
3681
|
+
{
|
|
3682
|
+
"url": item["url"],
|
|
3683
|
+
"error": f"{exc.__class__.__name__}: {exc}",
|
|
3684
|
+
"warnings": [],
|
|
3685
|
+
}
|
|
3686
|
+
)
|
|
3687
|
+
|
|
3688
|
+
payload: dict[str, Any] = {
|
|
3689
|
+
"mode": "site-search",
|
|
3690
|
+
"site": site_url,
|
|
3691
|
+
"query": query,
|
|
3692
|
+
"search_url": search_url,
|
|
3693
|
+
"candidates": candidates,
|
|
3694
|
+
"pages": pages,
|
|
3695
|
+
"attempts": attempts,
|
|
3696
|
+
}
|
|
3697
|
+
if warnings:
|
|
3698
|
+
payload["warnings"] = warnings
|
|
3699
|
+
return payload
|
|
3700
|
+
finally:
|
|
3701
|
+
if own_fetcher:
|
|
3702
|
+
fetcher.close()
|
|
3703
|
+
|
|
3704
|
+
|
|
3705
|
+
_FEED_LINK_TYPES = ("rss", "atom", "feed")
|
|
3706
|
+
|
|
3707
|
+
|
|
3708
|
+
def discover_feed(
|
|
3709
|
+
url: str,
|
|
3710
|
+
config: Optional[FetchConfig] = None,
|
|
3711
|
+
fetcher: Optional[HttpFetcher] = None,
|
|
3712
|
+
) -> dict[str, Any]:
|
|
3713
|
+
"""Read a feed, discovering it from a page's ``<link rel=alternate>`` first.
|
|
3714
|
+
|
|
3715
|
+
A site's own feed needs no search engine and no rendering: it is the
|
|
3716
|
+
cheapest way to follow a publication in a network where the engines fail.
|
|
3717
|
+
"""
|
|
3718
|
+
|
|
3719
|
+
from . import feeds
|
|
3720
|
+
|
|
3721
|
+
config = config or FetchConfig()
|
|
3722
|
+
config.validate()
|
|
3723
|
+
url = (url or "").strip()
|
|
3724
|
+
if not url:
|
|
3725
|
+
raise ValueError("url must not be empty")
|
|
3726
|
+
own_fetcher = fetcher is None
|
|
3727
|
+
fetcher = fetcher or HttpFetcher(config)
|
|
3728
|
+
|
|
3729
|
+
try:
|
|
3730
|
+
response = fetcher.fetch(normalize_url(url))
|
|
3731
|
+
if feeds.is_feed(response.text) or response.content_type in _FEED_CONTENT_TYPES:
|
|
3732
|
+
feed_url = response.final_url
|
|
3733
|
+
feed = feeds.extract(response.text, limit=max(1, config.num_results))
|
|
3734
|
+
else:
|
|
3735
|
+
soup = make_soup(response.text)
|
|
3736
|
+
href = ""
|
|
3737
|
+
for link in soup.find_all("link"):
|
|
3738
|
+
rel = " ".join(str(value) for value in (link.get("rel") or [])).lower()
|
|
3739
|
+
kind = str(link.get("type") or "").lower()
|
|
3740
|
+
if "alternate" in rel and any(word in kind for word in _FEED_LINK_TYPES):
|
|
3741
|
+
href = absolute_url(_attr_text(link, "href"), response.final_url)
|
|
3742
|
+
if href:
|
|
3743
|
+
break
|
|
3744
|
+
if not href:
|
|
3745
|
+
raise ExtractionError(
|
|
3746
|
+
f"no RSS/Atom <link rel=alternate> found at {response.final_url}"
|
|
3747
|
+
)
|
|
3748
|
+
feed_response = fetcher.fetch(href)
|
|
3749
|
+
if not feeds.is_feed(feed_response.text):
|
|
3750
|
+
raise ExtractionError(
|
|
3751
|
+
f"{feed_response.final_url} is not an RSS/Atom feed"
|
|
3752
|
+
)
|
|
3753
|
+
feed = feeds.extract(feed_response.text, limit=max(1, config.num_results))
|
|
3754
|
+
feed_url = feed_response.final_url
|
|
3755
|
+
return {
|
|
3756
|
+
"mode": "feed",
|
|
3757
|
+
"feed_url": feed_url,
|
|
3758
|
+
"format": feed["format"],
|
|
3759
|
+
"count": feed["count"],
|
|
3760
|
+
"items": feed.get("items") or [],
|
|
3761
|
+
"markdown": feed["markdown"],
|
|
3762
|
+
"metadata": feeds.to_metadata(feed, feed_url),
|
|
3763
|
+
}
|
|
3764
|
+
finally:
|
|
3765
|
+
if own_fetcher:
|
|
3766
|
+
fetcher.close()
|
|
3767
|
+
|
|
3768
|
+
|
|
3769
|
+
def probe_engines(
|
|
3770
|
+
config: Optional[FetchConfig] = None,
|
|
3771
|
+
fetcher: Optional[HttpFetcher] = None,
|
|
3772
|
+
query: str = "web forager",
|
|
3773
|
+
) -> list[dict[str, Any]]:
|
|
3774
|
+
"""Ask every search engine once and report which ones answer right now.
|
|
3775
|
+
|
|
3776
|
+
A network can block one engine and allow another; the report turns "search
|
|
3777
|
+
is broken" into "Bing answers, Sogou shows a verification page".
|
|
3778
|
+
"""
|
|
3779
|
+
|
|
3780
|
+
config = config or FetchConfig()
|
|
3781
|
+
config.validate()
|
|
3782
|
+
# A probe is a diagnostic, not a retry exercise: one attempt per engine
|
|
3783
|
+
# keeps a blocking network from turning the report into a 60-second wait.
|
|
3784
|
+
probe_config = dataclasses.replace(config, retries=0)
|
|
3785
|
+
own_fetcher = fetcher is None
|
|
3786
|
+
fetcher = fetcher or HttpFetcher(probe_config)
|
|
3787
|
+
report: list[dict[str, Any]] = []
|
|
3788
|
+
try:
|
|
3789
|
+
for name in ("360", "bing", "sogou", "weixin", "duckduckgo"):
|
|
3790
|
+
started = time.monotonic()
|
|
3791
|
+
try:
|
|
3792
|
+
items = _search_one(name, query, probe_config, 1, fetcher, pages=1)
|
|
3793
|
+
entry: dict[str, Any] = {
|
|
3794
|
+
"engine": name,
|
|
3795
|
+
"ok": bool(items),
|
|
3796
|
+
"results": len(items),
|
|
3797
|
+
"ms": round((time.monotonic() - started) * 1000),
|
|
3798
|
+
}
|
|
3799
|
+
if not items:
|
|
3800
|
+
entry["error"] = "returned no results (empty page or a challenge page)"
|
|
3801
|
+
report.append(entry)
|
|
3802
|
+
except Exception as exc:
|
|
3803
|
+
report.append(
|
|
3804
|
+
{
|
|
3805
|
+
"engine": name,
|
|
3806
|
+
"ok": False,
|
|
3807
|
+
"results": 0,
|
|
3808
|
+
"error": str(exc),
|
|
3809
|
+
"ms": round((time.monotonic() - started) * 1000),
|
|
3810
|
+
}
|
|
3811
|
+
)
|
|
3812
|
+
finally:
|
|
3813
|
+
if own_fetcher:
|
|
3814
|
+
fetcher.close()
|
|
3815
|
+
return report
|
|
3816
|
+
|
|
3817
|
+
|
|
3818
|
+
def fetch_by_query(
|
|
3819
|
+
query: str,
|
|
3820
|
+
config: Optional[FetchConfig] = None,
|
|
3821
|
+
engine: str = "auto",
|
|
3822
|
+
fetcher: Optional[HttpFetcher] = None,
|
|
3823
|
+
) -> dict[str, Any]:
|
|
3824
|
+
"""Search for a keyword query, then fetch and clean the top results."""
|
|
3825
|
+
|
|
3826
|
+
config = config or FetchConfig()
|
|
3827
|
+
config.validate()
|
|
3828
|
+
own_fetcher = fetcher is None
|
|
3829
|
+
fetcher = fetcher or HttpFetcher(config)
|
|
3830
|
+
|
|
3831
|
+
try:
|
|
3832
|
+
engine_report: list[dict[str, Any]] = []
|
|
3833
|
+
results = search(
|
|
3834
|
+
query,
|
|
3835
|
+
config,
|
|
3836
|
+
engine=engine,
|
|
3837
|
+
limit=config.num_results,
|
|
3838
|
+
report=engine_report,
|
|
3839
|
+
)
|
|
3840
|
+
pages: list[dict[str, Any]] = []
|
|
3841
|
+
skipped: list[dict[str, str]] = []
|
|
3842
|
+
attempts = 0
|
|
3843
|
+
render_fallbacks = 0
|
|
3844
|
+
# ``max_pages`` means readable pages, not attempts. Keep scanning past
|
|
3845
|
+
# robots-denied or unreadable results, with a cap so one query cannot
|
|
3846
|
+
# turn into an unbounded site walk.
|
|
3847
|
+
max_attempts = min(len(results), max(config.max_pages * 3, config.max_pages + 5))
|
|
3848
|
+
for result in results:
|
|
3849
|
+
if len(pages) >= config.max_pages or attempts >= max_attempts:
|
|
3850
|
+
break
|
|
3851
|
+
attempts += 1
|
|
3852
|
+
try:
|
|
3853
|
+
page = fetch_page(
|
|
3854
|
+
result["url"],
|
|
3855
|
+
config,
|
|
3856
|
+
fetcher=fetcher,
|
|
3857
|
+
allow_render_fallback=(
|
|
3858
|
+
config.render == "auto" and render_fallbacks < 2
|
|
3859
|
+
),
|
|
3860
|
+
)
|
|
3861
|
+
if page.get("render_fallback"):
|
|
3862
|
+
render_fallbacks += 1
|
|
3863
|
+
pages.append(page)
|
|
3864
|
+
except RobotsDenied as exc:
|
|
3865
|
+
skipped.append(
|
|
3866
|
+
{"url": result["url"], "code": "robots_denied", "reason": str(exc)}
|
|
3867
|
+
)
|
|
3868
|
+
except VerificationRequired as exc:
|
|
3869
|
+
skipped.append(
|
|
3870
|
+
{
|
|
3871
|
+
"url": result["url"],
|
|
3872
|
+
"code": "verification_required",
|
|
3873
|
+
"reason": str(exc),
|
|
3874
|
+
}
|
|
3875
|
+
)
|
|
3876
|
+
except FetchError as exc:
|
|
3877
|
+
skipped.append(
|
|
3878
|
+
{"url": result["url"], "code": "fetch_error", "reason": str(exc)}
|
|
3879
|
+
)
|
|
3880
|
+
except ExtractionError as exc:
|
|
3881
|
+
skipped.append(
|
|
3882
|
+
{"url": result["url"], "code": "extraction_error", "reason": str(exc)}
|
|
3883
|
+
)
|
|
3884
|
+
except RenderUnavailable as exc:
|
|
3885
|
+
skipped.append(
|
|
3886
|
+
{"url": result["url"], "code": "render_unavailable", "reason": str(exc)}
|
|
3887
|
+
)
|
|
3888
|
+
except Exception as exc: # pragma: no cover - defensive
|
|
3889
|
+
skipped.append(
|
|
3890
|
+
{"url": result["url"], "code": "unexpected", "reason": str(exc)}
|
|
3891
|
+
)
|
|
3892
|
+
payload: dict[str, Any] = {
|
|
3893
|
+
"query": query,
|
|
3894
|
+
"engine": engine,
|
|
3895
|
+
"mode": "search",
|
|
3896
|
+
"search_results": results,
|
|
3897
|
+
"engine_report": engine_report,
|
|
3898
|
+
"pages": pages,
|
|
3899
|
+
"skipped": skipped,
|
|
3900
|
+
"pages_attempted": attempts,
|
|
3901
|
+
}
|
|
3902
|
+
warnings = search_warnings(query, results, engine=engine)
|
|
3903
|
+
if skipped:
|
|
3904
|
+
counts: dict[str, int] = {}
|
|
3905
|
+
for item in skipped:
|
|
3906
|
+
counts[item["code"]] = counts.get(item["code"], 0) + 1
|
|
3907
|
+
summary = ", ".join(f"{code} ({count})" for code, count in sorted(counts.items()))
|
|
3908
|
+
warnings.append(
|
|
3909
|
+
f"{len(skipped)} result(s) skipped while filling {config.max_pages} "
|
|
3910
|
+
f"readable page(s): {summary}"
|
|
3911
|
+
)
|
|
3912
|
+
if counts.get("robots_denied"):
|
|
3913
|
+
warnings.append(
|
|
3914
|
+
"robots.txt denied the skipped paths. Use --robots-policy warn only "
|
|
3915
|
+
"when you are authorised to read them; the record will mark the "
|
|
3916
|
+
"override. Do not silently switch to a tool that ignores robots."
|
|
3917
|
+
)
|
|
3918
|
+
if counts.get("verification_required"):
|
|
3919
|
+
warnings.append(
|
|
3920
|
+
"verification_required means the site answered with an anti-bot or "
|
|
3921
|
+
"verification page instead of the article. For a WeChat link, read it "
|
|
3922
|
+
"in browser mode: browser_run with playbook \"wechat-dig\" and that URL "
|
|
3923
|
+
"(CLI: node browser/run.js --playbook wechat-dig --url <URL> --backend cdp) "
|
|
3924
|
+
"after signing in once with browser/launch_chrome.py. Do not try to solve "
|
|
3925
|
+
"the challenge."
|
|
3926
|
+
)
|
|
3927
|
+
wechat_denied = [
|
|
3928
|
+
item
|
|
3929
|
+
for item in skipped
|
|
3930
|
+
if item["code"] == "robots_denied"
|
|
3931
|
+
and (
|
|
3932
|
+
urllib.parse.urlparse(item["url"]).hostname or ""
|
|
3933
|
+
).lower()
|
|
3934
|
+
== "mp.weixin.qq.com"
|
|
3935
|
+
]
|
|
3936
|
+
if wechat_denied:
|
|
3937
|
+
warnings.append(
|
|
3938
|
+
"WeChat article links cannot be read in HTTP mode: robots.txt disallows "
|
|
3939
|
+
"/s, and the page itself returns an environment check. Open one with "
|
|
3940
|
+
"browser mode instead: browser_run with playbook \"wechat-dig\" and that "
|
|
3941
|
+
"URL (CLI: node browser/run.js --playbook wechat-dig --url <URL> --backend cdp). "
|
|
3942
|
+
"Do not try to solve the challenge."
|
|
3943
|
+
)
|
|
3944
|
+
if not pages:
|
|
3945
|
+
warnings.append(
|
|
3946
|
+
"No readable page was left. Try --engine bing for a Latin query, "
|
|
3947
|
+
"--action find --site <host> for a site's own search, or "
|
|
3948
|
+
"--action feed --url <site> for its RSS/Atom feed."
|
|
3949
|
+
)
|
|
3950
|
+
if warnings:
|
|
3951
|
+
payload["warnings"] = warnings
|
|
3952
|
+
return payload
|
|
3953
|
+
finally:
|
|
3954
|
+
if own_fetcher:
|
|
3955
|
+
fetcher.close()
|
|
3956
|
+
|
|
3957
|
+
|
|
3958
|
+
def summarize_record(record: dict[str, Any]) -> dict[str, Any]:
|
|
3959
|
+
"""Return a compact index entry for one fetched page.
|
|
3960
|
+
|
|
3961
|
+
``results`` used to repeat every page in full, so a search response carried
|
|
3962
|
+
each Markdown body twice and callers with a response cap lost the tail.
|
|
3963
|
+
The body lives in ``pages`` once; this is the entry an agent loops over.
|
|
3964
|
+
"""
|
|
3965
|
+
|
|
3966
|
+
if not isinstance(record, dict):
|
|
3967
|
+
return {"value": record}
|
|
3968
|
+
summary: dict[str, Any] = {
|
|
3969
|
+
"url": record.get("url") or record.get("final_url") or "",
|
|
3970
|
+
"title": record.get("title", ""),
|
|
3971
|
+
"status": record.get("status"),
|
|
3972
|
+
"content_type": record.get("content_type", ""),
|
|
3973
|
+
"markdown_chars": len(record.get("markdown") or ""),
|
|
3974
|
+
"tables": len(record.get("tables") or []),
|
|
3975
|
+
}
|
|
3976
|
+
if record.get("error"):
|
|
3977
|
+
summary.pop("status", None)
|
|
3978
|
+
summary["error"] = record["error"]
|
|
3979
|
+
if record.get("warnings"):
|
|
3980
|
+
summary["warnings"] = record["warnings"]
|
|
3981
|
+
return summary
|
|
3982
|
+
|
|
3983
|
+
|
|
3984
|
+
def redact_url(value: str) -> str:
|
|
3985
|
+
"""Drop credentials from a URL before it is printed.
|
|
3986
|
+
|
|
3987
|
+
A proxy URL often carries ``user:password@``; a report must show the host
|
|
3988
|
+
and port without turning the password into plain text in a log.
|
|
3989
|
+
"""
|
|
3990
|
+
|
|
3991
|
+
text = str(value or "").strip()
|
|
3992
|
+
if not text:
|
|
3993
|
+
return ""
|
|
3994
|
+
if "://" not in text:
|
|
3995
|
+
text = "http://" + text
|
|
3996
|
+
try:
|
|
3997
|
+
parts = urllib.parse.urlsplit(text)
|
|
3998
|
+
except ValueError:
|
|
3999
|
+
return ""
|
|
4000
|
+
host = parts.hostname or ""
|
|
4001
|
+
port = f":{parts.port}" if parts.port else ""
|
|
4002
|
+
scheme = parts.scheme or "http"
|
|
4003
|
+
return f"{scheme}://{host}{port}" if host else ""
|
|
4004
|
+
|
|
4005
|
+
|
|
4006
|
+
def doctor_report(config: Optional[FetchConfig] = None) -> dict[str, Any]:
|
|
4007
|
+
"""Report which capabilities this interpreter can serve, with install hints.
|
|
4008
|
+
|
|
4009
|
+
An agent that gets a rendered-less page can run this and install what is
|
|
4010
|
+
missing instead of guessing why the output was thin.
|
|
4011
|
+
"""
|
|
4012
|
+
|
|
4013
|
+
import importlib.metadata
|
|
4014
|
+
import importlib.util
|
|
4015
|
+
import platform
|
|
4016
|
+
import shutil
|
|
4017
|
+
|
|
4018
|
+
def version_of(distribution: str) -> str:
|
|
4019
|
+
try:
|
|
4020
|
+
return importlib.metadata.version(distribution)
|
|
4021
|
+
except Exception:
|
|
4022
|
+
return ""
|
|
4023
|
+
|
|
4024
|
+
def check(name: str, module: str, distribution: str, hint: str, required: bool) -> dict[str, Any]:
|
|
4025
|
+
entry: dict[str, Any] = {"name": name, "required": required}
|
|
4026
|
+
try:
|
|
4027
|
+
available = importlib.util.find_spec(module) is not None
|
|
4028
|
+
except (ImportError, ValueError):
|
|
4029
|
+
available = False
|
|
4030
|
+
entry["ok"] = available
|
|
4031
|
+
if available:
|
|
4032
|
+
entry["version"] = version_of(distribution)
|
|
4033
|
+
elif hint:
|
|
4034
|
+
entry["hint"] = hint
|
|
4035
|
+
return entry
|
|
4036
|
+
|
|
4037
|
+
checks = [
|
|
4038
|
+
check("requests", "requests", "requests", "pip install requests", True),
|
|
4039
|
+
check("beautifulsoup4", "bs4", "beautifulsoup4", "pip install beautifulsoup4", True),
|
|
4040
|
+
check("trafilatura", "trafilatura", "trafilatura", "pip install trafilatura", True),
|
|
4041
|
+
check(
|
|
4042
|
+
"playwright",
|
|
4043
|
+
"playwright",
|
|
4044
|
+
"playwright",
|
|
4045
|
+
"pip install playwright && playwright install chromium",
|
|
4046
|
+
False,
|
|
4047
|
+
),
|
|
4048
|
+
check("pypdf", "pypdf", "pypdf", "pip install pypdf", False),
|
|
4049
|
+
check("defusedxml", "defusedxml", "defusedxml", "pip install defusedxml", False),
|
|
4050
|
+
]
|
|
4051
|
+
node_path = shutil.which("node")
|
|
4052
|
+
if node_path:
|
|
4053
|
+
checks.append({"name": "node", "required": False, "ok": True, "path": node_path})
|
|
4054
|
+
else:
|
|
4055
|
+
checks.append(
|
|
4056
|
+
{
|
|
4057
|
+
"name": "node",
|
|
4058
|
+
"required": False,
|
|
4059
|
+
"ok": False,
|
|
4060
|
+
"hint": "install Node.js 18+ only if you use browser mode",
|
|
4061
|
+
}
|
|
4062
|
+
)
|
|
4063
|
+
|
|
4064
|
+
proxy_value = ""
|
|
4065
|
+
proxy_source = ""
|
|
4066
|
+
if config is not None and config.proxy:
|
|
4067
|
+
proxy_value = config.proxy
|
|
4068
|
+
proxy_source = "config"
|
|
4069
|
+
else:
|
|
4070
|
+
for key in (
|
|
4071
|
+
"HTTPS_PROXY",
|
|
4072
|
+
"https_proxy",
|
|
4073
|
+
"ALL_PROXY",
|
|
4074
|
+
"all_proxy",
|
|
4075
|
+
"HTTP_PROXY",
|
|
4076
|
+
"http_proxy",
|
|
4077
|
+
):
|
|
4078
|
+
value = os.environ.get(key, "").strip()
|
|
4079
|
+
if value:
|
|
4080
|
+
proxy_value = value
|
|
4081
|
+
proxy_source = f"env:{key}"
|
|
4082
|
+
break
|
|
4083
|
+
|
|
4084
|
+
missing_required = [item["name"] for item in checks if item["required"] and not item["ok"]]
|
|
4085
|
+
return {
|
|
4086
|
+
"ok": not missing_required,
|
|
4087
|
+
"python": platform.python_version(),
|
|
4088
|
+
"platform": platform.platform(),
|
|
4089
|
+
"version": __version__,
|
|
4090
|
+
"missing_required": missing_required,
|
|
4091
|
+
"checks": checks,
|
|
4092
|
+
"proxy": {
|
|
4093
|
+
"configured": bool(proxy_value),
|
|
4094
|
+
"source": proxy_source,
|
|
4095
|
+
"url": redact_url(proxy_value),
|
|
4096
|
+
},
|
|
4097
|
+
}
|
|
4098
|
+
|
|
4099
|
+
|
|
4100
|
+
def build_envelope(
|
|
4101
|
+
action: str, payload: dict[str, Any], full_results: bool = False
|
|
4102
|
+
) -> dict[str, Any]:
|
|
4103
|
+
"""Wrap a payload with tool metadata for stable downstream parsing.
|
|
4104
|
+
|
|
4105
|
+
``results`` is the flat list an agent loops over: fetched pages, videos or
|
|
4106
|
+
Bilibili items. It holds a compact index unless ``full_results`` is set,
|
|
4107
|
+
because repeating every page body in both ``pages`` and ``results`` doubled
|
|
4108
|
+
the response and pushed the tail past a caller's size cap. A surf run keeps
|
|
4109
|
+
its extra ``trail``/``visited`` fields.
|
|
4110
|
+
"""
|
|
4111
|
+
|
|
4112
|
+
envelope: dict[str, Any] = {
|
|
4113
|
+
"tool": "web-forager",
|
|
4114
|
+
"version": __version__,
|
|
4115
|
+
"action": action,
|
|
4116
|
+
"generated_at": utc_now(),
|
|
4117
|
+
}
|
|
4118
|
+
envelope.update(payload)
|
|
4119
|
+
if "results" not in envelope:
|
|
4120
|
+
if "pages" in payload:
|
|
4121
|
+
if full_results:
|
|
4122
|
+
envelope["results"] = payload["pages"]
|
|
4123
|
+
else:
|
|
4124
|
+
snippets = {
|
|
4125
|
+
item.get("url", ""): item.get("snippet", "")
|
|
4126
|
+
for item in payload.get("search_results") or []
|
|
4127
|
+
}
|
|
4128
|
+
entries: list[dict[str, Any]] = []
|
|
4129
|
+
for page in payload["pages"]:
|
|
4130
|
+
entry = summarize_record(page)
|
|
4131
|
+
snippet = snippets.get(page.get("url") or page.get("final_url") or "")
|
|
4132
|
+
if snippet:
|
|
4133
|
+
entry["snippet"] = snippet
|
|
4134
|
+
entries.append(entry)
|
|
4135
|
+
envelope["results"] = entries
|
|
4136
|
+
elif "videos" in payload:
|
|
4137
|
+
envelope["results"] = payload["videos"]
|
|
4138
|
+
elif "search_results" in payload:
|
|
4139
|
+
envelope["results"] = payload["search_results"]
|
|
4140
|
+
elif "markdown" in payload or "error" in payload:
|
|
4141
|
+
envelope["results"] = [payload] if full_results else [summarize_record(payload)]
|
|
4142
|
+
else:
|
|
4143
|
+
envelope["results"] = [payload]
|
|
4144
|
+
return envelope
|
|
4145
|
+
|
|
4146
|
+
|
|
4147
|
+
CORE_SHARED_KEYS = (
|
|
4148
|
+
"metadata",
|
|
4149
|
+
"markdown",
|
|
4150
|
+
"tables",
|
|
4151
|
+
"warnings",
|
|
4152
|
+
"url",
|
|
4153
|
+
"final_url",
|
|
4154
|
+
"title",
|
|
4155
|
+
"captured_at",
|
|
4156
|
+
"fetched_at",
|
|
4157
|
+
"collected_at",
|
|
4158
|
+
"source",
|
|
4159
|
+
"mode",
|
|
4160
|
+
"schema_version",
|
|
4161
|
+
)
|
|
4162
|
+
|
|
4163
|
+
|
|
4164
|
+
def normalize_record(record: dict[str, Any]) -> dict[str, Any]:
|
|
4165
|
+
"""Fold an HTTP record onto the core shape the browser mode also emits.
|
|
4166
|
+
|
|
4167
|
+
The two modes share four fields -- ``metadata``, ``markdown``, ``tables``
|
|
4168
|
+
and ``warnings`` -- and differ everywhere else. ``normalize_record`` maps
|
|
4169
|
+
both onto ``core-1`` so a caller can read one list of results without
|
|
4170
|
+
branching on the mode; everything mode-specific is kept under ``extra``
|
|
4171
|
+
rather than dropped. The browser side has the same function in
|
|
4172
|
+
``browser/lib/record.js``.
|
|
4173
|
+
"""
|
|
4174
|
+
|
|
4175
|
+
value = dict(record or {})
|
|
4176
|
+
metadata = value.get("metadata") or {}
|
|
4177
|
+
if not isinstance(metadata, dict):
|
|
4178
|
+
metadata = {}
|
|
4179
|
+
raw_source = value.get("source")
|
|
4180
|
+
browser = raw_source == "web-forager-browser" or bool(value.get("captured_at"))
|
|
4181
|
+
extra = {key: item for key, item in value.items() if key not in CORE_SHARED_KEYS}
|
|
4182
|
+
# ``clean`` records use ``source`` for the local file path, which is not the
|
|
4183
|
+
# producer name; keep it in extra instead of overwriting the core field.
|
|
4184
|
+
if raw_source and raw_source not in ("web-forager", "web-forager-browser"):
|
|
4185
|
+
extra["record_source"] = raw_source
|
|
4186
|
+
tables = value.get("tables")
|
|
4187
|
+
warnings = value.get("warnings")
|
|
4188
|
+
return {
|
|
4189
|
+
"schema_version": "core-1",
|
|
4190
|
+
"source": "web-forager-browser" if browser else "web-forager",
|
|
4191
|
+
"mode": "browser" if browser else "http",
|
|
4192
|
+
"url": value.get("url") or value.get("final_url") or "",
|
|
4193
|
+
"title": value.get("title") or metadata.get("title") or "",
|
|
4194
|
+
"collected_at": value.get("collected_at") or value.get("captured_at") or value.get("fetched_at") or "",
|
|
4195
|
+
"metadata": metadata,
|
|
4196
|
+
"markdown": value.get("markdown") or "",
|
|
4197
|
+
"tables": tables if isinstance(tables, list) else [],
|
|
4198
|
+
"warnings": warnings if isinstance(warnings, list) else [],
|
|
4199
|
+
"extra": extra,
|
|
4200
|
+
}
|
|
4201
|
+
|
|
4202
|
+
|
|
4203
|
+
def apply_core(envelope: dict[str, Any]) -> dict[str, Any]:
|
|
4204
|
+
"""Replace an envelope's ``results`` list with core-1 records."""
|
|
4205
|
+
|
|
4206
|
+
output = dict(envelope)
|
|
4207
|
+
results = output.get("results")
|
|
4208
|
+
if isinstance(results, list):
|
|
4209
|
+
output["results"] = [normalize_record(item) for item in results if isinstance(item, dict)]
|
|
4210
|
+
elif isinstance(results, dict):
|
|
4211
|
+
output["results"] = normalize_record(results)
|
|
4212
|
+
elif "markdown" in output or "metadata" in output:
|
|
4213
|
+
# A single-record action (``clean``) keeps its record at the top level.
|
|
4214
|
+
record = {
|
|
4215
|
+
key: value
|
|
4216
|
+
for key, value in output.items()
|
|
4217
|
+
if key not in ("tool", "version", "action", "generated_at")
|
|
4218
|
+
}
|
|
4219
|
+
output["results"] = [normalize_record(record)]
|
|
4220
|
+
return output
|