web-forager 1.3.0 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.clinerules/web-forager.md +3 -3
- package/.codebuddy-plugin/marketplace.json +1 -1
- package/.codebuddy-plugin/plugin.json +1 -1
- package/.codex-plugin/marketplace.json +1 -1
- package/.codex-plugin/mcp.json +20 -20
- package/.codex-plugin/plugin.json +1 -1
- package/.continue/rules/web-forager.md +4 -4
- package/.cursor/rules/web-forager.mdc +4 -4
- package/.dsh-plugin/index.js +1 -1
- package/.dsh-plugin/plugin.json +1 -1
- package/.github/copilot-instructions.md +3 -3
- package/.roo/rules/web-forager.md +3 -3
- package/.windsurf/rules/web-forager.md +4 -4
- package/AGENTS.md +10 -3
- package/CHANGELOG.md +508 -454
- package/README.md +499 -477
- package/README_zh.md +498 -477
- package/SKILL.md +115 -15
- package/THIRD_PARTY_LICENSES.txt +5 -0
- package/browser/bridges/playwright-bridge.js +8 -1
- package/browser/launch_chrome.py +193 -144
- package/browser/lib/bili-api.js +12 -1
- package/browser/lib/safety.js +27 -1
- package/browser/lib/session.js +16 -2
- package/browser/lib/untrusted.js +215 -56
- package/browser/mcp_server.js +5 -25
- package/browser/package.json +1 -1
- package/browser/playbooks/search-dig.js +3 -1
- package/browser/playbooks/wechat-dig.js +78 -78
- package/browser/run.js +19 -10
- package/commands/research.md +35 -0
- package/cordis.patch.yml +19 -19
- package/docs/DEPENDENCIES.md +1 -1
- package/docs/INSTALL_COMMANDS.md +4 -4
- package/docs/INSTALL_COMMANDS_zh.md +4 -4
- package/docs/REFERENCE.md +349 -308
- package/docs/REFERENCE_zh.md +342 -309
- package/manifest.json +3 -2
- package/mcp_server.py +383 -130
- package/package.json +4 -2
- package/requirements.txt +7 -4
- package/run.py +61 -5
- package/scripts/bilibili.py +11 -3
- package/scripts/engine.py +1039 -118
- package/scripts/pdf.py +21 -0
- package/scripts/research.py +2087 -0
- package/scripts/scholar.py +33 -6
- package/scripts/sitemap.py +20 -5
- package/scripts/untrusted.py +312 -95
package/scripts/engine.py
CHANGED
|
@@ -51,7 +51,7 @@ from bs4.element import NavigableString, Tag
|
|
|
51
51
|
|
|
52
52
|
from . import untrusted
|
|
53
53
|
|
|
54
|
-
__version__ = "1.3.
|
|
54
|
+
__version__ = "1.3.1"
|
|
55
55
|
|
|
56
56
|
DEFAULT_USER_AGENT = (
|
|
57
57
|
f"WebForager/{__version__} (Information Gathering; "
|
|
@@ -162,6 +162,17 @@ def _allow_private_from_env() -> bool:
|
|
|
162
162
|
)
|
|
163
163
|
|
|
164
164
|
|
|
165
|
+
def _query_robots_from_env() -> str:
|
|
166
|
+
"""User-level policy for query endpoints (search, scholarly APIs).
|
|
167
|
+
|
|
168
|
+
This is deliberately an environment/CLI decision, not an MCP parameter:
|
|
169
|
+
a fetched page must never be able to turn robots off for the agent.
|
|
170
|
+
"""
|
|
171
|
+
|
|
172
|
+
value = os.environ.get("WEB_FORAGER_QUERY_ROBOTS", "warn").strip().lower()
|
|
173
|
+
return value if value in ("respect", "warn", "ignore") else "warn"
|
|
174
|
+
|
|
175
|
+
|
|
165
176
|
@dataclass
|
|
166
177
|
class FetchConfig:
|
|
167
178
|
"""Runtime knobs for one CLI invocation."""
|
|
@@ -187,6 +198,10 @@ class FetchConfig:
|
|
|
187
198
|
max_links: int = 20
|
|
188
199
|
follow: str = "same-site" # same-site | all | none
|
|
189
200
|
search_engine: str = "auto"
|
|
201
|
+
# Query interfaces (search engines, scholarly APIs) have their own policy.
|
|
202
|
+
# warn reads a disallowed endpoint and records the override; respect
|
|
203
|
+
# refuses it; ignore skips the check. MCP never lets the model choose.
|
|
204
|
+
query_robots: str = field(default_factory=_query_robots_from_env)
|
|
190
205
|
search_pages: int = 1
|
|
191
206
|
# Engines are different hosts, so they may run at once without breaking the
|
|
192
207
|
# per-origin delay. Set False for a strictly serial diagnostic run.
|
|
@@ -208,6 +223,8 @@ class FetchConfig:
|
|
|
208
223
|
raise ValueError("render must be auto, never or always")
|
|
209
224
|
if self.robots_policy not in ("respect", "warn", "ignore"):
|
|
210
225
|
raise ValueError("robots_policy must be respect, warn or ignore")
|
|
226
|
+
if self.query_robots not in ("respect", "warn", "ignore"):
|
|
227
|
+
raise ValueError("query_robots must be respect, warn or ignore")
|
|
211
228
|
if self.input_gate not in ("off", "mark", "strict"):
|
|
212
229
|
raise ValueError("input_gate must be off, mark or strict")
|
|
213
230
|
if self.follow not in ("same-site", "all", "none"):
|
|
@@ -451,6 +468,7 @@ def _guarded_request(
|
|
|
451
468
|
*,
|
|
452
469
|
allow_private: bool = False,
|
|
453
470
|
max_redirects: int = 5,
|
|
471
|
+
before_request: Any = None,
|
|
454
472
|
**kwargs: Any,
|
|
455
473
|
) -> Any:
|
|
456
474
|
"""GET ``url`` and follow redirects one hop at a time, checking each target.
|
|
@@ -463,6 +481,8 @@ def _guarded_request(
|
|
|
463
481
|
current = url
|
|
464
482
|
for _ in range(max_redirects + 1):
|
|
465
483
|
assert_public_url(current, allow_private=allow_private)
|
|
484
|
+
if before_request is not None:
|
|
485
|
+
before_request(current)
|
|
466
486
|
response = getter(current, allow_redirects=False, **kwargs)
|
|
467
487
|
location = response.headers.get("Location") if response.headers else None
|
|
468
488
|
if response.status_code in (301, 302, 303, 307, 308) and location:
|
|
@@ -689,7 +709,10 @@ class RobotsCache:
|
|
|
689
709
|
robots_url,
|
|
690
710
|
allow_private=self.config.allow_private,
|
|
691
711
|
headers=headers,
|
|
692
|
-
timeout=
|
|
712
|
+
timeout=(
|
|
713
|
+
min(max(self.config.timeout, 1.0), 3.0),
|
|
714
|
+
min(max(self.config.timeout, 1.0), 6.0),
|
|
715
|
+
),
|
|
693
716
|
)
|
|
694
717
|
except requests.RequestException as exc:
|
|
695
718
|
# RFC 9309 §2.3.1: an unreachable robots.txt is treated as a
|
|
@@ -814,6 +837,7 @@ class HttpResponse:
|
|
|
814
837
|
encoding: str
|
|
815
838
|
content_type: str
|
|
816
839
|
from_cache: bool = False
|
|
840
|
+
robots_overridden: bool = False
|
|
817
841
|
|
|
818
842
|
|
|
819
843
|
def _cache_root(config: FetchConfig) -> Path:
|
|
@@ -822,6 +846,17 @@ def _cache_root(config: FetchConfig) -> Path:
|
|
|
822
846
|
return Path.home() / ".web-forager" / "cache"
|
|
823
847
|
|
|
824
848
|
|
|
849
|
+
def _private_path(path: Path, mode: int) -> None:
|
|
850
|
+
"""Best-effort owner-only permissions for local state on POSIX."""
|
|
851
|
+
|
|
852
|
+
if os.name == "nt":
|
|
853
|
+
return
|
|
854
|
+
try:
|
|
855
|
+
path.chmod(mode)
|
|
856
|
+
except OSError:
|
|
857
|
+
pass
|
|
858
|
+
|
|
859
|
+
|
|
825
860
|
def _cache_key(url: str) -> str:
|
|
826
861
|
import hashlib
|
|
827
862
|
|
|
@@ -880,20 +915,42 @@ class HttpFetcher:
|
|
|
880
915
|
except Exception:
|
|
881
916
|
pass
|
|
882
917
|
|
|
918
|
+
def clone(self) -> "HttpFetcher":
|
|
919
|
+
"""A per-thread fetcher sharing robots and rate limits but not a session.
|
|
920
|
+
|
|
921
|
+
``requests.Session`` is not thread-safe, so parallel focus reads use one
|
|
922
|
+
clone per worker. The robots cache and rate limiter are shared, which
|
|
923
|
+
keeps politeness global; cookies already collected are copied once.
|
|
924
|
+
"""
|
|
925
|
+
|
|
926
|
+
other = HttpFetcher(self.config)
|
|
927
|
+
other.robots = self.robots
|
|
928
|
+
other.limiter = self.limiter
|
|
929
|
+
try:
|
|
930
|
+
other.session.cookies.update(self.session.cookies)
|
|
931
|
+
except Exception:
|
|
932
|
+
pass
|
|
933
|
+
return other
|
|
934
|
+
|
|
883
935
|
# -- robots ---------------------------------------------------------
|
|
884
936
|
|
|
885
|
-
def assert_allowed(
|
|
886
|
-
|
|
937
|
+
def assert_allowed(
|
|
938
|
+
self, url: str, robots_policy: Optional[str] = None
|
|
939
|
+
) -> RobotsPolicy:
|
|
940
|
+
"""Raise RobotsDenied when the effective policy is ``respect``.
|
|
887
941
|
|
|
888
942
|
``warn`` returns the denying policy instead of raising, so the caller
|
|
889
943
|
can still read the page and must record ``robots_overridden: true``.
|
|
944
|
+
``robots_policy`` lets a query interface use its own policy without
|
|
945
|
+
changing the policy for target pages.
|
|
890
946
|
"""
|
|
891
947
|
|
|
892
|
-
|
|
948
|
+
effective = (robots_policy or self.config.robots_policy).strip().lower()
|
|
949
|
+
if not self.config.respect_robots or effective == "ignore":
|
|
893
950
|
return RobotsPolicy.allowing_all(note="robots checking disabled by policy")
|
|
894
951
|
policy = self.robots.policy_for(url)
|
|
895
952
|
if not policy.allowed(url):
|
|
896
|
-
if
|
|
953
|
+
if effective == "warn":
|
|
897
954
|
return policy
|
|
898
955
|
detail = policy.note or policy.source or "robots.txt"
|
|
899
956
|
raise RobotsDenied(f"robots.txt disallows {url} ({detail})")
|
|
@@ -902,7 +959,12 @@ class HttpFetcher:
|
|
|
902
959
|
# -- main entry -----------------------------------------------------
|
|
903
960
|
|
|
904
961
|
def fetch(
|
|
905
|
-
self,
|
|
962
|
+
self,
|
|
963
|
+
url: str,
|
|
964
|
+
*,
|
|
965
|
+
check_robots: bool = True,
|
|
966
|
+
use_cache: bool = True,
|
|
967
|
+
robots_policy: Optional[str] = None,
|
|
906
968
|
) -> HttpResponse:
|
|
907
969
|
"""Fetch one URL.
|
|
908
970
|
|
|
@@ -914,8 +976,16 @@ class HttpFetcher:
|
|
|
914
976
|
url = normalize_url(url)
|
|
915
977
|
assert_public_url(url, allow_private=self.config.allow_private)
|
|
916
978
|
policy: Optional[RobotsPolicy] = None
|
|
979
|
+
overridden = {"value": False}
|
|
917
980
|
if check_robots:
|
|
918
|
-
policy = self.assert_allowed(url)
|
|
981
|
+
policy = self.assert_allowed(url, robots_policy)
|
|
982
|
+
if policy is not None and not policy.allowed(url):
|
|
983
|
+
overridden["value"] = True
|
|
984
|
+
|
|
985
|
+
def robots_guard(target: str) -> None:
|
|
986
|
+
checked = self.assert_allowed(target, robots_policy)
|
|
987
|
+
if checked is not None and not checked.allowed(target):
|
|
988
|
+
overridden["value"] = True
|
|
919
989
|
|
|
920
990
|
cached = self._read_cache(url) if use_cache else None
|
|
921
991
|
if cached is not None:
|
|
@@ -932,7 +1002,10 @@ class HttpFetcher:
|
|
|
932
1002
|
url,
|
|
933
1003
|
allow_private=self.config.allow_private,
|
|
934
1004
|
max_redirects=self.session.max_redirects,
|
|
935
|
-
|
|
1005
|
+
before_request=robots_guard if check_robots else None,
|
|
1006
|
+
# A connect timeout per resolved address: a blocked
|
|
1007
|
+
# address must not consume the full read budget.
|
|
1008
|
+
timeout=(min(self.config.timeout, 3.0), self.config.timeout),
|
|
936
1009
|
stream=True,
|
|
937
1010
|
)
|
|
938
1011
|
except requests.RequestException as exc:
|
|
@@ -954,6 +1027,7 @@ class HttpFetcher:
|
|
|
954
1027
|
raise FetchError(f"HTTP {status} for {url}")
|
|
955
1028
|
|
|
956
1029
|
result = self._read_response(response, url)
|
|
1030
|
+
result.robots_overridden = overridden["value"]
|
|
957
1031
|
if use_cache:
|
|
958
1032
|
self._write_cache(url, result)
|
|
959
1033
|
return result
|
|
@@ -984,7 +1058,7 @@ class HttpFetcher:
|
|
|
984
1058
|
content_type = str(meta.get("content_type") or "text/html")
|
|
985
1059
|
encoding = str(meta.get("encoding") or "utf-8")
|
|
986
1060
|
text = body.decode(encoding, errors="replace")
|
|
987
|
-
log(f"cache hit: {url}", self.config)
|
|
1061
|
+
log(f"cache hit: {redact_url_for_log(url)}", self.config)
|
|
988
1062
|
return HttpResponse(
|
|
989
1063
|
url=url,
|
|
990
1064
|
final_url=str(meta.get("final_url") or url),
|
|
@@ -995,6 +1069,7 @@ class HttpFetcher:
|
|
|
995
1069
|
encoding=encoding,
|
|
996
1070
|
content_type=content_type,
|
|
997
1071
|
from_cache=True,
|
|
1072
|
+
robots_overridden=bool(meta.get("robots_overridden", False)),
|
|
998
1073
|
)
|
|
999
1074
|
|
|
1000
1075
|
def _write_cache(self, url: str, response: HttpResponse) -> None:
|
|
@@ -1013,8 +1088,10 @@ class HttpFetcher:
|
|
|
1013
1088
|
}
|
|
1014
1089
|
meta_path, body_path = self._cache_paths(url)
|
|
1015
1090
|
try:
|
|
1016
|
-
meta_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1091
|
+
meta_path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
1092
|
+
_private_path(meta_path.parent, 0o700)
|
|
1017
1093
|
body_path.write_bytes(response.content)
|
|
1094
|
+
_private_path(body_path, 0o600)
|
|
1018
1095
|
meta_path.write_text(
|
|
1019
1096
|
json.dumps(
|
|
1020
1097
|
{
|
|
@@ -1024,11 +1101,13 @@ class HttpFetcher:
|
|
|
1024
1101
|
"headers": safe_headers,
|
|
1025
1102
|
"content_type": response.content_type,
|
|
1026
1103
|
"encoding": response.encoding,
|
|
1104
|
+
"robots_overridden": bool(response.robots_overridden),
|
|
1027
1105
|
},
|
|
1028
1106
|
ensure_ascii=False,
|
|
1029
1107
|
),
|
|
1030
1108
|
encoding="utf-8",
|
|
1031
1109
|
)
|
|
1110
|
+
_private_path(meta_path, 0o600)
|
|
1032
1111
|
except OSError as exc: # a cache failure must never fail the fetch
|
|
1033
1112
|
log(f"cache write skipped ({exc})", self.config)
|
|
1034
1113
|
|
|
@@ -1107,11 +1186,20 @@ class HttpFetcher:
|
|
|
1107
1186
|
self.limiter.wait(url)
|
|
1108
1187
|
response: Optional[requests.Response] = None
|
|
1109
1188
|
try:
|
|
1189
|
+
def resolve_robots_guard(target: str) -> None:
|
|
1190
|
+
if self.config.query_robots != "ignore":
|
|
1191
|
+
self.assert_allowed(target, self.config.query_robots)
|
|
1192
|
+
|
|
1110
1193
|
response = _guarded_request(
|
|
1111
1194
|
self.session.get,
|
|
1112
1195
|
url,
|
|
1113
1196
|
allow_private=self.config.allow_private,
|
|
1114
1197
|
max_redirects=self.session.max_redirects,
|
|
1198
|
+
before_request=(
|
|
1199
|
+
resolve_robots_guard
|
|
1200
|
+
if self.config.query_robots != "ignore"
|
|
1201
|
+
else None
|
|
1202
|
+
),
|
|
1115
1203
|
timeout=self.config.timeout,
|
|
1116
1204
|
stream=True,
|
|
1117
1205
|
)
|
|
@@ -1444,6 +1532,7 @@ def search(
|
|
|
1444
1532
|
limit: Optional[int] = None,
|
|
1445
1533
|
fetcher: Optional[HttpFetcher] = None,
|
|
1446
1534
|
report: Optional[list[dict[str, Any]]] = None,
|
|
1535
|
+
quality: Optional[dict[str, Any]] = None,
|
|
1447
1536
|
) -> list[dict[str, str]]:
|
|
1448
1537
|
"""Return ``[{title, url, snippet, source}]`` for a keyword query.
|
|
1449
1538
|
|
|
@@ -1454,11 +1543,14 @@ def search(
|
|
|
1454
1543
|
|
|
1455
1544
|
Pass ``report`` to collect one entry per engine tried — name, whether it
|
|
1456
1545
|
answered, how many results and how long it took — so a caller in a hostile
|
|
1457
|
-
network can see which backend is actually reachable.
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1546
|
+
network can see which backend is actually reachable. Pass ``quality`` to
|
|
1547
|
+
receive counts for the rows removed by the relevance filter.
|
|
1548
|
+
|
|
1549
|
+
Search endpoints are query interfaces, not target pages. Their robots
|
|
1550
|
+
policy is ``config.query_robots`` (default ``warn``): a disallowed query
|
|
1551
|
+
endpoint is read and the override is recorded; ``respect`` refuses it and
|
|
1552
|
+
``ignore`` skips the check. Result pages we later fetch still go through
|
|
1553
|
+
the normal target-page robots policy. Queries are rate-limited per host.
|
|
1462
1554
|
"""
|
|
1463
1555
|
|
|
1464
1556
|
query = (query or "").strip()
|
|
@@ -1508,6 +1600,8 @@ def search(
|
|
|
1508
1600
|
) -> tuple[dict[str, Any], Optional[list[dict[str, str]]]]:
|
|
1509
1601
|
started = time.monotonic()
|
|
1510
1602
|
entry: dict[str, Any] = {"engine": name}
|
|
1603
|
+
if config.query_robots != "respect":
|
|
1604
|
+
entry["query_robots"] = config.query_robots
|
|
1511
1605
|
if fallback:
|
|
1512
1606
|
entry["fallback"] = True
|
|
1513
1607
|
if qtext != query:
|
|
@@ -1531,6 +1625,7 @@ def search(
|
|
|
1531
1625
|
]
|
|
1532
1626
|
entry.update(
|
|
1533
1627
|
ok=True,
|
|
1628
|
+
status="ok" if items else "empty",
|
|
1534
1629
|
results=len(items),
|
|
1535
1630
|
ms=round((time.monotonic() - started) * 1000),
|
|
1536
1631
|
)
|
|
@@ -1539,6 +1634,7 @@ def search(
|
|
|
1539
1634
|
errors.append(f"{name}: {exc}")
|
|
1540
1635
|
entry.update(
|
|
1541
1636
|
ok=False,
|
|
1637
|
+
status=_engine_failure_status(exc),
|
|
1542
1638
|
results=0,
|
|
1543
1639
|
error=str(exc),
|
|
1544
1640
|
ms=round((time.monotonic() - started) * 1000),
|
|
@@ -1578,13 +1674,48 @@ def search(
|
|
|
1578
1674
|
if not groups:
|
|
1579
1675
|
raise FetchError("all search engines failed: " + "; ".join(errors) or "no engine available")
|
|
1580
1676
|
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1677
|
+
# Rank even when only one engine answered. The old single-group path
|
|
1678
|
+
# returned engine order untouched, which is how an Excel INDIRECT tutorial
|
|
1679
|
+
# could sit third for an "indirect prompt injection" query.
|
|
1680
|
+
merged = merge_search_results(groups, query, limit=limit)
|
|
1681
|
+
final = diversify_results(
|
|
1586
1682
|
dedupe_results(merged, limit=max(limit * 3, limit)), limit=limit
|
|
1587
1683
|
)
|
|
1684
|
+
annotate_results(final, query)
|
|
1685
|
+
before_filter = len(final)
|
|
1686
|
+
# When enough real answers exist, do not pad the list with results that
|
|
1687
|
+
# matched one word or the wrong language. With only one or two relevant
|
|
1688
|
+
# hits the unrelated rows stay at the bottom and a warning names them.
|
|
1689
|
+
relevant = [item for item in final if item.get("relevance")]
|
|
1690
|
+
if len(relevant) >= min(3, limit):
|
|
1691
|
+
final = relevant
|
|
1692
|
+
if quality is not None:
|
|
1693
|
+
quality["returned_before_filter"] = before_filter
|
|
1694
|
+
quality["dropped_off_topic"] = max(0, before_filter - len(final))
|
|
1695
|
+
return final
|
|
1696
|
+
|
|
1697
|
+
|
|
1698
|
+
def _engine_failure_status(exc: BaseException) -> str:
|
|
1699
|
+
"""Classify one engine failure for the machine-readable report.
|
|
1700
|
+
|
|
1701
|
+
A caller in a hostile network needs to tell an empty result from a block,
|
|
1702
|
+
a timeout and a parser change. The old ``ok: false`` plus free text hid
|
|
1703
|
+
that distinction, which made a blocked engine look like a thin topic.
|
|
1704
|
+
"""
|
|
1705
|
+
|
|
1706
|
+
text = f"{type(exc).__name__}: {exc}".lower()
|
|
1707
|
+
if any(word in text for word in ("timeout", "timed out", "time out")):
|
|
1708
|
+
return "timeout"
|
|
1709
|
+
if "robots" in text:
|
|
1710
|
+
return "robots_denied"
|
|
1711
|
+
if any(
|
|
1712
|
+
word in text
|
|
1713
|
+
for word in ("verification", "antispider", "captcha", "blocked", "forbidden")
|
|
1714
|
+
):
|
|
1715
|
+
return "blocked"
|
|
1716
|
+
if any(word in text for word in ("parse", "decode", "json", "xml", "unexpected")):
|
|
1717
|
+
return "parse_error"
|
|
1718
|
+
return "error"
|
|
1588
1719
|
|
|
1589
1720
|
|
|
1590
1721
|
def _thin_results(groups: list[tuple[str, list[dict[str, str]]]], limit: int) -> bool:
|
|
@@ -1627,6 +1758,13 @@ def _search_one(
|
|
|
1627
1758
|
) -> list[dict[str, str]]:
|
|
1628
1759
|
quoted = urllib.parse.quote_plus(query)
|
|
1629
1760
|
pages = max(1, pages)
|
|
1761
|
+
# Search endpoints are query interfaces, not target pages. They still get
|
|
1762
|
+
# a policy: respect (default) refuses a disallowed endpoint, warn reads it
|
|
1763
|
+
# and marks the override, ignore skips the check. MCP force-sets respect.
|
|
1764
|
+
query_fetch = {
|
|
1765
|
+
"check_robots": config.query_robots != "ignore",
|
|
1766
|
+
"robots_policy": config.query_robots,
|
|
1767
|
+
}
|
|
1630
1768
|
collected: list[dict[str, str]] = []
|
|
1631
1769
|
|
|
1632
1770
|
for page in range(1, pages + 1):
|
|
@@ -1634,14 +1772,14 @@ def _search_one(
|
|
|
1634
1772
|
break
|
|
1635
1773
|
if name == "360":
|
|
1636
1774
|
response = fetcher.fetch(
|
|
1637
|
-
_SO_360.format(query=quoted, page=page),
|
|
1775
|
+
_SO_360.format(query=quoted, page=page), **query_fetch
|
|
1638
1776
|
)
|
|
1639
1777
|
if is_360_blocked(response.text):
|
|
1640
1778
|
raise FetchError("360 returned a verification page; wait and retry later")
|
|
1641
1779
|
found = parse_360_html(response.text, limit=limit)
|
|
1642
1780
|
elif name == "sogou":
|
|
1643
1781
|
response = fetcher.fetch(
|
|
1644
|
-
_SOGOU.format(query=quoted, page=page),
|
|
1782
|
+
_SOGOU.format(query=quoted, page=page), **query_fetch
|
|
1645
1783
|
)
|
|
1646
1784
|
if is_sogou_blocked(response.text):
|
|
1647
1785
|
raise FetchError("Sogou returned a verification page (antispider)")
|
|
@@ -1651,7 +1789,7 @@ def _search_one(
|
|
|
1651
1789
|
found = resolve_sogou_results(parsed, fetcher, limit=limit)
|
|
1652
1790
|
elif name == "weixin":
|
|
1653
1791
|
response = fetcher.fetch(
|
|
1654
|
-
_WEIXIN.format(query=quoted, page=page),
|
|
1792
|
+
_WEIXIN.format(query=quoted, page=page), **query_fetch
|
|
1655
1793
|
)
|
|
1656
1794
|
if is_sogou_blocked(response.text):
|
|
1657
1795
|
raise FetchError("weixin.sogou.com returned a verification page (antispider)")
|
|
@@ -1660,7 +1798,7 @@ def _search_one(
|
|
|
1660
1798
|
)
|
|
1661
1799
|
found = resolve_sogou_results(parsed, fetcher, limit=limit)
|
|
1662
1800
|
elif name == "duckduckgo":
|
|
1663
|
-
response = fetcher.fetch(_DDG_HTML.format(query=quoted),
|
|
1801
|
+
response = fetcher.fetch(_DDG_HTML.format(query=quoted), **query_fetch)
|
|
1664
1802
|
found = parse_ddg_html(response.text, limit=limit)
|
|
1665
1803
|
else:
|
|
1666
1804
|
first = 1 + (page - 1) * 10
|
|
@@ -1668,7 +1806,7 @@ def _search_one(
|
|
|
1668
1806
|
try:
|
|
1669
1807
|
response = fetcher.fetch(
|
|
1670
1808
|
_BING_RSS.format(query=quoted, count=limit, first=first),
|
|
1671
|
-
|
|
1809
|
+
**query_fetch,
|
|
1672
1810
|
)
|
|
1673
1811
|
found = parse_bing_rss(response.text, limit=limit)
|
|
1674
1812
|
except FetchError:
|
|
@@ -1676,7 +1814,7 @@ def _search_one(
|
|
|
1676
1814
|
if not found:
|
|
1677
1815
|
response = fetcher.fetch(
|
|
1678
1816
|
_BING_HTML.format(query=quoted, count=limit, first=first),
|
|
1679
|
-
|
|
1817
|
+
**query_fetch,
|
|
1680
1818
|
)
|
|
1681
1819
|
found = parse_bing_html(response.text, limit=limit)
|
|
1682
1820
|
|
|
@@ -1705,46 +1843,495 @@ def result_coverage(result: dict[str, str], tokens: list[str]) -> float:
|
|
|
1705
1843
|
return sum(1 for token in tokens if token in haystack) / len(tokens)
|
|
1706
1844
|
|
|
1707
1845
|
|
|
1708
|
-
|
|
1709
|
-
"
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
"
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1846
|
+
def _content_tokens(query: str) -> list[str]:
|
|
1847
|
+
"""Query terms that carry meaning; stopwords and one-letter words are noise.
|
|
1848
|
+
|
|
1849
|
+
Chinese is not whitespace-separated. A long CJK run is expanded into
|
|
1850
|
+
character bigrams for matching; a mixed token carrying digits (``2026年11月``)
|
|
1851
|
+
stays whole because the date is itself the searchable unit. This is not a
|
|
1852
|
+
word segmenter, but it stops a whole sentence from becoming one token that
|
|
1853
|
+
no result can ever match.
|
|
1854
|
+
"""
|
|
1855
|
+
|
|
1856
|
+
tokens = _query_tokens(query)
|
|
1857
|
+
content: list[str] = []
|
|
1858
|
+
for token in tokens:
|
|
1859
|
+
if token in _QUERY_STOPWORDS or len(token) < 2:
|
|
1860
|
+
continue
|
|
1861
|
+
if re.search(r"\d", token):
|
|
1862
|
+
content.append(token)
|
|
1863
|
+
continue
|
|
1864
|
+
if not _CJK_RE.search(token):
|
|
1865
|
+
content.append(token)
|
|
1866
|
+
continue
|
|
1867
|
+
for run in re.findall(
|
|
1868
|
+
r"[㐀-䶿一-鿿豈--ヿ가-]+"
|
|
1869
|
+
r"|[A-Za-z0-9]+",
|
|
1870
|
+
token,
|
|
1871
|
+
):
|
|
1872
|
+
if _CJK_RE.search(run):
|
|
1873
|
+
if len(run) <= 3:
|
|
1874
|
+
content.append(run)
|
|
1875
|
+
else:
|
|
1876
|
+
content.extend(run[index : index + 2] for index in range(len(run) - 1))
|
|
1877
|
+
elif len(run) >= 2:
|
|
1878
|
+
content.append(run.lower())
|
|
1879
|
+
return list(dict.fromkeys(content)) or tokens
|
|
1880
|
+
|
|
1881
|
+
|
|
1882
|
+
_LATIN_WORD_RE = re.compile(r"[A-Za-z][A-Za-z0-9+#.-]*")
|
|
1883
|
+
|
|
1884
|
+
|
|
1885
|
+
def _stem_token(token: str) -> str:
|
|
1886
|
+
"""A deliberately small English stemmer for matching, not display.
|
|
1887
|
+
|
|
1888
|
+
``injections`` and ``injection`` should meet; a full Porter stemmer would
|
|
1889
|
+
also merge unrelated words and make the relevance check harder to explain.
|
|
1890
|
+
Tokens containing CJK or punctuation are returned unchanged.
|
|
1891
|
+
"""
|
|
1892
|
+
|
|
1893
|
+
value = token.lower()
|
|
1894
|
+
if not re.fullmatch(r"[a-z][a-z0-9+-]*", value) or len(value) <= 4:
|
|
1895
|
+
return value
|
|
1896
|
+
if value.endswith("ies"):
|
|
1897
|
+
return value[:-3] + "y"
|
|
1898
|
+
if value.endswith("ing") and len(value) > 5:
|
|
1899
|
+
return value[:-3]
|
|
1900
|
+
if value.endswith("ed") and len(value) > 4:
|
|
1901
|
+
return value[:-2]
|
|
1902
|
+
if value.endswith("sses"):
|
|
1903
|
+
return value[:-2]
|
|
1904
|
+
if value.endswith(("ches", "shes", "xes", "zes", "oes")):
|
|
1905
|
+
return value[:-2]
|
|
1906
|
+
if value.endswith("s") and not value.endswith("ss"):
|
|
1907
|
+
return value[:-1]
|
|
1908
|
+
return value
|
|
1909
|
+
|
|
1910
|
+
|
|
1911
|
+
def _stems(text: str) -> set[str]:
|
|
1912
|
+
return {_stem_token(word) for word in _LATIN_WORD_RE.findall(text.lower())}
|
|
1913
|
+
|
|
1914
|
+
|
|
1915
|
+
def _token_in_text(token: str, text: str, stems: set[str]) -> bool:
|
|
1916
|
+
token = (token or "").lower()
|
|
1917
|
+
if token and token in text:
|
|
1918
|
+
return True
|
|
1919
|
+
return bool(re.fullmatch(r"[a-z][a-z0-9+-]*", token)) and _stem_token(token) in stems
|
|
1920
|
+
|
|
1921
|
+
|
|
1922
|
+
def _script_stats(text: str) -> tuple[int, int]:
|
|
1923
|
+
"""Return ``(CJK characters, Latin words)`` for a language sanity check."""
|
|
1924
|
+
|
|
1925
|
+
value = str(text or "")
|
|
1926
|
+
return (
|
|
1927
|
+
len(_CJK_RE.findall(value)),
|
|
1928
|
+
len(re.findall(r"[A-Za-z]{2,}", value)),
|
|
1929
|
+
)
|
|
1930
|
+
|
|
1931
|
+
|
|
1932
|
+
_INTENT_PATTERNS: dict[str, re.Pattern[str]] = {
|
|
1933
|
+
"news": re.compile(
|
|
1934
|
+
r"新闻|资讯|最新|今日|今天|本周|本月|今年|刚刚|发布会|财报|营收|股价|"
|
|
1935
|
+
r"202[4-9]年|\bnews\b|\bupdate\b",
|
|
1936
|
+
re.I,
|
|
1937
|
+
),
|
|
1938
|
+
"comparison": re.compile(
|
|
1939
|
+
r"对比|比较|区别|差异|怎么选|如何选|哪个好|哪个更|优缺点|取舍|"
|
|
1940
|
+
r"\bvs\.?\b|\bversus\b|\bcomparison\b|\bcompare\b|\bbetter\b|\bchoose\b",
|
|
1941
|
+
re.I,
|
|
1942
|
+
),
|
|
1943
|
+
"technical": re.compile(
|
|
1944
|
+
r"python|javascript|typescript|java|golang|rust|asyncio|api|sdk|cli|"
|
|
1945
|
+
r"http|json|sql|docker|kubernetes|git|linux|regex|报错|异常|安装|配置|"
|
|
1946
|
+
r"函数|接口|依赖|编译|代码|文档|版本|任务组|协程",
|
|
1947
|
+
re.I,
|
|
1948
|
+
),
|
|
1949
|
+
"policy": re.compile(
|
|
1950
|
+
r"法规|法律|政策|合规|监管|标准|条例|法案|司法|欧盟|ai act|gdpr|"
|
|
1951
|
+
r"regulation|compliance|\blaw\b|\bact\b",
|
|
1952
|
+
re.I,
|
|
1953
|
+
),
|
|
1954
|
+
"howto": re.compile(
|
|
1955
|
+
r"怎么办|咋办|怎么解决|如何解决|修复|连不上|打不开|用不了|设置|教程|"
|
|
1956
|
+
r"步骤|方法|技巧|how to|fix|troubleshoot|setup|repair",
|
|
1957
|
+
re.I,
|
|
1958
|
+
),
|
|
1959
|
+
"research": re.compile(
|
|
1960
|
+
r"arxiv|论文|文献|综述|survey|paper|study|research|academic|acl|ieee|acm",
|
|
1961
|
+
re.I,
|
|
1962
|
+
),
|
|
1963
|
+
"factual": re.compile(
|
|
1964
|
+
r"是谁|多少|哪年|何时|什么时候|定义|作者|创始人|营收|人口|面积|"
|
|
1965
|
+
r"what is|who is|when|how many|definition",
|
|
1966
|
+
re.I,
|
|
1967
|
+
),
|
|
1968
|
+
}
|
|
1969
|
+
_INTENT_ORDER = (
|
|
1970
|
+
"news",
|
|
1971
|
+
"policy",
|
|
1972
|
+
"comparison",
|
|
1973
|
+
"technical",
|
|
1974
|
+
"howto",
|
|
1975
|
+
"research",
|
|
1976
|
+
"factual",
|
|
1977
|
+
"general",
|
|
1723
1978
|
)
|
|
1724
|
-
|
|
1725
|
-
"
|
|
1726
|
-
"
|
|
1727
|
-
"
|
|
1728
|
-
"
|
|
1729
|
-
"
|
|
1979
|
+
_INTENT_STRONG_SIGNALS = {
|
|
1980
|
+
"comparison": ("怎么选", "如何选", "哪个好", "对比", "vs"),
|
|
1981
|
+
"howto": ("怎么办", "咋办", "连不上", "打不开", "怎么解决"),
|
|
1982
|
+
"factual": ("是谁", "的作者", "多少", "定义"),
|
|
1983
|
+
"policy": ("法规", "合规", "监管", "ai act"),
|
|
1984
|
+
"news": ("新闻", "最新", "今天", "本月"),
|
|
1985
|
+
"technical": ("报错", "异常", "api", "sdk", "asyncio"),
|
|
1986
|
+
}
|
|
1987
|
+
|
|
1988
|
+
|
|
1989
|
+
def classify_intent(query: str) -> dict[str, Any]:
|
|
1990
|
+
"""Classify what kind of source will actually answer ``query``.
|
|
1991
|
+
|
|
1992
|
+
Free and deterministic: a small keyword vote, with a few strong phrases
|
|
1993
|
+
counted twice. It is deliberately conservative — ``general`` is a valid
|
|
1994
|
+
answer, and authority weighting keeps working without it.
|
|
1995
|
+
"""
|
|
1996
|
+
|
|
1997
|
+
text = (query or "").lower()
|
|
1998
|
+
scores: dict[str, int] = {}
|
|
1999
|
+
signals: dict[str, list[str]] = {}
|
|
2000
|
+
for name, pattern in _INTENT_PATTERNS.items():
|
|
2001
|
+
matches = pattern.findall(text)
|
|
2002
|
+
if matches:
|
|
2003
|
+
scores[name] = len(matches)
|
|
2004
|
+
signals[name] = list(dict.fromkeys(item.lower() for item in matches))[:5]
|
|
2005
|
+
for name, phrases in _INTENT_STRONG_SIGNALS.items():
|
|
2006
|
+
for phrase in phrases:
|
|
2007
|
+
if phrase in text:
|
|
2008
|
+
scores[name] = scores.get(name, 0) + 1
|
|
2009
|
+
signals.setdefault(name, []).append(phrase)
|
|
2010
|
+
if not scores:
|
|
2011
|
+
return {"intent": "general", "confidence": 0.3, "signals": []}
|
|
2012
|
+
best = max(
|
|
2013
|
+
scores,
|
|
2014
|
+
key=lambda name: (scores[name], -_INTENT_ORDER.index(name)),
|
|
2015
|
+
)
|
|
2016
|
+
confidence = min(0.95, round(0.45 + 0.1 * scores[best], 2))
|
|
2017
|
+
return {
|
|
2018
|
+
"intent": best,
|
|
2019
|
+
"confidence": confidence,
|
|
2020
|
+
"signals": list(dict.fromkeys(signals.get(best, [])))[:5],
|
|
2021
|
+
}
|
|
2022
|
+
|
|
2023
|
+
|
|
2024
|
+
_AUTHORITY_TIERS: tuple[tuple[str, tuple[str, ...]], ...] = (
|
|
2025
|
+
(
|
|
2026
|
+
"primary",
|
|
2027
|
+
(
|
|
2028
|
+
"docs.python.org",
|
|
2029
|
+
"developer.mozilla.org",
|
|
2030
|
+
"go.dev",
|
|
2031
|
+
"rust-lang.org",
|
|
2032
|
+
"nodejs.org",
|
|
2033
|
+
"developer.apple.com",
|
|
2034
|
+
"learn.microsoft.com",
|
|
2035
|
+
"support.microsoft.com",
|
|
2036
|
+
"kubernetes.io",
|
|
2037
|
+
"docs.docker.com",
|
|
2038
|
+
"postgresql.org",
|
|
2039
|
+
"nginx.org",
|
|
2040
|
+
"apache.org",
|
|
2041
|
+
"w3.org",
|
|
2042
|
+
"ietf.org",
|
|
2043
|
+
"unicode.org",
|
|
2044
|
+
"docs.github.com",
|
|
2045
|
+
),
|
|
2046
|
+
),
|
|
2047
|
+
(
|
|
2048
|
+
"official",
|
|
2049
|
+
(
|
|
2050
|
+
"gov.cn",
|
|
2051
|
+
"gov",
|
|
2052
|
+
"europa.eu",
|
|
2053
|
+
"sec.gov",
|
|
2054
|
+
"nist.gov",
|
|
2055
|
+
"who.int",
|
|
2056
|
+
"worldbank.org",
|
|
2057
|
+
"un.org",
|
|
2058
|
+
"nvidia.com",
|
|
2059
|
+
"openai.com",
|
|
2060
|
+
"anthropic.com",
|
|
2061
|
+
),
|
|
2062
|
+
),
|
|
2063
|
+
(
|
|
2064
|
+
"academic",
|
|
2065
|
+
(
|
|
2066
|
+
"arxiv.org",
|
|
2067
|
+
"aclanthology.org",
|
|
2068
|
+
"openreview.net",
|
|
2069
|
+
"semanticscholar.org",
|
|
2070
|
+
"pubmed.ncbi.nlm.nih.gov",
|
|
2071
|
+
"nature.com",
|
|
2072
|
+
"science.org",
|
|
2073
|
+
"springer.com",
|
|
2074
|
+
"sciencedirect.com",
|
|
2075
|
+
"ieee.org",
|
|
2076
|
+
"acm.org",
|
|
2077
|
+
"plos.org",
|
|
2078
|
+
"biorxiv.org",
|
|
2079
|
+
),
|
|
2080
|
+
),
|
|
2081
|
+
(
|
|
2082
|
+
"media",
|
|
2083
|
+
(
|
|
2084
|
+
"reuters.com",
|
|
2085
|
+
"bloomberg.com",
|
|
2086
|
+
"ft.com",
|
|
2087
|
+
"wsj.com",
|
|
2088
|
+
"nytimes.com",
|
|
2089
|
+
"bbc.com",
|
|
2090
|
+
"bbc.co.uk",
|
|
2091
|
+
"caixin.com",
|
|
2092
|
+
"yicai.com",
|
|
2093
|
+
"cls.cn",
|
|
2094
|
+
"nbd.com.cn",
|
|
2095
|
+
"stcn.com",
|
|
2096
|
+
"thepaper.cn",
|
|
2097
|
+
"jiemian.com",
|
|
2098
|
+
"chinanews.com.cn",
|
|
2099
|
+
"finance.sina.com.cn",
|
|
2100
|
+
),
|
|
2101
|
+
),
|
|
2102
|
+
(
|
|
2103
|
+
"reference",
|
|
2104
|
+
(
|
|
2105
|
+
"wikipedia.org",
|
|
2106
|
+
"britannica.com",
|
|
2107
|
+
"imdb.com",
|
|
2108
|
+
"themoviedb.org",
|
|
2109
|
+
),
|
|
2110
|
+
),
|
|
2111
|
+
(
|
|
2112
|
+
"community",
|
|
2113
|
+
(
|
|
2114
|
+
"stackoverflow.com",
|
|
2115
|
+
"stackexchange.com",
|
|
2116
|
+
"github.com",
|
|
2117
|
+
"reddit.com",
|
|
2118
|
+
"news.ycombinator.com",
|
|
2119
|
+
"zhihu.com",
|
|
2120
|
+
"cnblogs.com",
|
|
2121
|
+
"medium.com",
|
|
2122
|
+
"dev.to",
|
|
2123
|
+
"quora.com",
|
|
2124
|
+
),
|
|
2125
|
+
),
|
|
2126
|
+
(
|
|
2127
|
+
"low",
|
|
2128
|
+
(
|
|
2129
|
+
"csdn.net",
|
|
2130
|
+
"php.cn",
|
|
2131
|
+
"doc88.com",
|
|
2132
|
+
"docin.com",
|
|
2133
|
+
"wenku.baidu.com",
|
|
2134
|
+
"360doc.com",
|
|
2135
|
+
"book118.com",
|
|
2136
|
+
"jianshu.com",
|
|
2137
|
+
"toutiao.com",
|
|
2138
|
+
),
|
|
2139
|
+
),
|
|
1730
2140
|
)
|
|
2141
|
+
_AUTHORITY_BASE = {
|
|
2142
|
+
"primary": 0.18,
|
|
2143
|
+
"official": 0.18,
|
|
2144
|
+
"academic": 0.16,
|
|
2145
|
+
"media": 0.10,
|
|
2146
|
+
"reference": 0.06,
|
|
2147
|
+
"community": 0.05,
|
|
2148
|
+
"low": -0.16,
|
|
2149
|
+
"unknown": 0.0,
|
|
2150
|
+
}
|
|
2151
|
+
_AUTHORITY_MULTIPLIER = {
|
|
2152
|
+
"technical": {
|
|
2153
|
+
"primary": 1.3,
|
|
2154
|
+
"official": 0.8,
|
|
2155
|
+
"academic": 0.9,
|
|
2156
|
+
"media": 0.4,
|
|
2157
|
+
"reference": 0.6,
|
|
2158
|
+
"community": 0.9,
|
|
2159
|
+
"low": 1.2,
|
|
2160
|
+
"unknown": 1.0,
|
|
2161
|
+
},
|
|
2162
|
+
"news": {
|
|
2163
|
+
"primary": 0.4,
|
|
2164
|
+
"official": 0.9,
|
|
2165
|
+
"academic": 0.4,
|
|
2166
|
+
"media": 1.4,
|
|
2167
|
+
"reference": 0.4,
|
|
2168
|
+
"community": 0.3,
|
|
2169
|
+
"low": 1.0,
|
|
2170
|
+
"unknown": 1.0,
|
|
2171
|
+
},
|
|
2172
|
+
"comparison": {
|
|
2173
|
+
"primary": 0.8,
|
|
2174
|
+
"official": 0.6,
|
|
2175
|
+
"academic": 0.8,
|
|
2176
|
+
"media": 0.5,
|
|
2177
|
+
"reference": 0.6,
|
|
2178
|
+
"community": 1.0,
|
|
2179
|
+
"low": 1.2,
|
|
2180
|
+
"unknown": 1.0,
|
|
2181
|
+
},
|
|
2182
|
+
"policy": {
|
|
2183
|
+
"primary": 0.6,
|
|
2184
|
+
"official": 1.3,
|
|
2185
|
+
"academic": 1.0,
|
|
2186
|
+
"media": 0.8,
|
|
2187
|
+
"reference": 0.7,
|
|
2188
|
+
"community": 0.4,
|
|
2189
|
+
"low": 1.2,
|
|
2190
|
+
"unknown": 1.0,
|
|
2191
|
+
},
|
|
2192
|
+
"howto": {
|
|
2193
|
+
"primary": 1.3,
|
|
2194
|
+
"official": 1.0,
|
|
2195
|
+
"academic": 0.5,
|
|
2196
|
+
"media": 0.4,
|
|
2197
|
+
"reference": 0.6,
|
|
2198
|
+
"community": 1.0,
|
|
2199
|
+
"low": 0.8,
|
|
2200
|
+
"unknown": 1.0,
|
|
2201
|
+
},
|
|
2202
|
+
"factual": {
|
|
2203
|
+
"primary": 1.0,
|
|
2204
|
+
"official": 1.2,
|
|
2205
|
+
"academic": 1.1,
|
|
2206
|
+
"media": 0.9,
|
|
2207
|
+
"reference": 1.1,
|
|
2208
|
+
"community": 0.6,
|
|
2209
|
+
"low": 1.0,
|
|
2210
|
+
"unknown": 1.0,
|
|
2211
|
+
},
|
|
2212
|
+
"research": {
|
|
2213
|
+
"primary": 0.9,
|
|
2214
|
+
"official": 0.7,
|
|
2215
|
+
"academic": 1.4,
|
|
2216
|
+
"media": 0.4,
|
|
2217
|
+
"reference": 0.6,
|
|
2218
|
+
"community": 0.6,
|
|
2219
|
+
"low": 1.2,
|
|
2220
|
+
"unknown": 1.0,
|
|
2221
|
+
},
|
|
2222
|
+
"general": {tier: 1.0 for tier in _AUTHORITY_BASE},
|
|
2223
|
+
}
|
|
2224
|
+
|
|
2225
|
+
|
|
2226
|
+
def authority_for(host: str, intent: str = "general") -> dict[str, Any]:
|
|
2227
|
+
"""Return an explicit source-authority tier and a small ranking bonus."""
|
|
2228
|
+
|
|
2229
|
+
host = (host or "").lower().lstrip(".")
|
|
2230
|
+
tier = "unknown"
|
|
2231
|
+
for name, suffixes in _AUTHORITY_TIERS:
|
|
2232
|
+
if any(host == suffix or host.endswith("." + suffix) for suffix in suffixes):
|
|
2233
|
+
tier = name
|
|
2234
|
+
break
|
|
2235
|
+
multiplier = _AUTHORITY_MULTIPLIER.get(
|
|
2236
|
+
intent, _AUTHORITY_MULTIPLIER["general"]
|
|
2237
|
+
).get(tier, 1.0)
|
|
2238
|
+
score = _AUTHORITY_BASE[tier] * multiplier
|
|
2239
|
+
return {
|
|
2240
|
+
"tier": tier,
|
|
2241
|
+
"score": round(score, 3),
|
|
2242
|
+
"intent": intent,
|
|
2243
|
+
}
|
|
1731
2244
|
|
|
1732
2245
|
|
|
1733
|
-
def
|
|
1734
|
-
|
|
2246
|
+
def result_match(
|
|
2247
|
+
result: dict[str, str], query: str, tokens: Optional[list[str]] = None
|
|
2248
|
+
) -> dict[str, Any]:
|
|
2249
|
+
"""Grade how well one search result answers ``query``.
|
|
1735
2250
|
|
|
1736
|
-
|
|
1737
|
-
|
|
2251
|
+
A title is not a match just because the URL contains one query word. The
|
|
2252
|
+
result must either contain the full phrase or at least two meaningful query
|
|
2253
|
+
terms with half the query covered. A Chinese page that merely repeats one
|
|
2254
|
+
English product name does not pass an English four-word query; it is marked
|
|
2255
|
+
with ``language_match: false`` and ranked below real answers.
|
|
1738
2256
|
"""
|
|
1739
2257
|
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
2258
|
+
tokens = tokens if tokens is not None else _content_tokens(query)
|
|
2259
|
+
haystack = " ".join(
|
|
2260
|
+
(result.get("title", ""), result.get("snippet", ""))
|
|
2261
|
+
).lower()
|
|
2262
|
+
title = (result.get("title") or "").lower()
|
|
2263
|
+
stems = _stems(haystack)
|
|
2264
|
+
title_stems = _stems(title)
|
|
2265
|
+
matched = [token for token in tokens if _token_in_text(token, haystack, stems)]
|
|
2266
|
+
coverage = len(matched) / len(tokens) if tokens else 0.0
|
|
2267
|
+
title_hits = (
|
|
2268
|
+
sum(1 for token in tokens if _token_in_text(token, title, title_stems)) / len(tokens)
|
|
2269
|
+
if tokens
|
|
2270
|
+
else 0.0
|
|
2271
|
+
)
|
|
2272
|
+
phrase = " ".join((query or "").lower().split())
|
|
2273
|
+
exact = bool(phrase and phrase in haystack)
|
|
2274
|
+
title_exact = bool(phrase and title.strip() == phrase)
|
|
2275
|
+
cjk_query = any(_CJK_RE.search(token) for token in tokens)
|
|
2276
|
+
# A CJK query has no word boundaries, so its tokens are character bigrams
|
|
2277
|
+
# and full coverage is the wrong bar: an eight-character question has seven
|
|
2278
|
+
# bigrams, and a correct title often carries only a few of them. Requiring
|
|
2279
|
+
# several *overlapping* pieces is the useful signal there; Latin queries
|
|
2280
|
+
# keep the stricter coverage rule.
|
|
2281
|
+
required = 1 if len(tokens) <= 1 else (3 if cjk_query and len(tokens) >= 8 else 2)
|
|
2282
|
+
if cjk_query:
|
|
2283
|
+
relevant = exact or len(matched) >= required
|
|
2284
|
+
else:
|
|
2285
|
+
relevant = exact or (len(matched) >= required and coverage >= 0.5)
|
|
2286
|
+
|
|
2287
|
+
query_cjk, query_latin = _script_stats(query)
|
|
2288
|
+
result_cjk, result_latin = _script_stats(haystack)
|
|
2289
|
+
language_match = True
|
|
2290
|
+
# Latin query, overwhelmingly Chinese result that does not carry the query
|
|
2291
|
+
# terms. This is the Excel INDIRECT failure mode: one English product name
|
|
2292
|
+
# in a Chinese tutorial is not an answer to an English concept query.
|
|
2293
|
+
if query_latin >= 2 and query_cjk == 0:
|
|
2294
|
+
if (
|
|
2295
|
+
result_cjk >= 8
|
|
2296
|
+
and result_latin < max(2, len(tokens))
|
|
2297
|
+
and coverage < 0.5
|
|
2298
|
+
):
|
|
2299
|
+
language_match = False
|
|
2300
|
+
# A Chinese query answered by an unrelated English page.
|
|
2301
|
+
elif query_cjk >= 2 and query_latin == 0:
|
|
2302
|
+
if result_latin >= 8 and result_cjk < 4 and coverage < 0.5:
|
|
2303
|
+
language_match = False
|
|
2304
|
+
|
|
2305
|
+
return {
|
|
2306
|
+
"coverage": round(coverage, 3),
|
|
2307
|
+
"title_coverage": round(title_hits, 3),
|
|
2308
|
+
"matched_terms": matched,
|
|
2309
|
+
"exact": exact,
|
|
2310
|
+
"title_exact": title_exact,
|
|
2311
|
+
"relevant": relevant,
|
|
2312
|
+
"language_match": language_match,
|
|
2313
|
+
}
|
|
2314
|
+
|
|
2315
|
+
|
|
2316
|
+
def annotate_results(results: list[dict[str, Any]], query: str) -> list[dict[str, Any]]:
|
|
2317
|
+
"""Attach relevance and authority fields in place and return the same list."""
|
|
2318
|
+
|
|
2319
|
+
tokens = _content_tokens(query)
|
|
2320
|
+
intent = classify_intent(query)
|
|
2321
|
+
for item in results:
|
|
2322
|
+
details = result_match(item, query, tokens)
|
|
2323
|
+
host = urllib.parse.urlsplit(item.get("url", "")).netloc
|
|
2324
|
+
authority = authority_for(host, intent["intent"])
|
|
2325
|
+
item.update(
|
|
2326
|
+
coverage=details["coverage"],
|
|
2327
|
+
matched_terms=details["matched_terms"],
|
|
2328
|
+
relevance=details["relevant"],
|
|
2329
|
+
language_match=details["language_match"],
|
|
2330
|
+
authority_tier=authority["tier"],
|
|
2331
|
+
authority_score=authority["score"],
|
|
2332
|
+
intent=intent["intent"],
|
|
2333
|
+
)
|
|
2334
|
+
return results
|
|
1748
2335
|
|
|
1749
2336
|
|
|
1750
2337
|
def _freshness_bonus(text: str) -> float:
|
|
@@ -1835,42 +2422,141 @@ def diversify_results(
|
|
|
1835
2422
|
return (chosen + overflow)[:limit]
|
|
1836
2423
|
|
|
1837
2424
|
|
|
2425
|
+
_SEARCH_ENGINE_HOSTS = {
|
|
2426
|
+
"sogou.com",
|
|
2427
|
+
"www.sogou.com",
|
|
2428
|
+
"m.sogou.com",
|
|
2429
|
+
"weixin.sogou.com",
|
|
2430
|
+
"m.weixin.sogou.com",
|
|
2431
|
+
"so.com",
|
|
2432
|
+
"www.so.com",
|
|
2433
|
+
"m.so.com",
|
|
2434
|
+
"bing.com",
|
|
2435
|
+
"www.bing.com",
|
|
2436
|
+
"cn.bing.com",
|
|
2437
|
+
"baidu.com",
|
|
2438
|
+
"www.baidu.com",
|
|
2439
|
+
"m.baidu.com",
|
|
2440
|
+
"google.com",
|
|
2441
|
+
"www.google.com",
|
|
2442
|
+
"duckduckgo.com",
|
|
2443
|
+
"html.duckduckgo.com",
|
|
2444
|
+
"yandex.com",
|
|
2445
|
+
"yandex.ru",
|
|
2446
|
+
}
|
|
2447
|
+
|
|
2448
|
+
|
|
2449
|
+
def _is_search_engine_page(url: str) -> bool:
|
|
2450
|
+
"""True for a search engine's own result page, which is not an answer.
|
|
2451
|
+
|
|
2452
|
+
One backend occasionally echoes a Sogou/360 results URL into its own list.
|
|
2453
|
+
Ranking it first would cite the search box instead of the source, so these
|
|
2454
|
+
intermediate pages are dropped before scoring. Article hosts that merely
|
|
2455
|
+
live under a portal (``baijiahao.baidu.com``) are not in this set.
|
|
2456
|
+
"""
|
|
2457
|
+
|
|
2458
|
+
try:
|
|
2459
|
+
parts = urllib.parse.urlsplit(str(url or "").strip())
|
|
2460
|
+
except ValueError:
|
|
2461
|
+
return False
|
|
2462
|
+
host = (parts.hostname or "").lower().rstrip(".")
|
|
2463
|
+
if host not in _SEARCH_ENGINE_HOSTS:
|
|
2464
|
+
return False
|
|
2465
|
+
path = (parts.path or "").lower()
|
|
2466
|
+
query = (parts.query or "").lower()
|
|
2467
|
+
if any(
|
|
2468
|
+
marker in query
|
|
2469
|
+
for marker in ("query=", "q=", "wd=", "keyword=", "word=")
|
|
2470
|
+
):
|
|
2471
|
+
return True
|
|
2472
|
+
return path.startswith(("/sogou", "/search", "/web", "/s", "/antispider"))
|
|
2473
|
+
|
|
2474
|
+
|
|
1838
2475
|
def merge_search_results(
|
|
1839
|
-
groups: list[tuple[str, list[dict[str,
|
|
2476
|
+
groups: list[tuple[str, list[dict[str, Any]]]],
|
|
1840
2477
|
query: str,
|
|
1841
2478
|
limit: int = 8,
|
|
1842
|
-
) -> list[dict[str,
|
|
1843
|
-
"""Merge several engines by coverage, title match,
|
|
2479
|
+
) -> list[dict[str, Any]]:
|
|
2480
|
+
"""Merge several engines by coverage, title match, authority and freshness.
|
|
1844
2481
|
|
|
1845
2482
|
Engine order only breaks exact ties, so two engines that return the same
|
|
1846
2483
|
page still produce one stable ordering.
|
|
1847
2484
|
"""
|
|
1848
2485
|
|
|
1849
|
-
tokens =
|
|
1850
|
-
|
|
1851
|
-
|
|
2486
|
+
tokens = _content_tokens(query)
|
|
2487
|
+
intent = classify_intent(query)
|
|
2488
|
+
rrf_by_url: dict[str, float] = {}
|
|
2489
|
+
for _, items in groups:
|
|
2490
|
+
for rank, item in enumerate(items):
|
|
2491
|
+
key = _canonical_url(item.get("url", ""))
|
|
2492
|
+
if key:
|
|
2493
|
+
rrf_by_url[key] = rrf_by_url.get(key, 0.0) + 1.0 / (60 + rank + 1)
|
|
2494
|
+
|
|
2495
|
+
scored: list[tuple[float, int, int, dict[str, Any]]] = []
|
|
1852
2496
|
for group_index, (name, items) in enumerate(groups):
|
|
1853
2497
|
for rank, item in enumerate(items):
|
|
1854
|
-
|
|
2498
|
+
if _is_search_engine_page(item.get("url", "")):
|
|
2499
|
+
continue
|
|
2500
|
+
candidate: dict[str, Any] = dict(item)
|
|
1855
2501
|
candidate.setdefault("source", name)
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
sum(1 for token in tokens if token in title) / len(tokens)
|
|
1860
|
-
if tokens
|
|
1861
|
-
else 0.0
|
|
1862
|
-
)
|
|
1863
|
-
exact = 0.1 if phrase and phrase in title else 0.0
|
|
2502
|
+
details = result_match(candidate, query, tokens)
|
|
2503
|
+
key = _canonical_url(candidate.get("url", ""))
|
|
2504
|
+
rrf = rrf_by_url.get(key, 0.0)
|
|
1864
2505
|
host = urllib.parse.urlsplit(candidate.get("url", "")).netloc
|
|
2506
|
+
authority = authority_for(host, intent["intent"])
|
|
2507
|
+
freshness = _freshness_bonus(
|
|
2508
|
+
f"{candidate.get('title', '')} {candidate.get('snippet', '')}"
|
|
2509
|
+
)
|
|
2510
|
+
components = {
|
|
2511
|
+
"coverage": round(details["coverage"], 3),
|
|
2512
|
+
"title": round(details["title_coverage"], 3),
|
|
2513
|
+
"exact": bool(details["exact"]),
|
|
2514
|
+
"title_exact": bool(details["title_exact"]),
|
|
2515
|
+
"relevant": bool(details["relevant"]),
|
|
2516
|
+
"rrf": round(rrf, 4),
|
|
2517
|
+
"authority": round(float(authority["score"]), 3),
|
|
2518
|
+
"freshness": round(freshness, 3),
|
|
2519
|
+
"language_penalty": 0.0 if details["language_match"] else -0.35,
|
|
2520
|
+
}
|
|
2521
|
+
candidate.update(
|
|
2522
|
+
coverage=details["coverage"],
|
|
2523
|
+
matched_terms=details["matched_terms"],
|
|
2524
|
+
relevance=details["relevant"],
|
|
2525
|
+
language_match=details["language_match"],
|
|
2526
|
+
rrf=round(rrf, 4),
|
|
2527
|
+
authority_tier=authority["tier"],
|
|
2528
|
+
authority_score=authority["score"],
|
|
2529
|
+
intent=intent["intent"],
|
|
2530
|
+
rank_components=components,
|
|
2531
|
+
rank_reason=[
|
|
2532
|
+
name
|
|
2533
|
+
for name in (
|
|
2534
|
+
"coverage",
|
|
2535
|
+
"title",
|
|
2536
|
+
"rrf",
|
|
2537
|
+
"exact",
|
|
2538
|
+
"title_exact",
|
|
2539
|
+
"relevant",
|
|
2540
|
+
"authority",
|
|
2541
|
+
"freshness",
|
|
2542
|
+
"language_penalty",
|
|
2543
|
+
)
|
|
2544
|
+
if components[name]
|
|
2545
|
+
],
|
|
2546
|
+
)
|
|
1865
2547
|
score = (
|
|
1866
|
-
coverage * 0.
|
|
1867
|
-
+
|
|
1868
|
-
+ exact
|
|
1869
|
-
+
|
|
1870
|
-
+
|
|
1871
|
-
|
|
1872
|
-
|
|
2548
|
+
components["coverage"] * 0.5
|
|
2549
|
+
+ components["title"] * 0.15
|
|
2550
|
+
+ (0.15 if components["exact"] else 0.0)
|
|
2551
|
+
+ (0.05 if components["title_exact"] else 0.0)
|
|
2552
|
+
+ (0.2 if components["relevant"] else 0.0)
|
|
2553
|
+
+ min(0.12, components["rrf"] * 3.0)
|
|
2554
|
+
+ components["authority"]
|
|
2555
|
+
+ components["freshness"]
|
|
2556
|
+
+ components["language_penalty"]
|
|
1873
2557
|
)
|
|
2558
|
+
if not components["relevant"]:
|
|
2559
|
+
score -= 0.15
|
|
1874
2560
|
scored.append((score, group_index, rank, candidate))
|
|
1875
2561
|
scored.sort(key=lambda entry: (-entry[0], entry[1], entry[2]))
|
|
1876
2562
|
return [entry[3] for entry in scored[: max(limit * 3, limit)]]
|
|
@@ -2050,6 +2736,63 @@ def resolve_sogou_results(
|
|
|
2050
2736
|
return resolved
|
|
2051
2737
|
|
|
2052
2738
|
|
|
2739
|
+
_TRACKING_QUERY = re.compile(
|
|
2740
|
+
r"^(?:utm_|gclid$|fbclid$|yclid$|spm$|ref$|source$|share_token$|"
|
|
2741
|
+
r"from$|mc_cid$|mc_eid$|igshid$)",
|
|
2742
|
+
re.I,
|
|
2743
|
+
)
|
|
2744
|
+
|
|
2745
|
+
|
|
2746
|
+
def _canonical_url(url: str) -> str:
|
|
2747
|
+
"""A dedupe key for one URL, without tracking noise.
|
|
2748
|
+
|
|
2749
|
+
Search engines return the same article as ``/a``, ``/a/``, ``/a#top`` and
|
|
2750
|
+
``/a?utm_source=...``. Keeping the display URL untouched is important;
|
|
2751
|
+
this key only decides whether two rows are the same result.
|
|
2752
|
+
"""
|
|
2753
|
+
|
|
2754
|
+
raw = str(url or "").strip()
|
|
2755
|
+
if not raw:
|
|
2756
|
+
return ""
|
|
2757
|
+
try:
|
|
2758
|
+
parts = urllib.parse.urlsplit(raw)
|
|
2759
|
+
port = parts.port
|
|
2760
|
+
except ValueError:
|
|
2761
|
+
return raw
|
|
2762
|
+
if not parts.netloc:
|
|
2763
|
+
return raw
|
|
2764
|
+
host = (parts.hostname or "").lower().rstrip(".")
|
|
2765
|
+
if not host:
|
|
2766
|
+
return raw
|
|
2767
|
+
if host.startswith("www."):
|
|
2768
|
+
host = host[4:]
|
|
2769
|
+
if ":" in host and not host.startswith("["):
|
|
2770
|
+
host = f"[{host}]"
|
|
2771
|
+
if port and not (
|
|
2772
|
+
(parts.scheme.lower() == "http" and port == 80)
|
|
2773
|
+
or (parts.scheme.lower() == "https" and port == 443)
|
|
2774
|
+
):
|
|
2775
|
+
host = f"{host}:{port}"
|
|
2776
|
+
path = re.sub(r"/+", "/", parts.path or "/")
|
|
2777
|
+
if len(path) > 1:
|
|
2778
|
+
path = path.rstrip("/")
|
|
2779
|
+
if path.endswith("/amp"):
|
|
2780
|
+
path = path[:-4] or "/"
|
|
2781
|
+
query = urllib.parse.parse_qsl(parts.query, keep_blank_values=True)
|
|
2782
|
+
query = sorted(
|
|
2783
|
+
(key, value) for key, value in query if not _TRACKING_QUERY.match(key)
|
|
2784
|
+
)
|
|
2785
|
+
return urllib.parse.urlunsplit(
|
|
2786
|
+
(
|
|
2787
|
+
(parts.scheme or "https").lower(),
|
|
2788
|
+
host,
|
|
2789
|
+
path,
|
|
2790
|
+
urllib.parse.urlencode(query, doseq=True),
|
|
2791
|
+
"",
|
|
2792
|
+
)
|
|
2793
|
+
)
|
|
2794
|
+
|
|
2795
|
+
|
|
2053
2796
|
def dedupe_results(results: Iterable[dict[str, str]], limit: int = 8) -> list[dict[str, str]]:
|
|
2054
2797
|
"""Remove duplicate URLs and unusable titles while preserving rank order."""
|
|
2055
2798
|
|
|
@@ -2057,7 +2800,8 @@ def dedupe_results(results: Iterable[dict[str, str]], limit: int = 8) -> list[di
|
|
|
2057
2800
|
unique: list[dict[str, str]] = []
|
|
2058
2801
|
for item in results:
|
|
2059
2802
|
url = item.get("url", "")
|
|
2060
|
-
|
|
2803
|
+
key = _canonical_url(url)
|
|
2804
|
+
if not url or key in seen:
|
|
2061
2805
|
continue
|
|
2062
2806
|
title = str(item.get("title") or "")
|
|
2063
2807
|
# Engines sometimes answer a query with an anti-bot interstitial and
|
|
@@ -2066,7 +2810,7 @@ def dedupe_results(results: Iterable[dict[str, str]], limit: int = 8) -> list[di
|
|
|
2066
2810
|
# dedupe_results bare URLs.
|
|
2067
2811
|
if title and (looks_garbled(title) or not _USEFUL_TITLE_RE.search(title)):
|
|
2068
2812
|
continue
|
|
2069
|
-
seen.add(
|
|
2813
|
+
seen.add(key)
|
|
2070
2814
|
unique.append(item)
|
|
2071
2815
|
if len(unique) >= limit:
|
|
2072
2816
|
break
|
|
@@ -2955,6 +3699,32 @@ def render_html(url: str, config: FetchConfig) -> str:
|
|
|
2955
3699
|
browser = playwright.chromium.launch(headless=True)
|
|
2956
3700
|
try:
|
|
2957
3701
|
context = browser.new_context(user_agent=config.user_agent)
|
|
3702
|
+
|
|
3703
|
+
def guard_request(route: Any) -> None:
|
|
3704
|
+
"""Block private/file subresources, not just the document.
|
|
3705
|
+
|
|
3706
|
+
``page.goto`` checks the requested and final document URL,
|
|
3707
|
+
but a hostile page can still ask Chromium to load an image,
|
|
3708
|
+
script or fetch() from 127.0.0.1, the LAN or a metadata
|
|
3709
|
+
endpoint. The browser-mode bridge already installs this
|
|
3710
|
+
filter; the Python render path needs the same boundary.
|
|
3711
|
+
"""
|
|
3712
|
+
|
|
3713
|
+
request_url = route.request.url
|
|
3714
|
+
if request_url.startswith(("file:", "filesystem:")):
|
|
3715
|
+
route.abort()
|
|
3716
|
+
return
|
|
3717
|
+
if request_url.startswith(("http://", "https://")):
|
|
3718
|
+
try:
|
|
3719
|
+
assert_public_url(
|
|
3720
|
+
request_url, allow_private=config.allow_private
|
|
3721
|
+
)
|
|
3722
|
+
except UnsafeAddress:
|
|
3723
|
+
route.abort()
|
|
3724
|
+
return
|
|
3725
|
+
route.continue_()
|
|
3726
|
+
|
|
3727
|
+
context.route("**/*", guard_request)
|
|
2958
3728
|
page = context.new_page()
|
|
2959
3729
|
page.set_default_timeout(timeout_ms)
|
|
2960
3730
|
page.goto(url, wait_until="domcontentloaded", timeout=timeout_ms)
|
|
@@ -3096,13 +3866,15 @@ def load_session(name: str) -> dict[str, Any]:
|
|
|
3096
3866
|
|
|
3097
3867
|
def save_session(name: str, data: dict[str, Any]) -> Path:
|
|
3098
3868
|
path = session_path(name)
|
|
3099
|
-
path.parent.mkdir(parents=True, exist_ok=True)
|
|
3869
|
+
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
3870
|
+
_private_path(path.parent, 0o700)
|
|
3100
3871
|
payload = {
|
|
3101
3872
|
"visited": list(data.get("visited", []))[-1000:],
|
|
3102
3873
|
"trail": list(data.get("trail", []))[-500:],
|
|
3103
3874
|
"updated_at": utc_now(),
|
|
3104
3875
|
}
|
|
3105
3876
|
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
3877
|
+
_private_path(path, 0o600)
|
|
3106
3878
|
return path
|
|
3107
3879
|
|
|
3108
3880
|
|
|
@@ -3295,6 +4067,7 @@ def fetch_page(
|
|
|
3295
4067
|
warnings: list[str] = []
|
|
3296
4068
|
|
|
3297
4069
|
try:
|
|
4070
|
+
response: Optional[HttpResponse] = None
|
|
3298
4071
|
policy = fetcher.assert_allowed(url)
|
|
3299
4072
|
robots_checked = bool(config.respect_robots and config.robots_policy != "ignore")
|
|
3300
4073
|
robots_overridden = bool(robots_checked and not policy.allowed(url))
|
|
@@ -3305,7 +4078,6 @@ def fetch_page(
|
|
|
3305
4078
|
)
|
|
3306
4079
|
extra_delay = policy.crawl_delay if policy and policy.crawl_delay else 0.0
|
|
3307
4080
|
|
|
3308
|
-
response: Optional[HttpResponse] = None
|
|
3309
4081
|
rendered = False
|
|
3310
4082
|
render_fallback = False
|
|
3311
4083
|
html = ""
|
|
@@ -3316,6 +4088,12 @@ def fetch_page(
|
|
|
3316
4088
|
rendered = True
|
|
3317
4089
|
else:
|
|
3318
4090
|
response = fetcher.fetch(url)
|
|
4091
|
+
if getattr(response, "robots_overridden", False) and not robots_overridden:
|
|
4092
|
+
robots_overridden = True
|
|
4093
|
+
warnings.append(
|
|
4094
|
+
"a redirect target was read because robots policy is 'warn'; "
|
|
4095
|
+
"this override is recorded as compliance.robots_overridden."
|
|
4096
|
+
)
|
|
3319
4097
|
html = response.text
|
|
3320
4098
|
if config.render == "auto" and looks_dynamic(html):
|
|
3321
4099
|
try:
|
|
@@ -3323,7 +4101,7 @@ def fetch_page(
|
|
|
3323
4101
|
if rendered_html and len(rendered_html) > len(html):
|
|
3324
4102
|
html = rendered_html
|
|
3325
4103
|
rendered = True
|
|
3326
|
-
log(f"rendered dynamically: {url}", config)
|
|
4104
|
+
log(f"rendered dynamically: {redact_url_for_log(url)}", config)
|
|
3327
4105
|
except RenderUnavailable as exc:
|
|
3328
4106
|
warnings.append(f"dynamic rendering skipped: {exc}")
|
|
3329
4107
|
except Exception as exc: # pragma: no cover - defensive
|
|
@@ -3461,26 +4239,30 @@ def fetch_page(
|
|
|
3461
4239
|
"the page may be empty, paywalled, or require JavaScript rendering"
|
|
3462
4240
|
)
|
|
3463
4241
|
|
|
3464
|
-
if config.max_chars and len(markdown) > config.max_chars:
|
|
3465
|
-
markdown = markdown[: config.max_chars].rstrip() + "\n\n<!-- truncated by web-forager -->"
|
|
3466
|
-
warnings.append(f"markdown truncated to {config.max_chars} characters")
|
|
3467
|
-
|
|
3468
4242
|
# Everything below came from a third party. Say so in the record, and
|
|
3469
4243
|
# name instruction-shaped passages instead of letting them read as
|
|
3470
|
-
# part of the surrounding conversation.
|
|
4244
|
+
# part of the surrounding conversation. Scan before max_chars
|
|
4245
|
+
# truncation: a hostile instruction near the tail of a long page must
|
|
4246
|
+
# not disappear just because the caller asked for a shorter body.
|
|
3471
4247
|
injection_flags: list[str] = []
|
|
4248
|
+
injection_findings: list[dict[str, Any]] = []
|
|
3472
4249
|
trust = "untrusted"
|
|
3473
4250
|
if config.input_gate != "off":
|
|
3474
|
-
|
|
4251
|
+
injection_findings = untrusted.scan_findings(
|
|
3475
4252
|
markdown + "\n" + str(metadata.get("title", ""))
|
|
3476
4253
|
)
|
|
3477
|
-
|
|
3478
|
-
|
|
3479
|
-
|
|
3480
|
-
|
|
3481
|
-
|
|
3482
|
-
|
|
3483
|
-
|
|
4254
|
+
injection_flags = untrusted.codes(injection_findings)
|
|
4255
|
+
hard = untrusted.hard_codes(injection_findings)
|
|
4256
|
+
if hard and config.input_gate == "strict":
|
|
4257
|
+
markdown = ""
|
|
4258
|
+
trust = "rejected"
|
|
4259
|
+
warnings.insert(0, untrusted.withheld(hard, injection_findings))
|
|
4260
|
+
elif injection_findings:
|
|
4261
|
+
warnings.insert(0, untrusted.warning(injection_flags, injection_findings))
|
|
4262
|
+
|
|
4263
|
+
if config.max_chars and len(markdown) > config.max_chars:
|
|
4264
|
+
markdown = markdown[: config.max_chars].rstrip() + "\n\n<!-- truncated by web-forager -->"
|
|
4265
|
+
warnings.append(f"markdown truncated to {config.max_chars} characters")
|
|
3484
4266
|
|
|
3485
4267
|
links = []
|
|
3486
4268
|
if config.extract_links and not is_pdf_payload and not document_kind and not feed_format:
|
|
@@ -3514,6 +4296,7 @@ def fetch_page(
|
|
|
3514
4296
|
"warnings": warnings,
|
|
3515
4297
|
"trust": trust,
|
|
3516
4298
|
"injection_flags": injection_flags,
|
|
4299
|
+
"injection_findings": injection_findings,
|
|
3517
4300
|
"compliance": {
|
|
3518
4301
|
"robots_checked": robots_checked,
|
|
3519
4302
|
"robots_policy": (
|
|
@@ -3538,26 +4321,52 @@ def search_warnings(
|
|
|
3538
4321
|
"""
|
|
3539
4322
|
|
|
3540
4323
|
warnings: list[str] = []
|
|
3541
|
-
tokens =
|
|
4324
|
+
tokens = _content_tokens(query)
|
|
4325
|
+
if results:
|
|
4326
|
+
annotate_results(results, query)
|
|
3542
4327
|
if not results:
|
|
3543
4328
|
warnings.append(
|
|
3544
4329
|
f"no results: {engine} returned nothing for {query!r}. Try another "
|
|
3545
4330
|
"engine (--engine all), a shorter query, or a direct URL."
|
|
3546
4331
|
)
|
|
3547
|
-
elif tokens and
|
|
3548
|
-
|
|
3549
|
-
|
|
3550
|
-
|
|
3551
|
-
|
|
3552
|
-
|
|
3553
|
-
|
|
4332
|
+
elif tokens and not any(item.get("relevance") for item in results):
|
|
4333
|
+
if max(result_coverage(item, tokens) for item in results) <= 0:
|
|
4334
|
+
warnings.append(
|
|
4335
|
+
"none of the returned results mention the query text; the engine "
|
|
4336
|
+
"most likely split the query into single characters (common with "
|
|
4337
|
+
"quoted or CJK names). Treat these results as unrelated and try "
|
|
4338
|
+
"another engine or a more specific query."
|
|
4339
|
+
)
|
|
4340
|
+
else:
|
|
4341
|
+
warnings.append(
|
|
4342
|
+
"none of the returned results pass the relevance check: each needs "
|
|
4343
|
+
"at least two meaningful query terms, the exact phrase, or an "
|
|
4344
|
+
"equivalent match. Treat these results as unrelated and try another "
|
|
4345
|
+
"engine or a shorter, more specific query."
|
|
4346
|
+
)
|
|
4347
|
+
else:
|
|
4348
|
+
off_topic = [item for item in results if item.get("relevance") is False]
|
|
4349
|
+
if off_topic:
|
|
4350
|
+
warnings.append(
|
|
4351
|
+
f"{len(off_topic)} of {len(results)} results look off-topic: they "
|
|
4352
|
+
"match only one query term or come from the wrong language. They "
|
|
4353
|
+
"are kept at the bottom of the list; check relevance before using "
|
|
4354
|
+
"them as findings."
|
|
4355
|
+
)
|
|
3554
4356
|
if results:
|
|
3555
|
-
|
|
3556
|
-
|
|
3557
|
-
|
|
3558
|
-
|
|
3559
|
-
|
|
3560
|
-
|
|
4357
|
+
probe_findings: list[dict[str, Any]] = []
|
|
4358
|
+
for item in results:
|
|
4359
|
+
findings = untrusted.scan_findings(
|
|
4360
|
+
f"{item.get('title', '')} {item.get('snippet', '')}"
|
|
4361
|
+
)
|
|
4362
|
+
if findings:
|
|
4363
|
+
item["injection_flags"] = untrusted.codes(findings)
|
|
4364
|
+
item["injection_findings"] = findings
|
|
4365
|
+
probe_findings.extend(findings)
|
|
4366
|
+
if probe_findings:
|
|
4367
|
+
warnings.append(
|
|
4368
|
+
untrusted.warning(untrusted.codes(probe_findings), probe_findings)
|
|
4369
|
+
)
|
|
3561
4370
|
if looks_like_person_name(query):
|
|
3562
4371
|
warnings.append(
|
|
3563
4372
|
"the query is a bare personal name. Searching a paper author or a "
|
|
@@ -3567,12 +4376,86 @@ def search_warnings(
|
|
|
3567
4376
|
return warnings
|
|
3568
4377
|
|
|
3569
4378
|
|
|
4379
|
+
def search_quality(
|
|
4380
|
+
query: str,
|
|
4381
|
+
results: list[dict[str, Any]],
|
|
4382
|
+
engine: str = "auto",
|
|
4383
|
+
engine_report: Optional[list[dict[str, Any]]] = None,
|
|
4384
|
+
) -> dict[str, Any]:
|
|
4385
|
+
"""A small, machine-readable summary of how good a result list is.
|
|
4386
|
+
|
|
4387
|
+
The prose warnings remain for humans; this block lets an agent decide
|
|
4388
|
+
whether to trust the list without parsing English. Counts describe what
|
|
4389
|
+
was returned, not what the engine could have returned.
|
|
4390
|
+
"""
|
|
4391
|
+
|
|
4392
|
+
annotate_results(results, query)
|
|
4393
|
+
intent = classify_intent(query)
|
|
4394
|
+
relevant = sum(1 for item in results if item.get("relevance"))
|
|
4395
|
+
off_topic = sum(1 for item in results if item.get("relevance") is False)
|
|
4396
|
+
language_mismatch = sum(
|
|
4397
|
+
1 for item in results if item.get("language_match") is False
|
|
4398
|
+
)
|
|
4399
|
+
tiers: dict[str, int] = {}
|
|
4400
|
+
for item in results:
|
|
4401
|
+
tier = str(item.get("authority_tier") or "unknown")
|
|
4402
|
+
tiers[tier] = tiers.get(tier, 0) + 1
|
|
4403
|
+
answered = [
|
|
4404
|
+
str(entry.get("engine", ""))
|
|
4405
|
+
for entry in (engine_report or [])
|
|
4406
|
+
if entry.get("ok") and entry.get("engine")
|
|
4407
|
+
]
|
|
4408
|
+
errors = [
|
|
4409
|
+
{"engine": entry.get("engine", ""), "error": entry.get("error", "")}
|
|
4410
|
+
for entry in (engine_report or [])
|
|
4411
|
+
if not entry.get("ok")
|
|
4412
|
+
]
|
|
4413
|
+
engine_status = {
|
|
4414
|
+
str(entry.get("engine", "")): str(
|
|
4415
|
+
entry.get("status") or ("ok" if entry.get("ok") else "error")
|
|
4416
|
+
)
|
|
4417
|
+
for entry in (engine_report or [])
|
|
4418
|
+
if entry.get("engine")
|
|
4419
|
+
}
|
|
4420
|
+
engine_states: dict[str, int] = {}
|
|
4421
|
+
for state in engine_status.values():
|
|
4422
|
+
engine_states[state] = engine_states.get(state, 0) + 1
|
|
4423
|
+
return {
|
|
4424
|
+
"query": query,
|
|
4425
|
+
"engine": engine,
|
|
4426
|
+
"intent": intent["intent"],
|
|
4427
|
+
"intent_confidence": intent["confidence"],
|
|
4428
|
+
"returned": len(results),
|
|
4429
|
+
"relevant": relevant,
|
|
4430
|
+
"off_topic": off_topic,
|
|
4431
|
+
"language_mismatch": language_mismatch,
|
|
4432
|
+
"authority_tiers": tiers,
|
|
4433
|
+
"high_authority": sum(
|
|
4434
|
+
tiers.get(tier, 0)
|
|
4435
|
+
for tier in ("primary", "official", "academic")
|
|
4436
|
+
),
|
|
4437
|
+
"engines_answered": answered,
|
|
4438
|
+
"engine_errors": errors,
|
|
4439
|
+
"engine_status": engine_status,
|
|
4440
|
+
"engine_states": engine_states,
|
|
4441
|
+
}
|
|
4442
|
+
|
|
4443
|
+
|
|
3570
4444
|
SITE_SEARCH_TEMPLATES = (
|
|
3571
4445
|
"/?s={query}",
|
|
3572
4446
|
"/search?q={query}",
|
|
3573
4447
|
"/?search={query}",
|
|
3574
4448
|
"/search?query={query}",
|
|
3575
4449
|
)
|
|
4450
|
+
# Hosts whose own search page is not one of the generic WordPress/Sphinx
|
|
4451
|
+
# shapes above. These are public search interfaces, not scraped result pages;
|
|
4452
|
+
# the links they return are still fetched through the normal robots-aware path.
|
|
4453
|
+
_SITE_SEARCH_URLS = {
|
|
4454
|
+
"support.microsoft.com": (
|
|
4455
|
+
"https://support.microsoft.com/en-us/search/results?query={query}"
|
|
4456
|
+
),
|
|
4457
|
+
"developer.mozilla.org": "https://developer.mozilla.org/en-US/search?q={query}",
|
|
4458
|
+
}
|
|
3576
4459
|
|
|
3577
4460
|
|
|
3578
4461
|
def find_within_site(
|
|
@@ -3613,11 +4496,19 @@ def find_within_site(
|
|
|
3613
4496
|
attempts: list[dict[str, Any]] = []
|
|
3614
4497
|
page: Optional[HttpResponse] = None
|
|
3615
4498
|
search_url = ""
|
|
4499
|
+
host = (urllib.parse.urlsplit(site_url).hostname or "").lower()
|
|
4500
|
+
preferred = _SITE_SEARCH_URLS.get(host) or _SITE_SEARCH_URLS.get(
|
|
4501
|
+
host.removeprefix("www.")
|
|
4502
|
+
)
|
|
4503
|
+
templates = ((preferred,) if preferred else ()) + SITE_SEARCH_TEMPLATES
|
|
3616
4504
|
try:
|
|
3617
|
-
for template in
|
|
3618
|
-
|
|
3619
|
-
query=urllib.parse.quote_plus(query)
|
|
3620
|
-
|
|
4505
|
+
for template in templates:
|
|
4506
|
+
if template.startswith(("http://", "https://")):
|
|
4507
|
+
candidate = template.format(query=urllib.parse.quote_plus(query))
|
|
4508
|
+
else:
|
|
4509
|
+
candidate = site_url.rstrip("/") + template.format(
|
|
4510
|
+
query=urllib.parse.quote_plus(query)
|
|
4511
|
+
)
|
|
3621
4512
|
started = time.monotonic()
|
|
3622
4513
|
try:
|
|
3623
4514
|
response = fetcher.fetch(candidate)
|
|
@@ -3830,12 +4721,14 @@ def fetch_by_query(
|
|
|
3830
4721
|
|
|
3831
4722
|
try:
|
|
3832
4723
|
engine_report: list[dict[str, Any]] = []
|
|
4724
|
+
quality_data: dict[str, Any] = {}
|
|
3833
4725
|
results = search(
|
|
3834
4726
|
query,
|
|
3835
4727
|
config,
|
|
3836
4728
|
engine=engine,
|
|
3837
4729
|
limit=config.num_results,
|
|
3838
4730
|
report=engine_report,
|
|
4731
|
+
quality=quality_data,
|
|
3839
4732
|
)
|
|
3840
4733
|
pages: list[dict[str, Any]] = []
|
|
3841
4734
|
skipped: list[dict[str, str]] = []
|
|
@@ -3889,11 +4782,14 @@ def fetch_by_query(
|
|
|
3889
4782
|
skipped.append(
|
|
3890
4783
|
{"url": result["url"], "code": "unexpected", "reason": str(exc)}
|
|
3891
4784
|
)
|
|
4785
|
+
quality = search_quality(query, results, engine, engine_report)
|
|
4786
|
+
quality.update(quality_data)
|
|
3892
4787
|
payload: dict[str, Any] = {
|
|
3893
4788
|
"query": query,
|
|
3894
4789
|
"engine": engine,
|
|
3895
4790
|
"mode": "search",
|
|
3896
4791
|
"search_results": results,
|
|
4792
|
+
"search_quality": quality,
|
|
3897
4793
|
"engine_report": engine_report,
|
|
3898
4794
|
"pages": pages,
|
|
3899
4795
|
"skipped": skipped,
|
|
@@ -4003,6 +4899,31 @@ def redact_url(value: str) -> str:
|
|
|
4003
4899
|
return f"{scheme}://{host}{port}" if host else ""
|
|
4004
4900
|
|
|
4005
4901
|
|
|
4902
|
+
def redact_url_for_log(value: str) -> str:
|
|
4903
|
+
"""A URL safe to put in a diagnostic line.
|
|
4904
|
+
|
|
4905
|
+
Query strings carry search terms, signed tokens and session identifiers,
|
|
4906
|
+
and fragments can carry the same. Verbose logging therefore keeps the
|
|
4907
|
+
scheme, host and path only; the caller already has the full URL in the
|
|
4908
|
+
result when it needs it.
|
|
4909
|
+
"""
|
|
4910
|
+
|
|
4911
|
+
text = str(value or "").strip()
|
|
4912
|
+
if not text:
|
|
4913
|
+
return ""
|
|
4914
|
+
if "://" not in text:
|
|
4915
|
+
text = "http://" + text
|
|
4916
|
+
try:
|
|
4917
|
+
parts = urllib.parse.urlsplit(text)
|
|
4918
|
+
except ValueError:
|
|
4919
|
+
return ""
|
|
4920
|
+
if not parts.scheme or not parts.netloc:
|
|
4921
|
+
return ""
|
|
4922
|
+
return urllib.parse.urlunsplit(
|
|
4923
|
+
(parts.scheme, parts.netloc, parts.path or "/", "", "")
|
|
4924
|
+
)
|
|
4925
|
+
|
|
4926
|
+
|
|
4006
4927
|
def doctor_report(config: Optional[FetchConfig] = None) -> dict[str, Any]:
|
|
4007
4928
|
"""Report which capabilities this interpreter can serve, with install hints.
|
|
4008
4929
|
|