web-forager 1.3.0 → 1.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/.claude-plugin/plugin.json +1 -1
  3. package/.clinerules/web-forager.md +3 -3
  4. package/.codebuddy-plugin/marketplace.json +1 -1
  5. package/.codebuddy-plugin/plugin.json +1 -1
  6. package/.codex-plugin/marketplace.json +1 -1
  7. package/.codex-plugin/mcp.json +20 -20
  8. package/.codex-plugin/plugin.json +1 -1
  9. package/.continue/rules/web-forager.md +4 -4
  10. package/.cursor/rules/web-forager.mdc +4 -4
  11. package/.dsh-plugin/index.js +1 -1
  12. package/.dsh-plugin/plugin.json +1 -1
  13. package/.github/copilot-instructions.md +3 -3
  14. package/.roo/rules/web-forager.md +3 -3
  15. package/.windsurf/rules/web-forager.md +4 -4
  16. package/AGENTS.md +10 -3
  17. package/CHANGELOG.md +508 -454
  18. package/README.md +499 -477
  19. package/README_zh.md +498 -477
  20. package/SKILL.md +115 -15
  21. package/THIRD_PARTY_LICENSES.txt +5 -0
  22. package/browser/bridges/playwright-bridge.js +8 -1
  23. package/browser/launch_chrome.py +193 -144
  24. package/browser/lib/bili-api.js +12 -1
  25. package/browser/lib/safety.js +27 -1
  26. package/browser/lib/session.js +16 -2
  27. package/browser/lib/untrusted.js +215 -56
  28. package/browser/mcp_server.js +5 -25
  29. package/browser/package.json +1 -1
  30. package/browser/playbooks/search-dig.js +3 -1
  31. package/browser/playbooks/wechat-dig.js +78 -78
  32. package/browser/run.js +19 -10
  33. package/commands/research.md +35 -0
  34. package/cordis.patch.yml +19 -19
  35. package/docs/DEPENDENCIES.md +1 -1
  36. package/docs/INSTALL_COMMANDS.md +4 -4
  37. package/docs/INSTALL_COMMANDS_zh.md +4 -4
  38. package/docs/REFERENCE.md +349 -308
  39. package/docs/REFERENCE_zh.md +342 -309
  40. package/manifest.json +3 -2
  41. package/mcp_server.py +383 -130
  42. package/package.json +4 -2
  43. package/requirements.txt +7 -4
  44. package/run.py +61 -5
  45. package/scripts/bilibili.py +11 -3
  46. package/scripts/engine.py +1039 -118
  47. package/scripts/pdf.py +21 -0
  48. package/scripts/research.py +2087 -0
  49. package/scripts/scholar.py +33 -6
  50. package/scripts/sitemap.py +20 -5
  51. package/scripts/untrusted.py +312 -95
package/scripts/engine.py CHANGED
@@ -51,7 +51,7 @@ from bs4.element import NavigableString, Tag
51
51
 
52
52
  from . import untrusted
53
53
 
54
- __version__ = "1.3.0"
54
+ __version__ = "1.3.1"
55
55
 
56
56
  DEFAULT_USER_AGENT = (
57
57
  f"WebForager/{__version__} (Information Gathering; "
@@ -162,6 +162,17 @@ def _allow_private_from_env() -> bool:
162
162
  )
163
163
 
164
164
 
165
+ def _query_robots_from_env() -> str:
166
+ """User-level policy for query endpoints (search, scholarly APIs).
167
+
168
+ This is deliberately an environment/CLI decision, not an MCP parameter:
169
+ a fetched page must never be able to turn robots off for the agent.
170
+ """
171
+
172
+ value = os.environ.get("WEB_FORAGER_QUERY_ROBOTS", "warn").strip().lower()
173
+ return value if value in ("respect", "warn", "ignore") else "warn"
174
+
175
+
165
176
  @dataclass
166
177
  class FetchConfig:
167
178
  """Runtime knobs for one CLI invocation."""
@@ -187,6 +198,10 @@ class FetchConfig:
187
198
  max_links: int = 20
188
199
  follow: str = "same-site" # same-site | all | none
189
200
  search_engine: str = "auto"
201
+ # Query interfaces (search engines, scholarly APIs) have their own policy.
202
+ # warn reads a disallowed endpoint and records the override; respect
203
+ # refuses it; ignore skips the check. MCP never lets the model choose.
204
+ query_robots: str = field(default_factory=_query_robots_from_env)
190
205
  search_pages: int = 1
191
206
  # Engines are different hosts, so they may run at once without breaking the
192
207
  # per-origin delay. Set False for a strictly serial diagnostic run.
@@ -208,6 +223,8 @@ class FetchConfig:
208
223
  raise ValueError("render must be auto, never or always")
209
224
  if self.robots_policy not in ("respect", "warn", "ignore"):
210
225
  raise ValueError("robots_policy must be respect, warn or ignore")
226
+ if self.query_robots not in ("respect", "warn", "ignore"):
227
+ raise ValueError("query_robots must be respect, warn or ignore")
211
228
  if self.input_gate not in ("off", "mark", "strict"):
212
229
  raise ValueError("input_gate must be off, mark or strict")
213
230
  if self.follow not in ("same-site", "all", "none"):
@@ -451,6 +468,7 @@ def _guarded_request(
451
468
  *,
452
469
  allow_private: bool = False,
453
470
  max_redirects: int = 5,
471
+ before_request: Any = None,
454
472
  **kwargs: Any,
455
473
  ) -> Any:
456
474
  """GET ``url`` and follow redirects one hop at a time, checking each target.
@@ -463,6 +481,8 @@ def _guarded_request(
463
481
  current = url
464
482
  for _ in range(max_redirects + 1):
465
483
  assert_public_url(current, allow_private=allow_private)
484
+ if before_request is not None:
485
+ before_request(current)
466
486
  response = getter(current, allow_redirects=False, **kwargs)
467
487
  location = response.headers.get("Location") if response.headers else None
468
488
  if response.status_code in (301, 302, 303, 307, 308) and location:
@@ -689,7 +709,10 @@ class RobotsCache:
689
709
  robots_url,
690
710
  allow_private=self.config.allow_private,
691
711
  headers=headers,
692
- timeout=min(max(self.config.timeout, 1.0), 10.0),
712
+ timeout=(
713
+ min(max(self.config.timeout, 1.0), 3.0),
714
+ min(max(self.config.timeout, 1.0), 6.0),
715
+ ),
693
716
  )
694
717
  except requests.RequestException as exc:
695
718
  # RFC 9309 §2.3.1: an unreachable robots.txt is treated as a
@@ -814,6 +837,7 @@ class HttpResponse:
814
837
  encoding: str
815
838
  content_type: str
816
839
  from_cache: bool = False
840
+ robots_overridden: bool = False
817
841
 
818
842
 
819
843
  def _cache_root(config: FetchConfig) -> Path:
@@ -822,6 +846,17 @@ def _cache_root(config: FetchConfig) -> Path:
822
846
  return Path.home() / ".web-forager" / "cache"
823
847
 
824
848
 
849
+ def _private_path(path: Path, mode: int) -> None:
850
+ """Best-effort owner-only permissions for local state on POSIX."""
851
+
852
+ if os.name == "nt":
853
+ return
854
+ try:
855
+ path.chmod(mode)
856
+ except OSError:
857
+ pass
858
+
859
+
825
860
  def _cache_key(url: str) -> str:
826
861
  import hashlib
827
862
 
@@ -880,20 +915,42 @@ class HttpFetcher:
880
915
  except Exception:
881
916
  pass
882
917
 
918
+ def clone(self) -> "HttpFetcher":
919
+ """A per-thread fetcher sharing robots and rate limits but not a session.
920
+
921
+ ``requests.Session`` is not thread-safe, so parallel focus reads use one
922
+ clone per worker. The robots cache and rate limiter are shared, which
923
+ keeps politeness global; cookies already collected are copied once.
924
+ """
925
+
926
+ other = HttpFetcher(self.config)
927
+ other.robots = self.robots
928
+ other.limiter = self.limiter
929
+ try:
930
+ other.session.cookies.update(self.session.cookies)
931
+ except Exception:
932
+ pass
933
+ return other
934
+
883
935
  # -- robots ---------------------------------------------------------
884
936
 
885
- def assert_allowed(self, url: str) -> RobotsPolicy:
886
- """Raise RobotsDenied when policy is ``respect`` and robots forbids it.
937
+ def assert_allowed(
938
+ self, url: str, robots_policy: Optional[str] = None
939
+ ) -> RobotsPolicy:
940
+ """Raise RobotsDenied when the effective policy is ``respect``.
887
941
 
888
942
  ``warn`` returns the denying policy instead of raising, so the caller
889
943
  can still read the page and must record ``robots_overridden: true``.
944
+ ``robots_policy`` lets a query interface use its own policy without
945
+ changing the policy for target pages.
890
946
  """
891
947
 
892
- if not self.config.respect_robots or self.config.robots_policy == "ignore":
948
+ effective = (robots_policy or self.config.robots_policy).strip().lower()
949
+ if not self.config.respect_robots or effective == "ignore":
893
950
  return RobotsPolicy.allowing_all(note="robots checking disabled by policy")
894
951
  policy = self.robots.policy_for(url)
895
952
  if not policy.allowed(url):
896
- if self.config.robots_policy == "warn":
953
+ if effective == "warn":
897
954
  return policy
898
955
  detail = policy.note or policy.source or "robots.txt"
899
956
  raise RobotsDenied(f"robots.txt disallows {url} ({detail})")
@@ -902,7 +959,12 @@ class HttpFetcher:
902
959
  # -- main entry -----------------------------------------------------
903
960
 
904
961
  def fetch(
905
- self, url: str, *, check_robots: bool = True, use_cache: bool = True
962
+ self,
963
+ url: str,
964
+ *,
965
+ check_robots: bool = True,
966
+ use_cache: bool = True,
967
+ robots_policy: Optional[str] = None,
906
968
  ) -> HttpResponse:
907
969
  """Fetch one URL.
908
970
 
@@ -914,8 +976,16 @@ class HttpFetcher:
914
976
  url = normalize_url(url)
915
977
  assert_public_url(url, allow_private=self.config.allow_private)
916
978
  policy: Optional[RobotsPolicy] = None
979
+ overridden = {"value": False}
917
980
  if check_robots:
918
- policy = self.assert_allowed(url)
981
+ policy = self.assert_allowed(url, robots_policy)
982
+ if policy is not None and not policy.allowed(url):
983
+ overridden["value"] = True
984
+
985
+ def robots_guard(target: str) -> None:
986
+ checked = self.assert_allowed(target, robots_policy)
987
+ if checked is not None and not checked.allowed(target):
988
+ overridden["value"] = True
919
989
 
920
990
  cached = self._read_cache(url) if use_cache else None
921
991
  if cached is not None:
@@ -932,7 +1002,10 @@ class HttpFetcher:
932
1002
  url,
933
1003
  allow_private=self.config.allow_private,
934
1004
  max_redirects=self.session.max_redirects,
935
- timeout=self.config.timeout,
1005
+ before_request=robots_guard if check_robots else None,
1006
+ # A connect timeout per resolved address: a blocked
1007
+ # address must not consume the full read budget.
1008
+ timeout=(min(self.config.timeout, 3.0), self.config.timeout),
936
1009
  stream=True,
937
1010
  )
938
1011
  except requests.RequestException as exc:
@@ -954,6 +1027,7 @@ class HttpFetcher:
954
1027
  raise FetchError(f"HTTP {status} for {url}")
955
1028
 
956
1029
  result = self._read_response(response, url)
1030
+ result.robots_overridden = overridden["value"]
957
1031
  if use_cache:
958
1032
  self._write_cache(url, result)
959
1033
  return result
@@ -984,7 +1058,7 @@ class HttpFetcher:
984
1058
  content_type = str(meta.get("content_type") or "text/html")
985
1059
  encoding = str(meta.get("encoding") or "utf-8")
986
1060
  text = body.decode(encoding, errors="replace")
987
- log(f"cache hit: {url}", self.config)
1061
+ log(f"cache hit: {redact_url_for_log(url)}", self.config)
988
1062
  return HttpResponse(
989
1063
  url=url,
990
1064
  final_url=str(meta.get("final_url") or url),
@@ -995,6 +1069,7 @@ class HttpFetcher:
995
1069
  encoding=encoding,
996
1070
  content_type=content_type,
997
1071
  from_cache=True,
1072
+ robots_overridden=bool(meta.get("robots_overridden", False)),
998
1073
  )
999
1074
 
1000
1075
  def _write_cache(self, url: str, response: HttpResponse) -> None:
@@ -1013,8 +1088,10 @@ class HttpFetcher:
1013
1088
  }
1014
1089
  meta_path, body_path = self._cache_paths(url)
1015
1090
  try:
1016
- meta_path.parent.mkdir(parents=True, exist_ok=True)
1091
+ meta_path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
1092
+ _private_path(meta_path.parent, 0o700)
1017
1093
  body_path.write_bytes(response.content)
1094
+ _private_path(body_path, 0o600)
1018
1095
  meta_path.write_text(
1019
1096
  json.dumps(
1020
1097
  {
@@ -1024,11 +1101,13 @@ class HttpFetcher:
1024
1101
  "headers": safe_headers,
1025
1102
  "content_type": response.content_type,
1026
1103
  "encoding": response.encoding,
1104
+ "robots_overridden": bool(response.robots_overridden),
1027
1105
  },
1028
1106
  ensure_ascii=False,
1029
1107
  ),
1030
1108
  encoding="utf-8",
1031
1109
  )
1110
+ _private_path(meta_path, 0o600)
1032
1111
  except OSError as exc: # a cache failure must never fail the fetch
1033
1112
  log(f"cache write skipped ({exc})", self.config)
1034
1113
 
@@ -1107,11 +1186,20 @@ class HttpFetcher:
1107
1186
  self.limiter.wait(url)
1108
1187
  response: Optional[requests.Response] = None
1109
1188
  try:
1189
+ def resolve_robots_guard(target: str) -> None:
1190
+ if self.config.query_robots != "ignore":
1191
+ self.assert_allowed(target, self.config.query_robots)
1192
+
1110
1193
  response = _guarded_request(
1111
1194
  self.session.get,
1112
1195
  url,
1113
1196
  allow_private=self.config.allow_private,
1114
1197
  max_redirects=self.session.max_redirects,
1198
+ before_request=(
1199
+ resolve_robots_guard
1200
+ if self.config.query_robots != "ignore"
1201
+ else None
1202
+ ),
1115
1203
  timeout=self.config.timeout,
1116
1204
  stream=True,
1117
1205
  )
@@ -1444,6 +1532,7 @@ def search(
1444
1532
  limit: Optional[int] = None,
1445
1533
  fetcher: Optional[HttpFetcher] = None,
1446
1534
  report: Optional[list[dict[str, Any]]] = None,
1535
+ quality: Optional[dict[str, Any]] = None,
1447
1536
  ) -> list[dict[str, str]]:
1448
1537
  """Return ``[{title, url, snippet, source}]`` for a keyword query.
1449
1538
 
@@ -1454,11 +1543,14 @@ def search(
1454
1543
 
1455
1544
  Pass ``report`` to collect one entry per engine tried — name, whether it
1456
1545
  answered, how many results and how long it took — so a caller in a hostile
1457
- network can see which backend is actually reachable.
1458
-
1459
- Search endpoints are query interfaces, not target pages. robots.txt
1460
- therefore applies to the *result pages* we later fetch, not to the query
1461
- request itself. Queries are still rate-limited per host.
1546
+ network can see which backend is actually reachable. Pass ``quality`` to
1547
+ receive counts for the rows removed by the relevance filter.
1548
+
1549
+ Search endpoints are query interfaces, not target pages. Their robots
1550
+ policy is ``config.query_robots`` (default ``warn``): a disallowed query
1551
+ endpoint is read and the override is recorded; ``respect`` refuses it and
1552
+ ``ignore`` skips the check. Result pages we later fetch still go through
1553
+ the normal target-page robots policy. Queries are rate-limited per host.
1462
1554
  """
1463
1555
 
1464
1556
  query = (query or "").strip()
@@ -1508,6 +1600,8 @@ def search(
1508
1600
  ) -> tuple[dict[str, Any], Optional[list[dict[str, str]]]]:
1509
1601
  started = time.monotonic()
1510
1602
  entry: dict[str, Any] = {"engine": name}
1603
+ if config.query_robots != "respect":
1604
+ entry["query_robots"] = config.query_robots
1511
1605
  if fallback:
1512
1606
  entry["fallback"] = True
1513
1607
  if qtext != query:
@@ -1531,6 +1625,7 @@ def search(
1531
1625
  ]
1532
1626
  entry.update(
1533
1627
  ok=True,
1628
+ status="ok" if items else "empty",
1534
1629
  results=len(items),
1535
1630
  ms=round((time.monotonic() - started) * 1000),
1536
1631
  )
@@ -1539,6 +1634,7 @@ def search(
1539
1634
  errors.append(f"{name}: {exc}")
1540
1635
  entry.update(
1541
1636
  ok=False,
1637
+ status=_engine_failure_status(exc),
1542
1638
  results=0,
1543
1639
  error=str(exc),
1544
1640
  ms=round((time.monotonic() - started) * 1000),
@@ -1578,13 +1674,48 @@ def search(
1578
1674
  if not groups:
1579
1675
  raise FetchError("all search engines failed: " + "; ".join(errors) or "no engine available")
1580
1676
 
1581
- if len(groups) == 1:
1582
- merged = groups[0][1]
1583
- else:
1584
- merged = merge_search_results(groups, query, limit=limit)
1585
- return diversify_results(
1677
+ # Rank even when only one engine answered. The old single-group path
1678
+ # returned engine order untouched, which is how an Excel INDIRECT tutorial
1679
+ # could sit third for an "indirect prompt injection" query.
1680
+ merged = merge_search_results(groups, query, limit=limit)
1681
+ final = diversify_results(
1586
1682
  dedupe_results(merged, limit=max(limit * 3, limit)), limit=limit
1587
1683
  )
1684
+ annotate_results(final, query)
1685
+ before_filter = len(final)
1686
+ # When enough real answers exist, do not pad the list with results that
1687
+ # matched one word or the wrong language. With only one or two relevant
1688
+ # hits the unrelated rows stay at the bottom and a warning names them.
1689
+ relevant = [item for item in final if item.get("relevance")]
1690
+ if len(relevant) >= min(3, limit):
1691
+ final = relevant
1692
+ if quality is not None:
1693
+ quality["returned_before_filter"] = before_filter
1694
+ quality["dropped_off_topic"] = max(0, before_filter - len(final))
1695
+ return final
1696
+
1697
+
1698
+ def _engine_failure_status(exc: BaseException) -> str:
1699
+ """Classify one engine failure for the machine-readable report.
1700
+
1701
+ A caller in a hostile network needs to tell an empty result from a block,
1702
+ a timeout and a parser change. The old ``ok: false`` plus free text hid
1703
+ that distinction, which made a blocked engine look like a thin topic.
1704
+ """
1705
+
1706
+ text = f"{type(exc).__name__}: {exc}".lower()
1707
+ if any(word in text for word in ("timeout", "timed out", "time out")):
1708
+ return "timeout"
1709
+ if "robots" in text:
1710
+ return "robots_denied"
1711
+ if any(
1712
+ word in text
1713
+ for word in ("verification", "antispider", "captcha", "blocked", "forbidden")
1714
+ ):
1715
+ return "blocked"
1716
+ if any(word in text for word in ("parse", "decode", "json", "xml", "unexpected")):
1717
+ return "parse_error"
1718
+ return "error"
1588
1719
 
1589
1720
 
1590
1721
  def _thin_results(groups: list[tuple[str, list[dict[str, str]]]], limit: int) -> bool:
@@ -1627,6 +1758,13 @@ def _search_one(
1627
1758
  ) -> list[dict[str, str]]:
1628
1759
  quoted = urllib.parse.quote_plus(query)
1629
1760
  pages = max(1, pages)
1761
+ # Search endpoints are query interfaces, not target pages. They still get
1762
+ # a policy: respect (default) refuses a disallowed endpoint, warn reads it
1763
+ # and marks the override, ignore skips the check. MCP force-sets respect.
1764
+ query_fetch = {
1765
+ "check_robots": config.query_robots != "ignore",
1766
+ "robots_policy": config.query_robots,
1767
+ }
1630
1768
  collected: list[dict[str, str]] = []
1631
1769
 
1632
1770
  for page in range(1, pages + 1):
@@ -1634,14 +1772,14 @@ def _search_one(
1634
1772
  break
1635
1773
  if name == "360":
1636
1774
  response = fetcher.fetch(
1637
- _SO_360.format(query=quoted, page=page), check_robots=False
1775
+ _SO_360.format(query=quoted, page=page), **query_fetch
1638
1776
  )
1639
1777
  if is_360_blocked(response.text):
1640
1778
  raise FetchError("360 returned a verification page; wait and retry later")
1641
1779
  found = parse_360_html(response.text, limit=limit)
1642
1780
  elif name == "sogou":
1643
1781
  response = fetcher.fetch(
1644
- _SOGOU.format(query=quoted, page=page), check_robots=False
1782
+ _SOGOU.format(query=quoted, page=page), **query_fetch
1645
1783
  )
1646
1784
  if is_sogou_blocked(response.text):
1647
1785
  raise FetchError("Sogou returned a verification page (antispider)")
@@ -1651,7 +1789,7 @@ def _search_one(
1651
1789
  found = resolve_sogou_results(parsed, fetcher, limit=limit)
1652
1790
  elif name == "weixin":
1653
1791
  response = fetcher.fetch(
1654
- _WEIXIN.format(query=quoted, page=page), check_robots=False
1792
+ _WEIXIN.format(query=quoted, page=page), **query_fetch
1655
1793
  )
1656
1794
  if is_sogou_blocked(response.text):
1657
1795
  raise FetchError("weixin.sogou.com returned a verification page (antispider)")
@@ -1660,7 +1798,7 @@ def _search_one(
1660
1798
  )
1661
1799
  found = resolve_sogou_results(parsed, fetcher, limit=limit)
1662
1800
  elif name == "duckduckgo":
1663
- response = fetcher.fetch(_DDG_HTML.format(query=quoted), check_robots=False)
1801
+ response = fetcher.fetch(_DDG_HTML.format(query=quoted), **query_fetch)
1664
1802
  found = parse_ddg_html(response.text, limit=limit)
1665
1803
  else:
1666
1804
  first = 1 + (page - 1) * 10
@@ -1668,7 +1806,7 @@ def _search_one(
1668
1806
  try:
1669
1807
  response = fetcher.fetch(
1670
1808
  _BING_RSS.format(query=quoted, count=limit, first=first),
1671
- check_robots=False,
1809
+ **query_fetch,
1672
1810
  )
1673
1811
  found = parse_bing_rss(response.text, limit=limit)
1674
1812
  except FetchError:
@@ -1676,7 +1814,7 @@ def _search_one(
1676
1814
  if not found:
1677
1815
  response = fetcher.fetch(
1678
1816
  _BING_HTML.format(query=quoted, count=limit, first=first),
1679
- check_robots=False,
1817
+ **query_fetch,
1680
1818
  )
1681
1819
  found = parse_bing_html(response.text, limit=limit)
1682
1820
 
@@ -1705,46 +1843,495 @@ def result_coverage(result: dict[str, str], tokens: list[str]) -> float:
1705
1843
  return sum(1 for token in tokens if token in haystack) / len(tokens)
1706
1844
 
1707
1845
 
1708
- _HIGH_VALUE_SUFFIXES = (
1709
- "docs.python.org",
1710
- "developer.mozilla.org",
1711
- "wikipedia.org",
1712
- "arxiv.org",
1713
- "github.com",
1714
- "stackoverflow.com",
1715
- "nature.com",
1716
- "science.org",
1717
- "who.int",
1718
- "worldbank.org",
1719
- "gov.cn",
1720
- "edu.cn",
1721
- "ac.uk",
1722
- "edu",
1846
+ def _content_tokens(query: str) -> list[str]:
1847
+ """Query terms that carry meaning; stopwords and one-letter words are noise.
1848
+
1849
+ Chinese is not whitespace-separated. A long CJK run is expanded into
1850
+ character bigrams for matching; a mixed token carrying digits (``2026年11月``)
1851
+ stays whole because the date is itself the searchable unit. This is not a
1852
+ word segmenter, but it stops a whole sentence from becoming one token that
1853
+ no result can ever match.
1854
+ """
1855
+
1856
+ tokens = _query_tokens(query)
1857
+ content: list[str] = []
1858
+ for token in tokens:
1859
+ if token in _QUERY_STOPWORDS or len(token) < 2:
1860
+ continue
1861
+ if re.search(r"\d", token):
1862
+ content.append(token)
1863
+ continue
1864
+ if not _CJK_RE.search(token):
1865
+ content.append(token)
1866
+ continue
1867
+ for run in re.findall(
1868
+ r"[㐀-䶿一-鿿豈-﫿぀-ヿ가-힯]+"
1869
+ r"|[A-Za-z0-9]+",
1870
+ token,
1871
+ ):
1872
+ if _CJK_RE.search(run):
1873
+ if len(run) <= 3:
1874
+ content.append(run)
1875
+ else:
1876
+ content.extend(run[index : index + 2] for index in range(len(run) - 1))
1877
+ elif len(run) >= 2:
1878
+ content.append(run.lower())
1879
+ return list(dict.fromkeys(content)) or tokens
1880
+
1881
+
1882
+ _LATIN_WORD_RE = re.compile(r"[A-Za-z][A-Za-z0-9+#.-]*")
1883
+
1884
+
1885
+ def _stem_token(token: str) -> str:
1886
+ """A deliberately small English stemmer for matching, not display.
1887
+
1888
+ ``injections`` and ``injection`` should meet; a full Porter stemmer would
1889
+ also merge unrelated words and make the relevance check harder to explain.
1890
+ Tokens containing CJK or punctuation are returned unchanged.
1891
+ """
1892
+
1893
+ value = token.lower()
1894
+ if not re.fullmatch(r"[a-z][a-z0-9+-]*", value) or len(value) <= 4:
1895
+ return value
1896
+ if value.endswith("ies"):
1897
+ return value[:-3] + "y"
1898
+ if value.endswith("ing") and len(value) > 5:
1899
+ return value[:-3]
1900
+ if value.endswith("ed") and len(value) > 4:
1901
+ return value[:-2]
1902
+ if value.endswith("sses"):
1903
+ return value[:-2]
1904
+ if value.endswith(("ches", "shes", "xes", "zes", "oes")):
1905
+ return value[:-2]
1906
+ if value.endswith("s") and not value.endswith("ss"):
1907
+ return value[:-1]
1908
+ return value
1909
+
1910
+
1911
+ def _stems(text: str) -> set[str]:
1912
+ return {_stem_token(word) for word in _LATIN_WORD_RE.findall(text.lower())}
1913
+
1914
+
1915
+ def _token_in_text(token: str, text: str, stems: set[str]) -> bool:
1916
+ token = (token or "").lower()
1917
+ if token and token in text:
1918
+ return True
1919
+ return bool(re.fullmatch(r"[a-z][a-z0-9+-]*", token)) and _stem_token(token) in stems
1920
+
1921
+
1922
+ def _script_stats(text: str) -> tuple[int, int]:
1923
+ """Return ``(CJK characters, Latin words)`` for a language sanity check."""
1924
+
1925
+ value = str(text or "")
1926
+ return (
1927
+ len(_CJK_RE.findall(value)),
1928
+ len(re.findall(r"[A-Za-z]{2,}", value)),
1929
+ )
1930
+
1931
+
1932
+ _INTENT_PATTERNS: dict[str, re.Pattern[str]] = {
1933
+ "news": re.compile(
1934
+ r"新闻|资讯|最新|今日|今天|本周|本月|今年|刚刚|发布会|财报|营收|股价|"
1935
+ r"202[4-9]年|\bnews\b|\bupdate\b",
1936
+ re.I,
1937
+ ),
1938
+ "comparison": re.compile(
1939
+ r"对比|比较|区别|差异|怎么选|如何选|哪个好|哪个更|优缺点|取舍|"
1940
+ r"\bvs\.?\b|\bversus\b|\bcomparison\b|\bcompare\b|\bbetter\b|\bchoose\b",
1941
+ re.I,
1942
+ ),
1943
+ "technical": re.compile(
1944
+ r"python|javascript|typescript|java|golang|rust|asyncio|api|sdk|cli|"
1945
+ r"http|json|sql|docker|kubernetes|git|linux|regex|报错|异常|安装|配置|"
1946
+ r"函数|接口|依赖|编译|代码|文档|版本|任务组|协程",
1947
+ re.I,
1948
+ ),
1949
+ "policy": re.compile(
1950
+ r"法规|法律|政策|合规|监管|标准|条例|法案|司法|欧盟|ai act|gdpr|"
1951
+ r"regulation|compliance|\blaw\b|\bact\b",
1952
+ re.I,
1953
+ ),
1954
+ "howto": re.compile(
1955
+ r"怎么办|咋办|怎么解决|如何解决|修复|连不上|打不开|用不了|设置|教程|"
1956
+ r"步骤|方法|技巧|how to|fix|troubleshoot|setup|repair",
1957
+ re.I,
1958
+ ),
1959
+ "research": re.compile(
1960
+ r"arxiv|论文|文献|综述|survey|paper|study|research|academic|acl|ieee|acm",
1961
+ re.I,
1962
+ ),
1963
+ "factual": re.compile(
1964
+ r"是谁|多少|哪年|何时|什么时候|定义|作者|创始人|营收|人口|面积|"
1965
+ r"what is|who is|when|how many|definition",
1966
+ re.I,
1967
+ ),
1968
+ }
1969
+ _INTENT_ORDER = (
1970
+ "news",
1971
+ "policy",
1972
+ "comparison",
1973
+ "technical",
1974
+ "howto",
1975
+ "research",
1976
+ "factual",
1977
+ "general",
1723
1978
  )
1724
- _LOW_VALUE_SUFFIXES = (
1725
- "doc88.com",
1726
- "docin.com",
1727
- "wenku.baidu.com",
1728
- "360doc.com",
1729
- "book118.com",
1979
+ _INTENT_STRONG_SIGNALS = {
1980
+ "comparison": ("怎么选", "如何选", "哪个好", "对比", "vs"),
1981
+ "howto": ("怎么办", "咋办", "连不上", "打不开", "怎么解决"),
1982
+ "factual": ("是谁", "的作者", "多少", "定义"),
1983
+ "policy": ("法规", "合规", "监管", "ai act"),
1984
+ "news": ("新闻", "最新", "今天", "本月"),
1985
+ "technical": ("报错", "异常", "api", "sdk", "asyncio"),
1986
+ }
1987
+
1988
+
1989
+ def classify_intent(query: str) -> dict[str, Any]:
1990
+ """Classify what kind of source will actually answer ``query``.
1991
+
1992
+ Free and deterministic: a small keyword vote, with a few strong phrases
1993
+ counted twice. It is deliberately conservative — ``general`` is a valid
1994
+ answer, and authority weighting keeps working without it.
1995
+ """
1996
+
1997
+ text = (query or "").lower()
1998
+ scores: dict[str, int] = {}
1999
+ signals: dict[str, list[str]] = {}
2000
+ for name, pattern in _INTENT_PATTERNS.items():
2001
+ matches = pattern.findall(text)
2002
+ if matches:
2003
+ scores[name] = len(matches)
2004
+ signals[name] = list(dict.fromkeys(item.lower() for item in matches))[:5]
2005
+ for name, phrases in _INTENT_STRONG_SIGNALS.items():
2006
+ for phrase in phrases:
2007
+ if phrase in text:
2008
+ scores[name] = scores.get(name, 0) + 1
2009
+ signals.setdefault(name, []).append(phrase)
2010
+ if not scores:
2011
+ return {"intent": "general", "confidence": 0.3, "signals": []}
2012
+ best = max(
2013
+ scores,
2014
+ key=lambda name: (scores[name], -_INTENT_ORDER.index(name)),
2015
+ )
2016
+ confidence = min(0.95, round(0.45 + 0.1 * scores[best], 2))
2017
+ return {
2018
+ "intent": best,
2019
+ "confidence": confidence,
2020
+ "signals": list(dict.fromkeys(signals.get(best, [])))[:5],
2021
+ }
2022
+
2023
+
2024
+ _AUTHORITY_TIERS: tuple[tuple[str, tuple[str, ...]], ...] = (
2025
+ (
2026
+ "primary",
2027
+ (
2028
+ "docs.python.org",
2029
+ "developer.mozilla.org",
2030
+ "go.dev",
2031
+ "rust-lang.org",
2032
+ "nodejs.org",
2033
+ "developer.apple.com",
2034
+ "learn.microsoft.com",
2035
+ "support.microsoft.com",
2036
+ "kubernetes.io",
2037
+ "docs.docker.com",
2038
+ "postgresql.org",
2039
+ "nginx.org",
2040
+ "apache.org",
2041
+ "w3.org",
2042
+ "ietf.org",
2043
+ "unicode.org",
2044
+ "docs.github.com",
2045
+ ),
2046
+ ),
2047
+ (
2048
+ "official",
2049
+ (
2050
+ "gov.cn",
2051
+ "gov",
2052
+ "europa.eu",
2053
+ "sec.gov",
2054
+ "nist.gov",
2055
+ "who.int",
2056
+ "worldbank.org",
2057
+ "un.org",
2058
+ "nvidia.com",
2059
+ "openai.com",
2060
+ "anthropic.com",
2061
+ ),
2062
+ ),
2063
+ (
2064
+ "academic",
2065
+ (
2066
+ "arxiv.org",
2067
+ "aclanthology.org",
2068
+ "openreview.net",
2069
+ "semanticscholar.org",
2070
+ "pubmed.ncbi.nlm.nih.gov",
2071
+ "nature.com",
2072
+ "science.org",
2073
+ "springer.com",
2074
+ "sciencedirect.com",
2075
+ "ieee.org",
2076
+ "acm.org",
2077
+ "plos.org",
2078
+ "biorxiv.org",
2079
+ ),
2080
+ ),
2081
+ (
2082
+ "media",
2083
+ (
2084
+ "reuters.com",
2085
+ "bloomberg.com",
2086
+ "ft.com",
2087
+ "wsj.com",
2088
+ "nytimes.com",
2089
+ "bbc.com",
2090
+ "bbc.co.uk",
2091
+ "caixin.com",
2092
+ "yicai.com",
2093
+ "cls.cn",
2094
+ "nbd.com.cn",
2095
+ "stcn.com",
2096
+ "thepaper.cn",
2097
+ "jiemian.com",
2098
+ "chinanews.com.cn",
2099
+ "finance.sina.com.cn",
2100
+ ),
2101
+ ),
2102
+ (
2103
+ "reference",
2104
+ (
2105
+ "wikipedia.org",
2106
+ "britannica.com",
2107
+ "imdb.com",
2108
+ "themoviedb.org",
2109
+ ),
2110
+ ),
2111
+ (
2112
+ "community",
2113
+ (
2114
+ "stackoverflow.com",
2115
+ "stackexchange.com",
2116
+ "github.com",
2117
+ "reddit.com",
2118
+ "news.ycombinator.com",
2119
+ "zhihu.com",
2120
+ "cnblogs.com",
2121
+ "medium.com",
2122
+ "dev.to",
2123
+ "quora.com",
2124
+ ),
2125
+ ),
2126
+ (
2127
+ "low",
2128
+ (
2129
+ "csdn.net",
2130
+ "php.cn",
2131
+ "doc88.com",
2132
+ "docin.com",
2133
+ "wenku.baidu.com",
2134
+ "360doc.com",
2135
+ "book118.com",
2136
+ "jianshu.com",
2137
+ "toutiao.com",
2138
+ ),
2139
+ ),
1730
2140
  )
2141
+ _AUTHORITY_BASE = {
2142
+ "primary": 0.18,
2143
+ "official": 0.18,
2144
+ "academic": 0.16,
2145
+ "media": 0.10,
2146
+ "reference": 0.06,
2147
+ "community": 0.05,
2148
+ "low": -0.16,
2149
+ "unknown": 0.0,
2150
+ }
2151
+ _AUTHORITY_MULTIPLIER = {
2152
+ "technical": {
2153
+ "primary": 1.3,
2154
+ "official": 0.8,
2155
+ "academic": 0.9,
2156
+ "media": 0.4,
2157
+ "reference": 0.6,
2158
+ "community": 0.9,
2159
+ "low": 1.2,
2160
+ "unknown": 1.0,
2161
+ },
2162
+ "news": {
2163
+ "primary": 0.4,
2164
+ "official": 0.9,
2165
+ "academic": 0.4,
2166
+ "media": 1.4,
2167
+ "reference": 0.4,
2168
+ "community": 0.3,
2169
+ "low": 1.0,
2170
+ "unknown": 1.0,
2171
+ },
2172
+ "comparison": {
2173
+ "primary": 0.8,
2174
+ "official": 0.6,
2175
+ "academic": 0.8,
2176
+ "media": 0.5,
2177
+ "reference": 0.6,
2178
+ "community": 1.0,
2179
+ "low": 1.2,
2180
+ "unknown": 1.0,
2181
+ },
2182
+ "policy": {
2183
+ "primary": 0.6,
2184
+ "official": 1.3,
2185
+ "academic": 1.0,
2186
+ "media": 0.8,
2187
+ "reference": 0.7,
2188
+ "community": 0.4,
2189
+ "low": 1.2,
2190
+ "unknown": 1.0,
2191
+ },
2192
+ "howto": {
2193
+ "primary": 1.3,
2194
+ "official": 1.0,
2195
+ "academic": 0.5,
2196
+ "media": 0.4,
2197
+ "reference": 0.6,
2198
+ "community": 1.0,
2199
+ "low": 0.8,
2200
+ "unknown": 1.0,
2201
+ },
2202
+ "factual": {
2203
+ "primary": 1.0,
2204
+ "official": 1.2,
2205
+ "academic": 1.1,
2206
+ "media": 0.9,
2207
+ "reference": 1.1,
2208
+ "community": 0.6,
2209
+ "low": 1.0,
2210
+ "unknown": 1.0,
2211
+ },
2212
+ "research": {
2213
+ "primary": 0.9,
2214
+ "official": 0.7,
2215
+ "academic": 1.4,
2216
+ "media": 0.4,
2217
+ "reference": 0.6,
2218
+ "community": 0.6,
2219
+ "low": 1.2,
2220
+ "unknown": 1.0,
2221
+ },
2222
+ "general": {tier: 1.0 for tier in _AUTHORITY_BASE},
2223
+ }
2224
+
2225
+
2226
+ def authority_for(host: str, intent: str = "general") -> dict[str, Any]:
2227
+ """Return an explicit source-authority tier and a small ranking bonus."""
2228
+
2229
+ host = (host or "").lower().lstrip(".")
2230
+ tier = "unknown"
2231
+ for name, suffixes in _AUTHORITY_TIERS:
2232
+ if any(host == suffix or host.endswith("." + suffix) for suffix in suffixes):
2233
+ tier = name
2234
+ break
2235
+ multiplier = _AUTHORITY_MULTIPLIER.get(
2236
+ intent, _AUTHORITY_MULTIPLIER["general"]
2237
+ ).get(tier, 1.0)
2238
+ score = _AUTHORITY_BASE[tier] * multiplier
2239
+ return {
2240
+ "tier": tier,
2241
+ "score": round(score, 3),
2242
+ "intent": intent,
2243
+ }
1731
2244
 
1732
2245
 
1733
- def _host_prior(host: str) -> float:
1734
- """A small prior for domains that usually answer a query well.
2246
+ def result_match(
2247
+ result: dict[str, str], query: str, tokens: Optional[list[str]] = None
2248
+ ) -> dict[str, Any]:
2249
+ """Grade how well one search result answers ``query``.
1735
2250
 
1736
- It only breaks ties and nudges; query coverage carries most of the score,
1737
- so a content farm with the exact answer still outranks an off-topic paper.
2251
+ A title is not a match just because the URL contains one query word. The
2252
+ result must either contain the full phrase or at least two meaningful query
2253
+ terms with half the query covered. A Chinese page that merely repeats one
2254
+ English product name does not pass an English four-word query; it is marked
2255
+ with ``language_match: false`` and ranked below real answers.
1738
2256
  """
1739
2257
 
1740
- host = (host or "").lower()
1741
- for suffix in _HIGH_VALUE_SUFFIXES:
1742
- if host == suffix or host.endswith("." + suffix):
1743
- return 0.08
1744
- for suffix in _LOW_VALUE_SUFFIXES:
1745
- if host == suffix or host.endswith("." + suffix):
1746
- return -0.08
1747
- return 0.0
2258
+ tokens = tokens if tokens is not None else _content_tokens(query)
2259
+ haystack = " ".join(
2260
+ (result.get("title", ""), result.get("snippet", ""))
2261
+ ).lower()
2262
+ title = (result.get("title") or "").lower()
2263
+ stems = _stems(haystack)
2264
+ title_stems = _stems(title)
2265
+ matched = [token for token in tokens if _token_in_text(token, haystack, stems)]
2266
+ coverage = len(matched) / len(tokens) if tokens else 0.0
2267
+ title_hits = (
2268
+ sum(1 for token in tokens if _token_in_text(token, title, title_stems)) / len(tokens)
2269
+ if tokens
2270
+ else 0.0
2271
+ )
2272
+ phrase = " ".join((query or "").lower().split())
2273
+ exact = bool(phrase and phrase in haystack)
2274
+ title_exact = bool(phrase and title.strip() == phrase)
2275
+ cjk_query = any(_CJK_RE.search(token) for token in tokens)
2276
+ # A CJK query has no word boundaries, so its tokens are character bigrams
2277
+ # and full coverage is the wrong bar: an eight-character question has seven
2278
+ # bigrams, and a correct title often carries only a few of them. Requiring
2279
+ # several *overlapping* pieces is the useful signal there; Latin queries
2280
+ # keep the stricter coverage rule.
2281
+ required = 1 if len(tokens) <= 1 else (3 if cjk_query and len(tokens) >= 8 else 2)
2282
+ if cjk_query:
2283
+ relevant = exact or len(matched) >= required
2284
+ else:
2285
+ relevant = exact or (len(matched) >= required and coverage >= 0.5)
2286
+
2287
+ query_cjk, query_latin = _script_stats(query)
2288
+ result_cjk, result_latin = _script_stats(haystack)
2289
+ language_match = True
2290
+ # Latin query, overwhelmingly Chinese result that does not carry the query
2291
+ # terms. This is the Excel INDIRECT failure mode: one English product name
2292
+ # in a Chinese tutorial is not an answer to an English concept query.
2293
+ if query_latin >= 2 and query_cjk == 0:
2294
+ if (
2295
+ result_cjk >= 8
2296
+ and result_latin < max(2, len(tokens))
2297
+ and coverage < 0.5
2298
+ ):
2299
+ language_match = False
2300
+ # A Chinese query answered by an unrelated English page.
2301
+ elif query_cjk >= 2 and query_latin == 0:
2302
+ if result_latin >= 8 and result_cjk < 4 and coverage < 0.5:
2303
+ language_match = False
2304
+
2305
+ return {
2306
+ "coverage": round(coverage, 3),
2307
+ "title_coverage": round(title_hits, 3),
2308
+ "matched_terms": matched,
2309
+ "exact": exact,
2310
+ "title_exact": title_exact,
2311
+ "relevant": relevant,
2312
+ "language_match": language_match,
2313
+ }
2314
+
2315
+
2316
+ def annotate_results(results: list[dict[str, Any]], query: str) -> list[dict[str, Any]]:
2317
+ """Attach relevance and authority fields in place and return the same list."""
2318
+
2319
+ tokens = _content_tokens(query)
2320
+ intent = classify_intent(query)
2321
+ for item in results:
2322
+ details = result_match(item, query, tokens)
2323
+ host = urllib.parse.urlsplit(item.get("url", "")).netloc
2324
+ authority = authority_for(host, intent["intent"])
2325
+ item.update(
2326
+ coverage=details["coverage"],
2327
+ matched_terms=details["matched_terms"],
2328
+ relevance=details["relevant"],
2329
+ language_match=details["language_match"],
2330
+ authority_tier=authority["tier"],
2331
+ authority_score=authority["score"],
2332
+ intent=intent["intent"],
2333
+ )
2334
+ return results
1748
2335
 
1749
2336
 
1750
2337
  def _freshness_bonus(text: str) -> float:
@@ -1835,42 +2422,141 @@ def diversify_results(
1835
2422
  return (chosen + overflow)[:limit]
1836
2423
 
1837
2424
 
2425
+ _SEARCH_ENGINE_HOSTS = {
2426
+ "sogou.com",
2427
+ "www.sogou.com",
2428
+ "m.sogou.com",
2429
+ "weixin.sogou.com",
2430
+ "m.weixin.sogou.com",
2431
+ "so.com",
2432
+ "www.so.com",
2433
+ "m.so.com",
2434
+ "bing.com",
2435
+ "www.bing.com",
2436
+ "cn.bing.com",
2437
+ "baidu.com",
2438
+ "www.baidu.com",
2439
+ "m.baidu.com",
2440
+ "google.com",
2441
+ "www.google.com",
2442
+ "duckduckgo.com",
2443
+ "html.duckduckgo.com",
2444
+ "yandex.com",
2445
+ "yandex.ru",
2446
+ }
2447
+
2448
+
2449
+ def _is_search_engine_page(url: str) -> bool:
2450
+ """True for a search engine's own result page, which is not an answer.
2451
+
2452
+ One backend occasionally echoes a Sogou/360 results URL into its own list.
2453
+ Ranking it first would cite the search box instead of the source, so these
2454
+ intermediate pages are dropped before scoring. Article hosts that merely
2455
+ live under a portal (``baijiahao.baidu.com``) are not in this set.
2456
+ """
2457
+
2458
+ try:
2459
+ parts = urllib.parse.urlsplit(str(url or "").strip())
2460
+ except ValueError:
2461
+ return False
2462
+ host = (parts.hostname or "").lower().rstrip(".")
2463
+ if host not in _SEARCH_ENGINE_HOSTS:
2464
+ return False
2465
+ path = (parts.path or "").lower()
2466
+ query = (parts.query or "").lower()
2467
+ if any(
2468
+ marker in query
2469
+ for marker in ("query=", "q=", "wd=", "keyword=", "word=")
2470
+ ):
2471
+ return True
2472
+ return path.startswith(("/sogou", "/search", "/web", "/s", "/antispider"))
2473
+
2474
+
1838
2475
  def merge_search_results(
1839
- groups: list[tuple[str, list[dict[str, str]]]],
2476
+ groups: list[tuple[str, list[dict[str, Any]]]],
1840
2477
  query: str,
1841
2478
  limit: int = 8,
1842
- ) -> list[dict[str, str]]:
1843
- """Merge several engines by coverage, title match, domain prior and freshness.
2479
+ ) -> list[dict[str, Any]]:
2480
+ """Merge several engines by coverage, title match, authority and freshness.
1844
2481
 
1845
2482
  Engine order only breaks exact ties, so two engines that return the same
1846
2483
  page still produce one stable ordering.
1847
2484
  """
1848
2485
 
1849
- tokens = _query_tokens(query)
1850
- phrase = " ".join((query or "").lower().split())
1851
- scored: list[tuple[float, int, int, dict[str, str]]] = []
2486
+ tokens = _content_tokens(query)
2487
+ intent = classify_intent(query)
2488
+ rrf_by_url: dict[str, float] = {}
2489
+ for _, items in groups:
2490
+ for rank, item in enumerate(items):
2491
+ key = _canonical_url(item.get("url", ""))
2492
+ if key:
2493
+ rrf_by_url[key] = rrf_by_url.get(key, 0.0) + 1.0 / (60 + rank + 1)
2494
+
2495
+ scored: list[tuple[float, int, int, dict[str, Any]]] = []
1852
2496
  for group_index, (name, items) in enumerate(groups):
1853
2497
  for rank, item in enumerate(items):
1854
- candidate = dict(item)
2498
+ if _is_search_engine_page(item.get("url", "")):
2499
+ continue
2500
+ candidate: dict[str, Any] = dict(item)
1855
2501
  candidate.setdefault("source", name)
1856
- coverage = result_coverage(candidate, tokens)
1857
- title = (candidate.get("title") or "").lower()
1858
- title_hits = (
1859
- sum(1 for token in tokens if token in title) / len(tokens)
1860
- if tokens
1861
- else 0.0
1862
- )
1863
- exact = 0.1 if phrase and phrase in title else 0.0
2502
+ details = result_match(candidate, query, tokens)
2503
+ key = _canonical_url(candidate.get("url", ""))
2504
+ rrf = rrf_by_url.get(key, 0.0)
1864
2505
  host = urllib.parse.urlsplit(candidate.get("url", "")).netloc
2506
+ authority = authority_for(host, intent["intent"])
2507
+ freshness = _freshness_bonus(
2508
+ f"{candidate.get('title', '')} {candidate.get('snippet', '')}"
2509
+ )
2510
+ components = {
2511
+ "coverage": round(details["coverage"], 3),
2512
+ "title": round(details["title_coverage"], 3),
2513
+ "exact": bool(details["exact"]),
2514
+ "title_exact": bool(details["title_exact"]),
2515
+ "relevant": bool(details["relevant"]),
2516
+ "rrf": round(rrf, 4),
2517
+ "authority": round(float(authority["score"]), 3),
2518
+ "freshness": round(freshness, 3),
2519
+ "language_penalty": 0.0 if details["language_match"] else -0.35,
2520
+ }
2521
+ candidate.update(
2522
+ coverage=details["coverage"],
2523
+ matched_terms=details["matched_terms"],
2524
+ relevance=details["relevant"],
2525
+ language_match=details["language_match"],
2526
+ rrf=round(rrf, 4),
2527
+ authority_tier=authority["tier"],
2528
+ authority_score=authority["score"],
2529
+ intent=intent["intent"],
2530
+ rank_components=components,
2531
+ rank_reason=[
2532
+ name
2533
+ for name in (
2534
+ "coverage",
2535
+ "title",
2536
+ "rrf",
2537
+ "exact",
2538
+ "title_exact",
2539
+ "relevant",
2540
+ "authority",
2541
+ "freshness",
2542
+ "language_penalty",
2543
+ )
2544
+ if components[name]
2545
+ ],
2546
+ )
1865
2547
  score = (
1866
- coverage * 0.55
1867
- + title_hits * 0.2
1868
- + exact
1869
- + _host_prior(host)
1870
- + _freshness_bonus(
1871
- f"{candidate.get('title', '')} {candidate.get('snippet', '')}"
1872
- )
2548
+ components["coverage"] * 0.5
2549
+ + components["title"] * 0.15
2550
+ + (0.15 if components["exact"] else 0.0)
2551
+ + (0.05 if components["title_exact"] else 0.0)
2552
+ + (0.2 if components["relevant"] else 0.0)
2553
+ + min(0.12, components["rrf"] * 3.0)
2554
+ + components["authority"]
2555
+ + components["freshness"]
2556
+ + components["language_penalty"]
1873
2557
  )
2558
+ if not components["relevant"]:
2559
+ score -= 0.15
1874
2560
  scored.append((score, group_index, rank, candidate))
1875
2561
  scored.sort(key=lambda entry: (-entry[0], entry[1], entry[2]))
1876
2562
  return [entry[3] for entry in scored[: max(limit * 3, limit)]]
@@ -2050,6 +2736,63 @@ def resolve_sogou_results(
2050
2736
  return resolved
2051
2737
 
2052
2738
 
2739
+ _TRACKING_QUERY = re.compile(
2740
+ r"^(?:utm_|gclid$|fbclid$|yclid$|spm$|ref$|source$|share_token$|"
2741
+ r"from$|mc_cid$|mc_eid$|igshid$)",
2742
+ re.I,
2743
+ )
2744
+
2745
+
2746
+ def _canonical_url(url: str) -> str:
2747
+ """A dedupe key for one URL, without tracking noise.
2748
+
2749
+ Search engines return the same article as ``/a``, ``/a/``, ``/a#top`` and
2750
+ ``/a?utm_source=...``. Keeping the display URL untouched is important;
2751
+ this key only decides whether two rows are the same result.
2752
+ """
2753
+
2754
+ raw = str(url or "").strip()
2755
+ if not raw:
2756
+ return ""
2757
+ try:
2758
+ parts = urllib.parse.urlsplit(raw)
2759
+ port = parts.port
2760
+ except ValueError:
2761
+ return raw
2762
+ if not parts.netloc:
2763
+ return raw
2764
+ host = (parts.hostname or "").lower().rstrip(".")
2765
+ if not host:
2766
+ return raw
2767
+ if host.startswith("www."):
2768
+ host = host[4:]
2769
+ if ":" in host and not host.startswith("["):
2770
+ host = f"[{host}]"
2771
+ if port and not (
2772
+ (parts.scheme.lower() == "http" and port == 80)
2773
+ or (parts.scheme.lower() == "https" and port == 443)
2774
+ ):
2775
+ host = f"{host}:{port}"
2776
+ path = re.sub(r"/+", "/", parts.path or "/")
2777
+ if len(path) > 1:
2778
+ path = path.rstrip("/")
2779
+ if path.endswith("/amp"):
2780
+ path = path[:-4] or "/"
2781
+ query = urllib.parse.parse_qsl(parts.query, keep_blank_values=True)
2782
+ query = sorted(
2783
+ (key, value) for key, value in query if not _TRACKING_QUERY.match(key)
2784
+ )
2785
+ return urllib.parse.urlunsplit(
2786
+ (
2787
+ (parts.scheme or "https").lower(),
2788
+ host,
2789
+ path,
2790
+ urllib.parse.urlencode(query, doseq=True),
2791
+ "",
2792
+ )
2793
+ )
2794
+
2795
+
2053
2796
  def dedupe_results(results: Iterable[dict[str, str]], limit: int = 8) -> list[dict[str, str]]:
2054
2797
  """Remove duplicate URLs and unusable titles while preserving rank order."""
2055
2798
 
@@ -2057,7 +2800,8 @@ def dedupe_results(results: Iterable[dict[str, str]], limit: int = 8) -> list[di
2057
2800
  unique: list[dict[str, str]] = []
2058
2801
  for item in results:
2059
2802
  url = item.get("url", "")
2060
- if not url or url in seen:
2803
+ key = _canonical_url(url)
2804
+ if not url or key in seen:
2061
2805
  continue
2062
2806
  title = str(item.get("title") or "")
2063
2807
  # Engines sometimes answer a query with an anti-bot interstitial and
@@ -2066,7 +2810,7 @@ def dedupe_results(results: Iterable[dict[str, str]], limit: int = 8) -> list[di
2066
2810
  # dedupe_results bare URLs.
2067
2811
  if title and (looks_garbled(title) or not _USEFUL_TITLE_RE.search(title)):
2068
2812
  continue
2069
- seen.add(url)
2813
+ seen.add(key)
2070
2814
  unique.append(item)
2071
2815
  if len(unique) >= limit:
2072
2816
  break
@@ -2955,6 +3699,32 @@ def render_html(url: str, config: FetchConfig) -> str:
2955
3699
  browser = playwright.chromium.launch(headless=True)
2956
3700
  try:
2957
3701
  context = browser.new_context(user_agent=config.user_agent)
3702
+
3703
+ def guard_request(route: Any) -> None:
3704
+ """Block private/file subresources, not just the document.
3705
+
3706
+ ``page.goto`` checks the requested and final document URL,
3707
+ but a hostile page can still ask Chromium to load an image,
3708
+ script or fetch() from 127.0.0.1, the LAN or a metadata
3709
+ endpoint. The browser-mode bridge already installs this
3710
+ filter; the Python render path needs the same boundary.
3711
+ """
3712
+
3713
+ request_url = route.request.url
3714
+ if request_url.startswith(("file:", "filesystem:")):
3715
+ route.abort()
3716
+ return
3717
+ if request_url.startswith(("http://", "https://")):
3718
+ try:
3719
+ assert_public_url(
3720
+ request_url, allow_private=config.allow_private
3721
+ )
3722
+ except UnsafeAddress:
3723
+ route.abort()
3724
+ return
3725
+ route.continue_()
3726
+
3727
+ context.route("**/*", guard_request)
2958
3728
  page = context.new_page()
2959
3729
  page.set_default_timeout(timeout_ms)
2960
3730
  page.goto(url, wait_until="domcontentloaded", timeout=timeout_ms)
@@ -3096,13 +3866,15 @@ def load_session(name: str) -> dict[str, Any]:
3096
3866
 
3097
3867
  def save_session(name: str, data: dict[str, Any]) -> Path:
3098
3868
  path = session_path(name)
3099
- path.parent.mkdir(parents=True, exist_ok=True)
3869
+ path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
3870
+ _private_path(path.parent, 0o700)
3100
3871
  payload = {
3101
3872
  "visited": list(data.get("visited", []))[-1000:],
3102
3873
  "trail": list(data.get("trail", []))[-500:],
3103
3874
  "updated_at": utc_now(),
3104
3875
  }
3105
3876
  path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
3877
+ _private_path(path, 0o600)
3106
3878
  return path
3107
3879
 
3108
3880
 
@@ -3295,6 +4067,7 @@ def fetch_page(
3295
4067
  warnings: list[str] = []
3296
4068
 
3297
4069
  try:
4070
+ response: Optional[HttpResponse] = None
3298
4071
  policy = fetcher.assert_allowed(url)
3299
4072
  robots_checked = bool(config.respect_robots and config.robots_policy != "ignore")
3300
4073
  robots_overridden = bool(robots_checked and not policy.allowed(url))
@@ -3305,7 +4078,6 @@ def fetch_page(
3305
4078
  )
3306
4079
  extra_delay = policy.crawl_delay if policy and policy.crawl_delay else 0.0
3307
4080
 
3308
- response: Optional[HttpResponse] = None
3309
4081
  rendered = False
3310
4082
  render_fallback = False
3311
4083
  html = ""
@@ -3316,6 +4088,12 @@ def fetch_page(
3316
4088
  rendered = True
3317
4089
  else:
3318
4090
  response = fetcher.fetch(url)
4091
+ if getattr(response, "robots_overridden", False) and not robots_overridden:
4092
+ robots_overridden = True
4093
+ warnings.append(
4094
+ "a redirect target was read because robots policy is 'warn'; "
4095
+ "this override is recorded as compliance.robots_overridden."
4096
+ )
3319
4097
  html = response.text
3320
4098
  if config.render == "auto" and looks_dynamic(html):
3321
4099
  try:
@@ -3323,7 +4101,7 @@ def fetch_page(
3323
4101
  if rendered_html and len(rendered_html) > len(html):
3324
4102
  html = rendered_html
3325
4103
  rendered = True
3326
- log(f"rendered dynamically: {url}", config)
4104
+ log(f"rendered dynamically: {redact_url_for_log(url)}", config)
3327
4105
  except RenderUnavailable as exc:
3328
4106
  warnings.append(f"dynamic rendering skipped: {exc}")
3329
4107
  except Exception as exc: # pragma: no cover - defensive
@@ -3461,26 +4239,30 @@ def fetch_page(
3461
4239
  "the page may be empty, paywalled, or require JavaScript rendering"
3462
4240
  )
3463
4241
 
3464
- if config.max_chars and len(markdown) > config.max_chars:
3465
- markdown = markdown[: config.max_chars].rstrip() + "\n\n<!-- truncated by web-forager -->"
3466
- warnings.append(f"markdown truncated to {config.max_chars} characters")
3467
-
3468
4242
  # Everything below came from a third party. Say so in the record, and
3469
4243
  # name instruction-shaped passages instead of letting them read as
3470
- # part of the surrounding conversation.
4244
+ # part of the surrounding conversation. Scan before max_chars
4245
+ # truncation: a hostile instruction near the tail of a long page must
4246
+ # not disappear just because the caller asked for a shorter body.
3471
4247
  injection_flags: list[str] = []
4248
+ injection_findings: list[dict[str, Any]] = []
3472
4249
  trust = "untrusted"
3473
4250
  if config.input_gate != "off":
3474
- injection_flags = untrusted.scan(
4251
+ injection_findings = untrusted.scan_findings(
3475
4252
  markdown + "\n" + str(metadata.get("title", ""))
3476
4253
  )
3477
- if injection_flags:
3478
- if config.input_gate == "strict":
3479
- markdown = ""
3480
- trust = "rejected"
3481
- warnings.insert(0, untrusted.withheld(injection_flags))
3482
- else:
3483
- warnings.insert(0, untrusted.warning(injection_flags))
4254
+ injection_flags = untrusted.codes(injection_findings)
4255
+ hard = untrusted.hard_codes(injection_findings)
4256
+ if hard and config.input_gate == "strict":
4257
+ markdown = ""
4258
+ trust = "rejected"
4259
+ warnings.insert(0, untrusted.withheld(hard, injection_findings))
4260
+ elif injection_findings:
4261
+ warnings.insert(0, untrusted.warning(injection_flags, injection_findings))
4262
+
4263
+ if config.max_chars and len(markdown) > config.max_chars:
4264
+ markdown = markdown[: config.max_chars].rstrip() + "\n\n<!-- truncated by web-forager -->"
4265
+ warnings.append(f"markdown truncated to {config.max_chars} characters")
3484
4266
 
3485
4267
  links = []
3486
4268
  if config.extract_links and not is_pdf_payload and not document_kind and not feed_format:
@@ -3514,6 +4296,7 @@ def fetch_page(
3514
4296
  "warnings": warnings,
3515
4297
  "trust": trust,
3516
4298
  "injection_flags": injection_flags,
4299
+ "injection_findings": injection_findings,
3517
4300
  "compliance": {
3518
4301
  "robots_checked": robots_checked,
3519
4302
  "robots_policy": (
@@ -3538,26 +4321,52 @@ def search_warnings(
3538
4321
  """
3539
4322
 
3540
4323
  warnings: list[str] = []
3541
- tokens = _query_tokens(query)
4324
+ tokens = _content_tokens(query)
4325
+ if results:
4326
+ annotate_results(results, query)
3542
4327
  if not results:
3543
4328
  warnings.append(
3544
4329
  f"no results: {engine} returned nothing for {query!r}. Try another "
3545
4330
  "engine (--engine all), a shorter query, or a direct URL."
3546
4331
  )
3547
- elif tokens and max(result_coverage(item, tokens) for item in results) <= 0:
3548
- warnings.append(
3549
- "none of the returned results mention the query text; the engine "
3550
- "most likely split the query into single characters (common with "
3551
- "quoted or CJK names). Treat these results as unrelated and try "
3552
- "another engine or a more specific query."
3553
- )
4332
+ elif tokens and not any(item.get("relevance") for item in results):
4333
+ if max(result_coverage(item, tokens) for item in results) <= 0:
4334
+ warnings.append(
4335
+ "none of the returned results mention the query text; the engine "
4336
+ "most likely split the query into single characters (common with "
4337
+ "quoted or CJK names). Treat these results as unrelated and try "
4338
+ "another engine or a more specific query."
4339
+ )
4340
+ else:
4341
+ warnings.append(
4342
+ "none of the returned results pass the relevance check: each needs "
4343
+ "at least two meaningful query terms, the exact phrase, or an "
4344
+ "equivalent match. Treat these results as unrelated and try another "
4345
+ "engine or a shorter, more specific query."
4346
+ )
4347
+ else:
4348
+ off_topic = [item for item in results if item.get("relevance") is False]
4349
+ if off_topic:
4350
+ warnings.append(
4351
+ f"{len(off_topic)} of {len(results)} results look off-topic: they "
4352
+ "match only one query term or come from the wrong language. They "
4353
+ "are kept at the bottom of the list; check relevance before using "
4354
+ "them as findings."
4355
+ )
3554
4356
  if results:
3555
- probe = " ".join(
3556
- f"{item.get('title', '')} {item.get('snippet', '')}" for item in results[:8]
3557
- )
3558
- flags = untrusted.scan(probe)
3559
- if flags:
3560
- warnings.append(untrusted.warning(flags))
4357
+ probe_findings: list[dict[str, Any]] = []
4358
+ for item in results:
4359
+ findings = untrusted.scan_findings(
4360
+ f"{item.get('title', '')} {item.get('snippet', '')}"
4361
+ )
4362
+ if findings:
4363
+ item["injection_flags"] = untrusted.codes(findings)
4364
+ item["injection_findings"] = findings
4365
+ probe_findings.extend(findings)
4366
+ if probe_findings:
4367
+ warnings.append(
4368
+ untrusted.warning(untrusted.codes(probe_findings), probe_findings)
4369
+ )
3561
4370
  if looks_like_person_name(query):
3562
4371
  warnings.append(
3563
4372
  "the query is a bare personal name. Searching a paper author or a "
@@ -3567,12 +4376,86 @@ def search_warnings(
3567
4376
  return warnings
3568
4377
 
3569
4378
 
4379
+ def search_quality(
4380
+ query: str,
4381
+ results: list[dict[str, Any]],
4382
+ engine: str = "auto",
4383
+ engine_report: Optional[list[dict[str, Any]]] = None,
4384
+ ) -> dict[str, Any]:
4385
+ """A small, machine-readable summary of how good a result list is.
4386
+
4387
+ The prose warnings remain for humans; this block lets an agent decide
4388
+ whether to trust the list without parsing English. Counts describe what
4389
+ was returned, not what the engine could have returned.
4390
+ """
4391
+
4392
+ annotate_results(results, query)
4393
+ intent = classify_intent(query)
4394
+ relevant = sum(1 for item in results if item.get("relevance"))
4395
+ off_topic = sum(1 for item in results if item.get("relevance") is False)
4396
+ language_mismatch = sum(
4397
+ 1 for item in results if item.get("language_match") is False
4398
+ )
4399
+ tiers: dict[str, int] = {}
4400
+ for item in results:
4401
+ tier = str(item.get("authority_tier") or "unknown")
4402
+ tiers[tier] = tiers.get(tier, 0) + 1
4403
+ answered = [
4404
+ str(entry.get("engine", ""))
4405
+ for entry in (engine_report or [])
4406
+ if entry.get("ok") and entry.get("engine")
4407
+ ]
4408
+ errors = [
4409
+ {"engine": entry.get("engine", ""), "error": entry.get("error", "")}
4410
+ for entry in (engine_report or [])
4411
+ if not entry.get("ok")
4412
+ ]
4413
+ engine_status = {
4414
+ str(entry.get("engine", "")): str(
4415
+ entry.get("status") or ("ok" if entry.get("ok") else "error")
4416
+ )
4417
+ for entry in (engine_report or [])
4418
+ if entry.get("engine")
4419
+ }
4420
+ engine_states: dict[str, int] = {}
4421
+ for state in engine_status.values():
4422
+ engine_states[state] = engine_states.get(state, 0) + 1
4423
+ return {
4424
+ "query": query,
4425
+ "engine": engine,
4426
+ "intent": intent["intent"],
4427
+ "intent_confidence": intent["confidence"],
4428
+ "returned": len(results),
4429
+ "relevant": relevant,
4430
+ "off_topic": off_topic,
4431
+ "language_mismatch": language_mismatch,
4432
+ "authority_tiers": tiers,
4433
+ "high_authority": sum(
4434
+ tiers.get(tier, 0)
4435
+ for tier in ("primary", "official", "academic")
4436
+ ),
4437
+ "engines_answered": answered,
4438
+ "engine_errors": errors,
4439
+ "engine_status": engine_status,
4440
+ "engine_states": engine_states,
4441
+ }
4442
+
4443
+
3570
4444
  SITE_SEARCH_TEMPLATES = (
3571
4445
  "/?s={query}",
3572
4446
  "/search?q={query}",
3573
4447
  "/?search={query}",
3574
4448
  "/search?query={query}",
3575
4449
  )
4450
+ # Hosts whose own search page is not one of the generic WordPress/Sphinx
4451
+ # shapes above. These are public search interfaces, not scraped result pages;
4452
+ # the links they return are still fetched through the normal robots-aware path.
4453
+ _SITE_SEARCH_URLS = {
4454
+ "support.microsoft.com": (
4455
+ "https://support.microsoft.com/en-us/search/results?query={query}"
4456
+ ),
4457
+ "developer.mozilla.org": "https://developer.mozilla.org/en-US/search?q={query}",
4458
+ }
3576
4459
 
3577
4460
 
3578
4461
  def find_within_site(
@@ -3613,11 +4496,19 @@ def find_within_site(
3613
4496
  attempts: list[dict[str, Any]] = []
3614
4497
  page: Optional[HttpResponse] = None
3615
4498
  search_url = ""
4499
+ host = (urllib.parse.urlsplit(site_url).hostname or "").lower()
4500
+ preferred = _SITE_SEARCH_URLS.get(host) or _SITE_SEARCH_URLS.get(
4501
+ host.removeprefix("www.")
4502
+ )
4503
+ templates = ((preferred,) if preferred else ()) + SITE_SEARCH_TEMPLATES
3616
4504
  try:
3617
- for template in SITE_SEARCH_TEMPLATES:
3618
- candidate = site_url.rstrip("/") + template.format(
3619
- query=urllib.parse.quote_plus(query)
3620
- )
4505
+ for template in templates:
4506
+ if template.startswith(("http://", "https://")):
4507
+ candidate = template.format(query=urllib.parse.quote_plus(query))
4508
+ else:
4509
+ candidate = site_url.rstrip("/") + template.format(
4510
+ query=urllib.parse.quote_plus(query)
4511
+ )
3621
4512
  started = time.monotonic()
3622
4513
  try:
3623
4514
  response = fetcher.fetch(candidate)
@@ -3830,12 +4721,14 @@ def fetch_by_query(
3830
4721
 
3831
4722
  try:
3832
4723
  engine_report: list[dict[str, Any]] = []
4724
+ quality_data: dict[str, Any] = {}
3833
4725
  results = search(
3834
4726
  query,
3835
4727
  config,
3836
4728
  engine=engine,
3837
4729
  limit=config.num_results,
3838
4730
  report=engine_report,
4731
+ quality=quality_data,
3839
4732
  )
3840
4733
  pages: list[dict[str, Any]] = []
3841
4734
  skipped: list[dict[str, str]] = []
@@ -3889,11 +4782,14 @@ def fetch_by_query(
3889
4782
  skipped.append(
3890
4783
  {"url": result["url"], "code": "unexpected", "reason": str(exc)}
3891
4784
  )
4785
+ quality = search_quality(query, results, engine, engine_report)
4786
+ quality.update(quality_data)
3892
4787
  payload: dict[str, Any] = {
3893
4788
  "query": query,
3894
4789
  "engine": engine,
3895
4790
  "mode": "search",
3896
4791
  "search_results": results,
4792
+ "search_quality": quality,
3897
4793
  "engine_report": engine_report,
3898
4794
  "pages": pages,
3899
4795
  "skipped": skipped,
@@ -4003,6 +4899,31 @@ def redact_url(value: str) -> str:
4003
4899
  return f"{scheme}://{host}{port}" if host else ""
4004
4900
 
4005
4901
 
4902
+ def redact_url_for_log(value: str) -> str:
4903
+ """A URL safe to put in a diagnostic line.
4904
+
4905
+ Query strings carry search terms, signed tokens and session identifiers,
4906
+ and fragments can carry the same. Verbose logging therefore keeps the
4907
+ scheme, host and path only; the caller already has the full URL in the
4908
+ result when it needs it.
4909
+ """
4910
+
4911
+ text = str(value or "").strip()
4912
+ if not text:
4913
+ return ""
4914
+ if "://" not in text:
4915
+ text = "http://" + text
4916
+ try:
4917
+ parts = urllib.parse.urlsplit(text)
4918
+ except ValueError:
4919
+ return ""
4920
+ if not parts.scheme or not parts.netloc:
4921
+ return ""
4922
+ return urllib.parse.urlunsplit(
4923
+ (parts.scheme, parts.netloc, parts.path or "/", "", "")
4924
+ )
4925
+
4926
+
4006
4927
  def doctor_report(config: Optional[FetchConfig] = None) -> dict[str, Any]:
4007
4928
  """Report which capabilities this interpreter can serve, with install hints.
4008
4929