papersearch 0.4.0 → 0.4.3

package/carsi/paper_search_carsi/engine/batch_diagnostics.py

@@ -17,7 +17,7 @@ from ..extractors import pdf_extractor
 from ..download_diagnostics import serialize_download_result
 from .. import progress_reporter
 from . import access_rules
-from .batch_common import DownloadResult, PaperRecord, safe_name
+from .batch_common import DownloadResult, PaperRecord, safe_name, _is_browser_error, page_title
 
 
 class DiagnosticsMixin:
@@ -41,12 +41,7 @@ class DiagnosticsMixin:
         finally:
             self._handling_reports = False
 
-    @staticmethod
-    def _title(page: Any) -> str:
-        try:
-            return str(page.title() or "")
-        except Exception:
-            return ""
+    _title = staticmethod(page_title)
 
     @staticmethod
     def _body_text(page: Any, limit: int = 2_000) -> str:
@@ -54,7 +49,9 @@ class DiagnosticsMixin:
         observe_page(page)
         try:
             text = page.locator("body").inner_text(timeout=3_000)
-        except Exception:
+        except Exception as exc:
+            if not _is_browser_error(exc):
+                raise
             return ""
         return re.sub(r"\s+", " ", text).strip()[:limit]
 
@@ -92,7 +89,7 @@ class DiagnosticsMixin:
                 "login_navigation_interrupted": ("登录跳转中断，检查是否可重试", "waiting_login"),
                 "login_stopped": ("机构登录未完成", "route_failed"),
                 "challenge_manual_wait": ("等待人工安全验证", "waiting_verification"),
-                "challenge_wait": ("等待人工安全验证", "waiting_verification"),
+                "challenge_wait": ("正在检查安全验证", "checking_access"),
                 "challenge_resolved": ("安全验证已结束，继续检查页面", "checking_access"),
                 "article_access_ready": ("论文页面已就绪，检查正文入口", "checking_access"),
                 "publisher_content_error": ("出版社返回内容服务错误", "route_failed"),
@@ -101,11 +98,13 @@ class DiagnosticsMixin:
                 "browser_pdf_response_captured": ("PDF 已捕获，正在保存", "pdf_captured"),
                 "pdf_saved": ("PDF 已保存，待内容核验", "saved_pending_verification"),
             }.get(state)
+            if state == "challenge_wait" and detail.startswith("manual "):
+                report = ("等待人工安全验证", "waiting_verification")
             if state == "login_attempt_started" and detail in {"1/3", "2/3", "3/3"}:
                 report = (f"机构登录：第 {detail[0]} 次尝试，共 3 次", "waiting_login")
             if state == "login_observed":
                 if detail == "verification":
-                    report = ("等待人工安全验证", "waiting_verification")
+                    report = ("正在检查安全验证", "checking_access")
                 elif detail == "human_login":
                     report = ("等待在浏览器完成机构登录", "waiting_login")
                 else:
@@ -189,13 +188,19 @@ class DiagnosticsMixin:
             "final_url": getattr(page, "url", result.final_url)}, diagnostic=True)
         packet["adapter"] = type(self.adapter).__name__
         packet["browser"] = getattr(self, "browser", {})
-        try:
-            packet["access_state"], packet["access_evidence"] = self._access_state(page)
-        except Exception as exc:
-            packet["access_state"], packet["access_evidence"] = "unknown", [f"probe failed: {type(exc).__name__}"]
+        from ..browser import watchdog
+        stopped = bool(watchdog.fired) or result.state in {
+            "browser_unresponsive", "browser_operation_timeout", "user_skipped", "batch_stopped", "user_marked_no_entitlement"}
+        if stopped:
+            packet["access_state"], packet["access_evidence"] = "unknown", ["probe skipped: item stopped"]
+        else:
+            try:
+                packet["access_state"], packet["access_evidence"] = self._access_state(page)
+            except Exception as exc:
+                packet["access_state"], packet["access_evidence"] = "unknown", [f"probe failed: {type(exc).__name__}"]
         packet_path = diag_dir / "diagnostic.json"
         packet_path.write_text(json.dumps(packet, ensure_ascii=False, indent=2), encoding="utf-8")
-        if self.adapter.diagnostic_screenshots and result.reason != "publisher_access_blocked":
+        if not stopped and self.adapter.diagnostic_screenshots and result.reason != "publisher_access_blocked":
             try:
                 page.screenshot(path=str(diag_dir / "screenshot.png"), full_page=True)
             except Exception:

package/carsi/paper_search_carsi/engine/batch_login.py

@@ -8,6 +8,7 @@ import json
 import re
 import time
 from typing import Any
+from types import SimpleNamespace
 from urllib.parse import unquote, urljoin, urlparse
 
 from . import access_rules
@@ -65,20 +66,23 @@ class LoginMixin:
         self._wait_past_redirect_page(page, result)
         self._raise_on_publisher_content_error(page)
         self._dismiss_cookie_banners(page, result)
-        if self._institution_pdf_entitlement_missing(page):
+        snapshot = self._access_snapshot(page) if not self._is_auth_page(str(page.url)) else None
+        if self._institution_pdf_entitlement_missing(page, snapshot=snapshot):
             return False
-        if self._pdf_first_unless_signed_out(page, result):
+        if self._pdf_first_unless_signed_out(page, result, snapshot=snapshot):
             return True
         return self._complete_login_from_current_page(page, result)
 
-    def _pdf_first_unless_signed_out(self, page: Any, result: DownloadResult) -> bool:
+    def _pdf_first_unless_signed_out(self, page: Any, result: DownloadResult,
+                                   snapshot: SimpleNamespace | None = None) -> bool:
         """True unless the page says the reader is signed out (access_rules OUT).
 
         With the institution's session, or an open-access article, the PDF is
         taken. With nothing either way, or with only the reader's access (also
         a personal account's or ClinicalKey's, J Hepatol 2026-10-31), the PDF is
         tried first and the school's login follows once if it fails (batch.py)."""
-        verdict, basis, evidence = self._access_decision(page)
+        result._open_access_current = False
+        verdict, basis, evidence = self._access_decision(page, snapshot) if snapshot is not None else self._access_decision(page)
         if verdict != access_rules.UNKNOWN:
             detail = f"{verdict} ({basis}): {evidence}"[:240]
             if next((e["detail"] for e in reversed(result.events) if e["state"] == "access_decision"), None) != detail:
@@ -86,6 +90,7 @@ class LoginMixin:
         if verdict == access_rules.OUT:
             return False
         open_to_all = basis == "reader" and access_rules.OPEN_TO_ALL.search(evidence.split("badge: ", 1)[-1])
+        result._open_access_current = bool(open_to_all)
         if (basis != "session" and not open_to_all and not getattr(result, "_pdf_first", False)
                 and not self._school_login_done()):
             result._pdf_first = True
@@ -122,32 +127,33 @@ class LoginMixin:
             return url[:200]
         return f"{url[:200]} | {verdict} ({basis or 'no rule'}): {evidence[:160]}"
 
-    # How long a page that shows no text yet is given to draw it.
-    PAGE_TEXT_WAIT_SEC = 20
     _PAGE_TEXT_JS = "() => document.contentType !== 'text/html' ? -1 : (document.body ? document.body.innerText.trim().length : 0)"
 
     def _wait_for_page_text(self, page: Any, result: DownloadResult) -> None:
-        """Let a page without any text yet draw it before it is judged: De Gruyter's
-        article page stayed blank for 9 s and was taken for one without an
-        institution entry (2026-11-02). Still blank after ``PAGE_TEXT_WAIT_SEC``:
-        the publisher's page did not load (``PublisherContentError``)."""
+        """Wait for the document before judging access, within the page budget.
+
+        A verification can disappear before the article draws. The old fixed
+        20 s cut this wait short even with automation time left (AIP,
+        2026-11-04). The enclosing page operation still bounds the whole step.
+        """
         def shown() -> bool:
             try:
                 return page.evaluate(self._PAGE_TEXT_JS) != 0
             except Exception as exc:
                 if not _is_browser_error(exc):
                     raise
-                return True  # a page navigating meanwhile is judged by the steps after
+                return False  # a document being replaced has not been read yet
         started = time.time()
         if shown():
             return
-        drawn = pump_until(page, shown, self.PAGE_TEXT_WAIT_SEC * 1000, step_ms=500)
+        timeout_sec = self.page_wait_sec
+        drawn = pump_until(page, shown, int(timeout_sec * 1000), step_ms=500)
         # Recorded however short: a blank page is worth knowing about.
         self._event(result, "wait", f"blank page: {time.time() - started:.0f} s, ended: "
-                                    f"{'text shown' if drawn else f'its {self.PAGE_TEXT_WAIT_SEC} s'}")
+                                    f"{'text shown' if drawn else f'its {timeout_sec:g} s'}")
         if not drawn:
             url = str(getattr(page, "url", "") or "")
-            raise PublisherContentError(f"publisher_page_blank: {url[:160]} showed no text for {self.PAGE_TEXT_WAIT_SEC} s")
+            raise PublisherContentError(f"publisher_page_blank: {url[:160]} showed no text for {timeout_sec:g} s")
 
     def _wait_past_redirect_page(self, page: Any, result: DownloadResult, timeout_sec: float = 20) -> None:
         """Let a publisher's script redirect page (linkinghub "Redirecting") move on
@@ -187,9 +193,12 @@ class LoginMixin:
         self._phase(result, "login")
         started, signed_in = time.time(), False
         try:
-            signed_in = self._complete_login_steps(page, result)
-            if signed_in:
-                self._return_to_article_after_login(page, result)
+            from ..browser import watchdog
+            # Human authentication has its own budget, not the automation's.
+            with watchdog.pause_operation():
+                signed_in = self._complete_login_steps(page, result)
+                if signed_in:
+                    self._return_to_article_after_login(page, result)
             return signed_in
         finally:
             last = next((e["state"] for e in reversed(result.events) if e["state"] != "wait"), "")
@@ -260,6 +269,7 @@ class LoginMixin:
         last_auto_action_at = 0.0
         visited_idp = False
         page_since, page_inputs = time.time(), ()
+        outside_since = time.time()  # last seen on the user's own login step
         article_returns = 0
         session_returns = 0
         away_from_publisher = False  # on an IdP, federation or chooser page since the last probe
@@ -273,6 +283,14 @@ class LoginMixin:
                 result.reason = "institution_federation_unavailable"
                 self._event(result, result.reason, "the school is not in the publisher's institution list")
                 return False
+            over, outside_since = self._page_wait_over(page, outside_since)
+            if over:
+                # Off the school's login page for a whole page wait (a publisher
+                # page, a chooser, a redirect that goes nowhere): nobody is typing.
+                self._event(result, "page_wait_ended",
+                            f"{self._login_wait_page(page)} after {self.page_wait_sec:.0f} s")
+                self._wait_ran_out(result)
+                return False
             marker = f"{self._title(page)} | {getattr(page, 'url', '')[:160]}"
             here = urlparse(str(getattr(page, "url", "") or ""))
             on_publisher = self._on_publisher_host(here.geturl())
@@ -622,17 +640,23 @@ class LoginMixin:
         host = urlparse(self._entity_id()).hostname or getattr(result, "_idp_host", "")
         return access_rules.registrable_domain(host) if host else ""
 
-    def _school_or_federation_page(self, url: str, result: DownloadResult | None = None) -> bool:
+    def _school_or_federation_page(self, url: str, result: DownloadResult | None = None, *,
+                                   include_publisher_chooser: bool = True) -> bool:
         """The school's IdP or another page of its own site (its registrable domain),
-        or a federation's or the publisher's school chooser: the user's part of the
-        login, where the engine answers no banner and leaves the page where it is."""
+        or a federation's school chooser. Publisher-owned choosers are included
+        by default; their cookie prompts can be answered separately. The school's
+        own pages remain protected even if an adapter calls one a chooser."""
         host = (urlparse(url).hostname or "").lower()
         if not host:
             return False
-        if self._is_idp_host(host) or self._is_federation_chooser(url):
+        if self._is_idp_host(host):
             return True
         school = self._school_domain(result)
-        return bool(school) and access_rules.registrable_domain(host) == school
+        if school and access_rules.registrable_domain(host) == school:
+            return True
+        if not include_publisher_chooser and self.adapter.is_institution_chooser(url):
+            return False
+        return self._is_federation_chooser(url)
 
     def _stuck_on_school_site(self, page: Any, result: DownloadResult) -> str:
         """The page's URL when the login has stayed on a page of the school's own
@@ -789,7 +813,9 @@ class LoginMixin:
         for (const el of document.querySelectorAll('button, [role="button"], a')) {
             if (!shown(el) || el.closest('form')) continue;
             const label = (el.innerText || el.getAttribute('aria-label') || '').replace(/\s+/g, ' ').trim();
-            if (!MENU.test(label)) continue;
+            // A screen-reader hint can be part of innerText ("Sign In Open Menu").
+            const menuLabel = label.replace(/\s+open menu$/i, '');
+            if (!MENU.test(menuLabel)) continue;
             const href = (el.getAttribute('href') || '').trim();
             if (el.tagName === 'A' && href && !href.startsWith('#') && !/^javascript:/i.test(href)
                     && !TOGGLES.some(name => el.hasAttribute(name))) continue;
@@ -1143,6 +1169,26 @@ class LoginMixin:
             return True
         return self.adapter.is_article_page_url(url) is True
 
+    def _page_wait_over(self, page: Any, outside_since: float) -> tuple[bool, float]:
+        """Whether a login wait has been off the user's own login step for a whole
+        page wait, and when that step was last seen (now, when on it)."""
+        now = time.time()
+        if self._users_login_step(page):
+            return False, now
+        return now - outside_since >= self.page_wait_sec, outside_since
+
+    def _users_login_step(self, page: Any) -> bool:
+        """The user's own step of the login, which may take the long wait
+        (``login_timeout_sec``): the school's login page (a password field, or the
+        school's identity provider), or a school chooser when no school is
+        configured or the user chooses by hand. Any other page off the publisher's
+        site is not: Atypon's iam.atypon.com hub, where AHA's login stopped on
+        2026-11-03, was taken for the school's page and waited three minutes."""
+        url = str(getattr(page, "url", "") or "")
+        if self._school_password_box(page, url) or self._is_idp_host(urlparse(url).netloc):
+            return True
+        return self._is_auth_page(url) and (not self.institution_query or bool(getattr(self, "_manual_school", False)))
+
     def _login_wait_page(self, page: Any) -> str:
         """Where a login that waited on the user ended (ADR 0012):
 
@@ -1155,7 +1201,9 @@ class LoginMixin:
         if page.is_closed() is True:
             return "closed"
         url = str(getattr(page, "url", "") or "")
-        if self._is_human_login_page(page):
+        # The school's own page, by the same test as the long wait: a publisher's
+        # SSO hub (iam.atypon.com) is no school login (2026-11-03).
+        if self._school_password_box(page, url) or self._is_idp_host(urlparse(url).netloc):
             return "school_login"
         if self._is_auth_page(url):
             return "institution_chooser" if self.institution_query else "school_choice"
@@ -1165,13 +1213,17 @@ class LoginMixin:
         current_url = str(getattr(page, "url", "") or "")
         host = (urlparse(current_url).netloc or "").lower()
         if self.config.get("publisher_institution_federation") == "carsi":
-            if (page.locator('input[type="password"]:visible, input[autocomplete="one-time-code"]:visible').count()
-                    and not self._personal_login_box(page, current_url)):
+            if self._school_password_box(page, current_url):
                 return True
             if not self._on_publisher_host(current_url):
                 return not self._is_federation_chooser(current_url)
         return self._is_idp_host(host)
 
+    def _school_password_box(self, page: Any, url: str) -> bool:
+        """A password or one-time-code field that is not a personal account's box on the publisher's site."""
+        return bool(page.locator('input[type="password"]:visible, input[autocomplete="one-time-code"]:visible').count()
+                    and not self._personal_login_box(page, url))
+
     def _personal_login_box(self, page: Any, url: str) -> bool:
         """The password field is a personal account's login box on the publisher's
         own article page, nobody typing in it: BMJ shows one next to "Log in via
@@ -1339,8 +1391,9 @@ class LoginMixin:
     def _dismiss_cookie_banners(self, page: Any, result: DownloadResult | None = None, *,
                                 blocking: bool = False) -> bool:
         """Answer a cookie banner (``_COOKIE_BANNER_JS``) in the page or one of its
-        frames, at most ``BANNER_ANSWERS`` times per banner and page, never on the
-        school's or a federation's pages (``_school_or_federation_page``);
+        frames, at most ``BANNER_ANSWERS`` times per banner and page. The school's
+        and external federation pages stay protected; a publisher-owned chooser
+        may answer its own cookie prompt (``_school_or_federation_page``).
         ``blocking``: the banner covered a control the engine clicked. An
         advertisement over the page is closed first (``_close_advertisement_overlay``).
         True when either was done."""
@@ -1361,12 +1414,14 @@ class LoginMixin:
             pass
         for target in targets:
             where = str(getattr(target, "url", "") or "").split("#", 1)[0]
-            if self._school_or_federation_page(where, item):
+            if self._school_or_federation_page(where, item, include_publisher_chooser=False):
                 continue
             skip = [key for (url, key), count in answered.items() if url == where and count >= self.BANNER_ANSWERS]
             try:
                 clicked = target.evaluate(self._COOKIE_BANNER_JS, {"blocking": bool(blocking), "skip": skip})
-            except Exception:
+            except Exception as exc:
+                if not _is_browser_error(exc):
+                    raise
                 continue  # a frame gone meanwhile; the banner is a convenience, never the item's end
             if not isinstance(clicked, dict):
                 continue

package/carsi/paper_search_carsi/engine/batch_runs.py

@@ -610,13 +610,8 @@ class RunsMixin:
         if not any(kw in url_lower for kw in ("login", "cas", "sso", "signin", "sign-in")):
             return True
         # Check for SSO-specific indicators
-        try:
-            title = (page.title() or "").lower()
-            if any(kw in title for kw in ("登录成功", "welcome", "dashboard")):
-                return True
-        except Exception:
-            pass
-        return False
+        title = self._title(page).lower()
+        return any(kw in title for kw in ("登录成功", "welcome", "dashboard"))
 
     def _save_publisher_session(self, page: Any, result: DownloadResult) -> None:
         if not self.profile.base_domains:

package/carsi/paper_search_carsi/engine/batch_waits.py

@@ -7,12 +7,13 @@ from __future__ import annotations
 import inspect
 import re
 import time
+from contextlib import ExitStack
 from typing import Any
 from urllib.parse import urlparse
 
 from ..entitlements import UserMarkedNoEntitlement
-from .batch_common import (AWAY_WAIT_SEC, DownloadResult, _IP_BLOCK_STATUS_CODES, _observe_preview,
-                           looks_like_challenge)
+from ..browser import challenge_interaction, challenges, watchdog
+from .batch_common import (AWAY_WAIT_SEC, DownloadResult, _IP_BLOCK_STATUS_CODES, _observe_preview)
 
 
 class UserWaitMixin:
@@ -35,7 +36,16 @@ class UserWaitMixin:
             return self._wait_for_challenge(page, result, deadline=deadline)
         return self._wait_for_challenge(page, result)
 
-    SUSPECT_BLOCK_WAIT_SEC = 180
+    # A wait off the school's login page (verification, the publisher's pages and
+    # choosers, the PDF step): config ``browser_page_wait_timeout``, else this.
+    PAGE_WAIT_SEC = 60
+
+    @property
+    def page_wait_sec(self) -> float:
+        """How long a wait that is not the user's own login step may last; the
+        school's login page has ``login_timeout_sec`` (user's choice, 2026-11-03)."""
+        value = (getattr(self, "config", None) or {}).get("browser_page_wait_timeout")
+        return self.PAGE_WAIT_SEC if value is None else max(10.0, float(value))
 
     @staticmethod
     def _visible_text_excerpt(body: str | bytes | None, limit: int = 600) -> str:
@@ -103,7 +113,11 @@ class UserWaitMixin:
 
     def _wait_out_suspect_block(self, page: Any, result: DownloadResult, captured: dict[str, Any], suspect: str,
                                 suspect_host: str, started: float) -> bool:
-        deadline = started + min(max(self.login_timeout_sec, 30), self.SUSPECT_BLOCK_WAIT_SEC)
+        deadline = started + self.page_wait_sec
+        automatic: dict[str, Any] = {}
+        interaction = ExitStack()
+        manual = False
+        state = ""
         reporter = None
         attention_key = f"pdf-step:{id(self)}:{result.doi}"
         try:
@@ -147,6 +161,8 @@ class UserWaitMixin:
                         if self._pdf_step_needs_login(candidate):
                             state, tab = "login", candidate
                             break
+                    except (watchdog.BrowserUnresponsive, UserMarkedNoEntitlement):
+                        raise
                     except Exception:
                         continue
                 if not state and time.time() - started < self.SUSPECT_UNKNOWN_GRACE_SEC:
@@ -158,8 +174,8 @@ class UserWaitMixin:
                         except Exception:
                             continue
                 if not state:
-                    if shown:
-                        self._event(result, "challenge_resolved", f"{shown} page gone")
+                    if shown or automatic:
+                        self._event(result, "challenge_resolved", f"{shown or 'verification'} page gone")
                     captured["suspect_block"] = None
                     return True  # the capture loop retries the PDF now
                 if state == "login" and tab == page:
@@ -170,6 +186,13 @@ class UserWaitMixin:
                     self._event(result, "pdf_step_login_required", str(getattr(page, "url", "") or ""))
                     captured["suspect_block"] = None
                     return False
+                if state == "verification":
+                    self._raise_on_publisher_access_blocked(tab)
+                    if not automatic:
+                        deadline = min(deadline, self._wait_deadline(result))
+                    if self._automatic_challenge_step(tab, result, automatic, deadline):
+                        page.wait_for_timeout(min(1000, max(0, (deadline - time.time()) * 1000)))
+                        continue
                 if state != shown:
                     if state in ("verification", "login"):
                         deadline = min(deadline, self._wait_deadline(result))
@@ -189,14 +212,19 @@ class UserWaitMixin:
                         except Exception:
                             pass
                     shown = state
+                if not manual:
+                    interaction.enter_context(watchdog.pause_operation())
+                    manual = True
                 try:
                     _observe_preview(page)  # what the user is asked to act on
-                    page.wait_for_timeout(1000)  # pump browser events while the user acts
+                    page.wait_for_timeout(min(1000, max(0, (deadline - time.time()) * 1000)))
+                except (watchdog.BrowserUnresponsive, UserMarkedNoEntitlement):
+                    raise
                 except Exception:
-                    time.sleep(1)
-            if shown in ("verification", "login"):
+                    time.sleep(min(1, max(0, deadline - time.time())))
+            if state in ("verification", "login"):
                 self._wait_ran_out(result)
-            if shown == "verification":
+            if state == "verification":
                 # The verification page stayed up: nobody completed it. That
                 # is a timeout, not evidence that the publisher blocked us.
                 captured["block_reason"] = "challenge_or_viewer_timeout"
@@ -206,6 +234,7 @@ class UserWaitMixin:
             self._event(result, "ip_blocked_response", f"{suspect} | no PDF after waiting ({shown or 'nothing shown'})")
             return False
         finally:
+            interaction.close()
             if shown and reporter is not None:
                 try:
                     reporter.clear_attention(attention_key)
@@ -302,9 +331,10 @@ class UserWaitMixin:
         return deadline
 
     def _wait_deadline(self, result: DownloadResult, deadline: float | None = None) -> float:
-        """End of a wait on the user that starts now: the caller's bound, the
-        article's total, and when the user seems away the short wait."""
-        bounds = [self._user_wait_deadline(result)]
+        """End of a wait on the user that starts now, off the school's login page
+        (verification pages): the caller's bound, the article's total, the page
+        wait, and when the user seems away the short wait."""
+        bounds = [self._user_wait_deadline(result), time.time() + self.page_wait_sec]
         if deadline is not None:
             bounds.append(deadline)
         if self._user_away:
@@ -317,8 +347,43 @@ class UserWaitMixin:
             result._user_wait_expired = True
             self._event(result, "user_wait_expired", "short wait" if self._user_away else "")
 
+    def _automatic_challenge_step(self, page: Any, result: DownloadResult,
+                                  attempt: dict[str, Any], deadline: float) -> bool:
+        """A short automatic phase inside the existing wait, shared by both entry points.
+
+        True means keep observing, never that verification passed. Each wait
+        allows at most three interactions, including failed clicks, without a
+        new budget. Observe for at least three seconds after each attempt.
+        """
+        now = time.time()
+        if not attempt:
+            attempt.update(started=now, attempts=0, retry_at=now + 2)
+            self._event(result, "challenge_wait", "automatic verification check")
+        # Live widgets may not ask for a click until ~14 s after navigation.
+        # Reserve up to 2 s for manual handling without extending short waits.
+        manual_reserve = min(2, max(0, deadline - attempt["started"]) * 0.2)
+        automatic_end = min(deadline - manual_reserve, attempt["started"] + 20)
+        if now >= automatic_end:
+            return False
+        if now >= attempt["retry_at"] and attempt["attempts"] < 3:
+            # Camoufox's native humanized move can take 1.5 s before mouse-down.
+            # Leave time for that action, still inside the original wait budget.
+            outcome = challenge_interaction.click_checkbox(
+                page, timeout_ms=min(2500, max(1, int((automatic_end - now) * 1000))),
+                checkpoint=lambda: self._check_user_decision(result, page),
+            )
+            if outcome != "absent":
+                attempt["attempts"] += 1
+                attempt["retry_at"] = time.time() + 3
+                self._event(result, "challenge_wait",
+                            f"checkbox {outcome}; attempt {attempt['attempts']}/3; awaiting verification")
+        return time.time() < automatic_end
+
     def _wait_for_challenge(self, page: Any, result: DownloadResult, *, deadline: float | None = None) -> bool:
         """Wait while a verification page is up, until ``_wait_deadline``."""
+        interaction = ExitStack()
+        automatic: dict[str, Any] = {}
+        manual = False
         wait_interval_sec = 1  # a cleared verification goes on within a second
         reporter = None
         attention_key = f"publisher-challenge:{id(self)}:{result.doi}"
@@ -342,6 +407,14 @@ class UserWaitMixin:
                     if not waited:
                         deadline = self._wait_deadline(result, deadline)
                         started, phase_before = time.time(), self._phase(result, "verification")
+                    waited = True
+                    result.state = "challenge_or_viewer_timeout"
+                    if time.time() >= deadline:
+                        break
+                    auto_pending = self._automatic_challenge_step(page, result, automatic, deadline)
+                    if not auto_pending and not manual:
+                        interaction.enter_context(watchdog.pause_operation())
+                        manual = True
                         self._event(result, "challenge_manual_wait", "complete verification in visible browser")
                         if not any(s["why"] == "waiting for the user: verification" for s in result.snapshots):
                             self._snapshot(page, result, "waiting for the user: verification")
@@ -353,10 +426,8 @@ class UserWaitMixin:
                                 phase="人工验证",
                             )
                             attention_open = True
-                    waited = True
-                    result.state = "challenge_or_viewer_timeout"
                     checks += 1
-                    self._event(result, "challenge_wait", str(checks))
+                    self._event(result, "challenge_wait", f"{'manual' if manual else 'automatic'} {checks}")
                     # Pump browser events while the user completes verification.
                     # Plain sleep can leave sync-API navigation state stale.
                     wait_ms = min(wait_interval_sec * 1000, max(0, (deadline - time.time()) * 1000))
@@ -379,6 +450,7 @@ class UserWaitMixin:
                 self._wait_ran_out(result)
             return passed
         finally:
+            interaction.close()
             if waited:
                 self._phase(result, phase_before or "open")
                 self._note_wait(result, "verification", started, "passed" if result.state != "challenge_or_viewer_timeout"
@@ -387,45 +459,23 @@ class UserWaitMixin:
                 reporter.clear_attention(attention_key)
 
     def _challenge_shell(self, page: Any) -> bool:
-        """A Cloudflare interstitial recognised from title and URL alone.
-
-        Scripts reading the challenge page (innerText, DOM queries) can make
-        Cloudflare fail and re-issue the check (``__cf_chl_rt_tk``) in a loop;
-        an untouched page passes. Callers waiting on a challenge use this and
-        nothing else.
-        """
+        """Read only URL/title while a verification interstitial is active."""
         try:
             url = str(getattr(page, "url", "") or "")
         except Exception:
             url = ""
-        return looks_like_challenge(url, self._title(page))
+        return challenges.looks_like_challenge(url, self._title(page))
 
     def _is_challenge_page(self, page: Any) -> bool:
+        """Apply engine block precedence before the independent recogniser.
+
+        Read lazily: an interstitial or crawl-prevention URL needs no body read;
+        an explicit publisher block must never turn into a verification wait.
+        """
         if self._challenge_shell(page):
             return True
         if self._publisher_access_blocked(page):
             return False
-        # Silverchair sites (OUP and others) send unusual traffic to their own
-        # reCAPTCHA page, /crawlprevention/governor?content=<the article> ("Validate
-        # User", 2026-11-01); it leads back to the article once the user passes it.
-        if "/crawlprevention/" in urlparse(str(getattr(page, "url", "") or "")).path.lower():
+        if challenges.is_challenge_path(str(getattr(page, "url", "") or "")):
             return True
-        haystack = f"{self._title(page)} {self._body_text(page, 1_200)}".lower()
-        direct_markers = (
-            "just a moment",
-            "attention required",
-            "verify you are human",
-            "checking your browser",
-            "are you a robot",
-            "please confirm you are a human",
-            "confirm that you are not a robot",
-        )
-        if any(marker in haystack for marker in direct_markers):
-            return True
-        return "cloudflare" in haystack and (
-            "ray id:" in haystack
-            or "security verification" in haystack
-            or "security service" in haystack
-            or "not a robot" in haystack
-            or "瀹夊叏楠岃瘉" in haystack
-        )
+        return challenges.has_challenge_text(self._title(page), self._body_text(page, 1_200))