papersearch 0.4.0 → 0.4.3

package/carsi/paper_search_carsi/engine/batch.py

@@ -44,7 +44,8 @@ from .router import (
 from ..publishers.profiles import ACS_PROFILE, PublisherProfile
 from ..entitlements import UserMarkedNoEntitlement  # noqa: E402  (re-exported)
 from ..adapters import PublisherAdapter, adapter_class  # noqa: E402
-from .batch_common import (  # re-exported: other modules import these from batch
+from .batch_common import (
+    page_operation,  # re-exported: other modules import these from batch
     open_page,
     settle,
     _is_network_failure,
@@ -86,7 +87,8 @@ from .batch_pdf_urls import PdfUrlsMixin
 from .batch_runs import RunsMixin
 from .batch_diagnostics import DiagnosticsMixin
 from .batch_user import PdfFoundWhileWaiting, UserProvidedPdf, UserRequestsMixin
-from ..entitlements import UserSkippedArticle
+from ..browser import watchdog
+from ..entitlements import UserSkippedArticle, UserStoppedBatch
 
 
 class PublisherBatchDownloader(
@@ -147,12 +149,13 @@ class PublisherBatchDownloader(
             tab.close()
 
     def collect_supplements(self, context: Any, doi: str, output_dir: Path,
-                            result: "DownloadResult") -> list[dict[str, Any]]:
+                            result: "DownloadResult", *, manifest_dir: Path | None = None) -> list[dict[str, Any]]:
         """Open the article page in a new tab of ``context`` (already logged in)
         and download every supplementary file the adapter lists there."""
         from . import supplements
 
         self._phase(result, "supplements")
+        progress_reporter.update(part="supplements", supplements_saved=0, supplements_total=None)
         try:
             page = context.new_page()
         except Exception as exc:
@@ -162,34 +165,56 @@ class PublisherBatchDownloader(
             # Digest 2026-11-02): no supplements, and the article keeps its result.
             self._event(result, "supplements_skipped", f"browser closed: {type(exc).__name__}")
             return [{"status": "skipped", "reason": "browser_closed", "url": "", "label": ""}]
+        user_watch = self._start_user_watch(page, result)
         try:
-            self._track_article_for_adapter(page, doi)
-            links: list = []
-            # The journal's own article page the main text came from (cell.com,
-            # journal-of-hepatology.eu) lists every file it has: no detour to the
-            # platform, also when it lists none.
-            final_url = str(getattr(result, "final_url", "") or "")
-            own_page = [final_url] if final_url and self.adapter.is_article_page_url(final_url) else []
-            candidates = self.adapter.supplements_page_urls(doi) or [self._article_url(doi)]
-            for url in dict.fromkeys(own_page + candidates):
-                page.goto(url, wait_until="domcontentloaded", timeout=60_000)
-                if self._challenge_shell(page) or self._is_challenge_page(page):
-                    if not self._wait_for_challenge(page, result):
-                        self._event(result, "supplements_failed", "verification not completed")
-                        return [{"status": "failed", "reason": "challenge_timeout", "url": url, "label": ""}]
-                settle(page, 2_000)
-                links = self._settled_supplement_links(page, doi)
-                if links is None:
-                    self._event(result, "supplements_not_supported", self.profile.name)
-                    return []
-                if links or url in own_page:
-                    break
-            self._event(result, "supplements_found", str(len(links)))
+            with watchdog.operation(self.page_wait_sec, "supplement_discovery"):
+                self._track_article_for_adapter(page, doi)
+                links: list = []
+                # The journal's own article page the main text came from (cell.com,
+                # journal-of-hepatology.eu) lists every file it has: no detour to the
+                # platform, also when it lists none.
+                final_url = str(getattr(result, "final_url", "") or "")
+                own_page = [final_url] if final_url and self.adapter.is_article_page_url(final_url) else []
+                candidates = self.adapter.supplements_page_urls(doi) or [self._article_url(doi)]
+                for url in dict.fromkeys(own_page + candidates):
+                    open_page(page, url, timeout_ms=60_000)
+                    if self._challenge_shell(page) or self._is_challenge_page(page):
+                        if not self._wait_for_challenge(page, result):
+                            self._event(result, "supplements_failed", "verification not completed")
+                            return [{"status": "failed", "reason": "challenge_timeout", "url": url, "label": ""}]
+                    settle(page, 2_000)
+                    self._wait_for_page_text(page, result)
+                    links = self._settled_supplement_links(page, doi)
+                    if links is None:
+                        self._event(result, "supplements_not_supported", self.profile.name)
+                        return []
+                    if links or url in own_page:
+                        break
+                self._event(result, "supplements_found", str(len(links)))
+                progress_reporter.update(supplements_total=len(links))
             files = supplements.collect(page.context.request, links, Path(output_dir), doi,
-                                        download=lambda url: self._download_in_new_tab(page, url))
+                                        manifest_dir=manifest_dir,
+                                        target_dir=Path(output_dir) / "supplements" if self.config.get("diagnostics_dir") else None,
+                                        download=lambda url: self._download_in_new_tab(page, url),
+                                        progress=lambda saved: progress_reporter.update(supplements_saved=saved))
             saved = sum(1 for f in files if f.get("status") == "saved")
             self._event(result, "supplements_saved", f"{saved}/{len(files)}")
             return files
+        except PublisherContentError as exc:
+            self._event(result, "supplements_failed", str(exc))
+            return [{"status": "failed", "reason": "publisher_content_error", "url": "", "label": ""}]
+        except watchdog.BrowserUnresponsive as exc:
+            reason = ("browser_operation_timeout" if isinstance(exc, watchdog.BrowserOperationTimeout)
+                      else "browser_unresponsive")
+            self._event(result, "supplements_failed", reason)
+            return [{"status": "failed", "reason": reason, "url": "", "label": ""}]
+        except UserMarkedNoEntitlement as exc:
+            reason = ("batch_stopped" if isinstance(exc, UserStoppedBatch) else
+                      "user_skipped" if isinstance(exc, UserSkippedArticle) else "user_marked_no_entitlement")
+            self._event(result, "supplements_skipped", reason)
+            if not result.ok:
+                result.reason = result.state = reason
+            return [{"status": "skipped", "reason": reason, "url": "", "label": ""}]
         except PublisherAccessBlocked:
             # The publisher paused access (BJSM, 2026-11-01): the article ends as
             # blocked when its main text failed, so the batch stops as for any block.
@@ -203,6 +228,8 @@ class PublisherBatchDownloader(
             self._event(result, "supplements_failed", f"{type(exc).__name__}: {str(exc)[:160]}")
             return [{"status": "failed", "reason": type(exc).__name__, "url": "", "label": ""}]
         finally:
+            user_watch.set()
+            watchdog.clear_interrupt()
             self._close_phase(result)
             try:
                 page.close()
@@ -214,7 +241,13 @@ class PublisherBatchDownloader(
         moves /pii/ to /abs/pii/ by script after the document has loaded)."""
         for attempt in range(3):
             try:
-                return self.adapter.supplement_links(page, doi)
+                from ..adapters.platforms.figshare import supplement_links as external_links
+
+                own = self.adapter.supplement_links(page, doi)
+                external = external_links(page, doi)
+                if own is None and external is None:
+                    return None
+                return list(dict.fromkeys([*(own or []), *(external or [])]))
             except Exception as exc:
                 if (attempt == 2 or not _is_browser_error(exc)
                         or "Execution context was destroyed" not in str(exc)):
@@ -304,6 +337,8 @@ class PublisherBatchDownloader(
             config["browser_headless"] = False
         started = time.time()
         ctx = get_persistent_context(profile_path, config)
+        from ..browser.challenge_interaction import prepare_context
+        prepare_context(ctx)
         self._browser_start = {**dict(getattr(browser_backend, "LAST_LAUNCH", {}) or {}),
                                "seconds": round(time.time() - started, 1), **({"relaunch": relaunch} if relaunch else {})}
         from ..browser.backend import browser_info
@@ -341,11 +376,13 @@ class PublisherBatchDownloader(
         self._item_started = time.time()
         self._item_result = result
         self._record_title = record.title
+        self._expected_title = ""
         self._manual_school = False
         self._track_article_for_adapter(page, record.doi)
         from ..browser import watchdog
 
         self._phase(result, "open")
+        progress_reporter.update(part="main", supplements_saved=0, supplements_total=None)
         self._run_dir = run_dir
         watchdog.on_slow_call = lambda name, seconds, ended, where: self._event(
             result, "wait", f"browser call {name}: {seconds:.0f} s, ended: {ended} ({where})")
@@ -356,46 +393,17 @@ class PublisherBatchDownloader(
             result.browser_start = dict(start)
             self._event(result, "browser_start", self._browser_start_line(start))
         user_watch = self._start_user_watch(page, result)
-        try:
-            self._event(result, "article_open", result.article_url)
-            if not self._ensure_login(page, result):
-                if self._article_access_available(page):
-                    self._event(result, "login_completed_after_timeout", getattr(page, "url", ""))
-                else:
-                    return self._login_failed(page, result, run_dir)
-
-            self._return_from_other_article(page, result)
-            settle(page, 2_000)
-            if not self._pdf_first_unless_signed_out(page, result):
-                self._event(result, "auth_wall_after_article_load", self._decision_text(page))
-                # This item has been through the school's login and the article still
-                # denies it, shows only its abstract or offers it: the school has no
-                # access, and a second login would only wait out the user's time
-                # (JAMA Oncology back from the store page, 2026-11-02).
-                evidence = self._no_access_after_login(page, result)
-                if evidence:
-                    result.reason = result.state = "institution_pdf_entitlement_missing"
-                    self._event(result, "no_access_after_login", evidence[:300])
-                    self._snapshot(page, result, "not subscribed")
-                    result.final_url = page.url
-                    result.title = self._title(page)
-                    self._write_diagnostic(page, result, run_dir)
-                    return result
-                if not self._complete_login_from_current_page(page, result):
-                    if self._article_access_available(page):
-                        self._event(result, "login_completed_after_timeout", getattr(page, "url", ""))
-                    else:
-                        return self._login_failed(page, result, run_dir)
-                settle(page, 2_000)
-                if not self._pdf_first_unless_signed_out(page, result):
-                    return self._login_failed(page, result, run_dir)
+        # Reuse confirmed public access only for this document's initial PDF
+        # attempt. A redirect or reload (even at the same URL) invalidates it.
+        def invalidate_open_access(frame: Any) -> None:
+            if frame == page.main_frame:
+                result._open_access_current = False
 
-            self._return_from_other_article(page, result)
-            result.final_url = page.url
-            result.title = self._title(page)
-            result.state = "article_loaded"
-            self._event(result, "article_access_ready")
-            self._hold_after_login(page, result)
+        page.on("framenavigated", invalidate_open_access)
+        try:
+            preparation = self._prepare_article(page, result, run_dir)
+            if preparation is not None:
+                return preparation
             ip_block: str | None = None
             # Whether this capture comes after the school's login, and its first attempt.
             signed_in, attempts_from = self._school_login_done(), len(result.pdf_attempts)
@@ -512,6 +520,12 @@ class PublisherBatchDownloader(
             self._event(result, "pdf_found_while_waiting" if isinstance(provided, PdfFoundWhileWaiting)
                         else "pdf_from_user_report", provided.url)
             return self._save_captured_pdf(page, record, result, run_dir, provided.body, provided.url)
+        except UserStoppedBatch as decision:
+            watchdog.clear_interrupt()
+            result.reason = result.state = "batch_stopped"
+            self._event(result, "batch_stopped", str(decision))
+            self._write_diagnostic(page, result, run_dir)
+            return result
         except UserSkippedArticle as decision:
             watchdog.clear_interrupt()
             result.reason = result.state = "user_skipped"
@@ -542,6 +556,12 @@ class PublisherBatchDownloader(
             progress_reporter.update(phase="出版社内容服务错误，当前路径已停止")
             self._write_diagnostic(page, result, run_dir)
             return result
+        except watchdog.BrowserUnresponsive as exc:
+            result.reason = result.state = ("browser_operation_timeout" if isinstance(exc, watchdog.BrowserOperationTimeout)
+                                            else "browser_unresponsive")
+            self._event(result, result.state, str(exc))
+            self._write_diagnostic(page, result, run_dir)
+            return result
         except Exception as exc:
             import traceback
 
@@ -563,22 +583,72 @@ class PublisherBatchDownloader(
             self._write_diagnostic(page, result, run_dir)
             return result
         finally:
+            page.remove_listener("framenavigated", invalidate_open_access)
             user_watch.set()
             watchdog.on_slow_call = None
             if isinstance(watchdog.interrupted, UserMarkedNoEntitlement) and result.ok:
                 # The click came while the PDF was being saved.
                 self._tell_user("点击时这一篇已经保存好了，没有跳过", result)
             watchdog.clear_interrupt()
-            self._capture_reported_issues(page, result, act=False)
+            stopped = bool(watchdog.fired) or result.state in {
+                "browser_unresponsive", "browser_operation_timeout", "user_skipped", "batch_stopped", "user_marked_no_entitlement"}
+            if not stopped:
+                self._capture_reported_issues(page, result, act=False)
             if not result.ok:
                 progress_reporter.report_stage("route_failed", current=record.doi)
-            self._hold_after_run(page, result)
+            if not stopped:
+                self._hold_after_run(page, result)
             if _owns_page:
                 try:
                     page.close()
                 except Exception:
                     pass
 
+    @page_operation
+    def _prepare_article(self, page: Any, result: DownloadResult, run_dir: Path) -> DownloadResult | None:
+        """Reach the first PDF attempt within one shared page budget."""
+        self._event(result, "article_open", result.article_url)
+        if not self._ensure_login(page, result):
+            if self._article_access_available(page):
+                self._event(result, "login_completed_after_timeout", getattr(page, "url", ""))
+            else:
+                return self._login_failed(page, result, run_dir)
+
+        self._return_from_other_article(page, result)
+        settle(page, 2_000)
+        if not (getattr(result, "_open_access_current", False)
+                or self._pdf_first_unless_signed_out(page, result)):
+            self._event(result, "auth_wall_after_article_load", self._decision_text(page))
+            # This item has been through the school's login and the article still
+            # denies it, shows only its abstract or offers it: the school has no
+            # access, and a second login would only wait out the user's time
+            # (JAMA Oncology back from the store page, 2026-11-02).
+            evidence = self._no_access_after_login(page, result)
+            if evidence:
+                result.reason = result.state = "institution_pdf_entitlement_missing"
+                self._event(result, "no_access_after_login", evidence[:300])
+                self._snapshot(page, result, "not subscribed")
+                result.final_url = page.url
+                result.title = self._title(page)
+                self._write_diagnostic(page, result, run_dir)
+                return result
+            if not self._complete_login_from_current_page(page, result):
+                if self._article_access_available(page):
+                    self._event(result, "login_completed_after_timeout", getattr(page, "url", ""))
+                else:
+                    return self._login_failed(page, result, run_dir)
+            settle(page, 2_000)
+            if not self._pdf_first_unless_signed_out(page, result):
+                return self._login_failed(page, result, run_dir)
+
+        self._return_from_other_article(page, result)
+        result.final_url = page.url
+        result.title = self._title(page)
+        result.state = "article_loaded"
+        self._event(result, "article_access_ready")
+        self._hold_after_login(page, result)
+        return None
+
     @staticmethod
     def _browser_start_line(start: dict) -> str:
         """``browser_start``'s detail: "profile free; launch 3.1 s", or the holder,
@@ -658,7 +728,8 @@ class PublisherBatchDownloader(
         result.pdf_path = str(pdf_path)
         result.size_bytes = len(pdf_bytes)
         result.text_length = len(text or "")
-        result.verified_match = self._text_matches_record(text, record, fallback_title=result.title)
+        result.verified_match = self._text_matches_record(
+            text, record, fallback_title=self._identity_title() or result.title)
         self._save_publisher_session(page, result)
         if not any(e["state"] == "pdf_saved_unconfirmed" for e in result.events):
             from ..browser.preview import discard_frames

package/carsi/paper_search_carsi/engine/batch_access.py

@@ -267,19 +267,24 @@ class AccessMixin:
         return result is not None and (getattr(result, "_visited_idp", False) or getattr(result, "_login_round_trip", False)
                                        or any(e.get("state") in self._SCHOOL_CHOSEN for e in result.events))
 
-    def _institution_pdf_entitlement_missing(self, page: Any) -> bool:
+    def _institution_pdf_entitlement_missing(self, page: Any, snapshot: SimpleNamespace | None = None) -> bool:
         """The article page denies access (access_rules.denial_evidence) to a
         reader who is the institution: its session shows, this item has been
         through the school's login, or the denial names the configured school."""
         url = str(getattr(page, "url", "") or "")
         if not self._is_success_article_url(url) or self._is_auth_page(url):
             return False
-        if self._is_challenge_page(page):
+        if snapshot is None and self._is_challenge_page(page):
             return False
-        snap = self._access_snapshot(page)
+        snap = snapshot or self._access_snapshot(page)
         evidence, named = access_rules.denial_evidence(url, snap.lines, snap.probe)
         if not evidence:
             return False
+        # A supplied snapshot already rules out a denial without live reads.
+        # Only a possible denial needs the verification guard; re-reading a
+        # slow article here otherwise delays even confirmed public access.
+        if snapshot is not None and self._is_challenge_page(page):
+            return False
         if self._school_login_done() or self._institution_session(page, snap) == "ours":
             return True
         return bool(named) and self._names_configured_school(named)

package/carsi/paper_search_carsi/engine/batch_capture.py

@@ -4,6 +4,8 @@ Part of ``PublisherBatchDownloader`` (batch.py), which combines the mixins."""
 
 from __future__ import annotations
 
+from ..browser import watchdog
+
 import base64
 import inspect
 import json
@@ -18,6 +20,7 @@ import requests
 
 from . import access_rules
 from .batch_common import (
+    page_operation,
     POLL_MS,
     pump_until,
     settle,
@@ -194,6 +197,7 @@ class CaptureMixin:
             self._note_wait(result, "a PDF probe", probe_started, "done")
             self._capture_doi, self._expected_title, self._fallback_pdf = saved
 
+    @page_operation
     def _capture_pdf(self, page: Any, doi: str, result: DownloadResult) -> tuple[bytes | None, str, str | None]:
         """Run the capture channels; a PDF without DOI/title evidence is used
         only when nothing better arrived, and is marked unconfirmed."""
@@ -203,7 +207,10 @@ class CaptureMixin:
         self._fallback_pdf = None
         tried_before = len(result.pdf_attempts)
         try:
-            body, url, block = self._capture_pdf_channels(page, doi, result)
+            from ..browser.preview import pause as pause_preview
+
+            with pause_preview(getattr(page, "context", None)):
+                body, url, block = self._capture_pdf_channels(page, doi, result)
             if body is None and not block and self._fallback_pdf is not None:
                 body, url = self._fallback_pdf
                 self._event(result, "pdf_saved_unconfirmed", f"{url} | no DOI or title on the first pages")
@@ -578,6 +585,22 @@ class CaptureMixin:
                         for pattern in native_pdf_patterns
                     ]})
             self._return_to_record_article_if_needed(page, result, doi)
+            # A login can return directly to a PDF before these listeners were
+            # installed. PDF.js has no article button or metadata to discover.
+            current_pdf_url = str(getattr(page, "url", "") or "")
+            if (self._is_pdf_candidate_url(current_pdf_url) and not self._is_supplementary_url(current_pdf_url)
+                    and self._native_viewer_pdf_allowed(current_pdf_url, **fallback_options)):
+                if entitlement_denied():
+                    return None, "", None
+                if not self._is_chromium(page):
+                    begin("url", current_pdf_url, time.time())
+                    body, final_url = self._capture_native_viewer_pdf(page, current_pdf_url, result, **fallback_options)
+                    if body:
+                        captured["bytes"], captured["url"] = body, final_url
+                        return body, final_url, None
+                    finish()
+            else:
+                current_pdf_url = ""
             # The article page's own PDF links, read before the click: the click can
             # end on a reader page that links no PDF (ASM's "PDF/EPUB", while the
             # article page shows "Download PDF", 2026-11-02).
@@ -591,6 +614,8 @@ class CaptureMixin:
                 pump_until(page, lambda: bool(captured["bytes"] or captured["block_reason"]) or (
                     not self._is_chromium(page) and self._pdf_viewer_present(page)
                     and self._native_viewer_pdf_allowed(str(getattr(page, "url", "") or ""), **fallback_options)), 5_000)
+                if captured["bytes"] or captured["block_reason"]:
+                    return captured["bytes"], str(captured["url"]), captured["block_reason"]
                 self._wait_past_redirect_page(page, result)  # e.g. a sign-in round trip first
                 if not self._resolve_suspect_block(page, result, captured):
                     return None, "", captured["block_reason"]
@@ -621,8 +646,13 @@ class CaptureMixin:
                         captured["url"] = final_url
                     if block_reason and not captured["block_reason"]:
                         captured["block_reason"] = block_reason
-            candidates = (list(bound_pdf_urls) if bound_pdf_urls is not None
-                          else [*metadata_pdf_urls, *self._pdf_candidates(page, doi), *before_click])
+            # A validated PDF is already in hand. Reading the viewer DOM for
+            # more links can block (Royal Society, 2026-11-04) and turn this
+            # successful capture into a page-budget timeout.
+            if captured["bytes"] or captured["block_reason"]:
+                return captured["bytes"], str(captured["url"]), captured["block_reason"]
+            candidates = [current_pdf_url, *(list(bound_pdf_urls) if bound_pdf_urls is not None
+                          else [*metadata_pdf_urls, *self._pdf_candidates(page, doi), *before_click])]
             # In-page anchors (#pane-pcw-references, #addtoany …) are the same
             # document; try each URL once.
             candidates = list(dict.fromkeys(str(url).split("#", 1)[0] for url in candidates if url))
@@ -711,14 +741,21 @@ class CaptureMixin:
                     # Chrome's PDF having been kept out of it (a check at the PDF
                     # address, 2026-11-02).
                     break
+                response_before_wait = captured.get("nav")
                 if not self._wait_for_challenge(page, result):
                     return None, str(getattr(page, "url", "") or ""), captured["block_reason"]
                 if entitlement_denied():
                     return None, "", None
                 pump_until(page, lambda: bool(captured["bytes"]), 3_000)
                 if not captured["bytes"]:
+                    # A new PDF during verification can leave the tab on its
+                    # completion page. Only that new response warrants retrying
+                    # the candidate through the existing validated fallback.
+                    response_after_wait = captured.get("nav") or {}
+                    retry = ([pdf_url] if response_after_wait is not response_before_wait
+                             and str(response_after_wait.get("type", "")).lower() == "application/pdf" else [])
                     body, final_url, block_reason = self._fetch_page_state_pdf(
-                        page, extra_urls=[str(captured["deferred_url"])], **fallback_options)
+                        page, extra_urls=[*retry, str(captured["deferred_url"])], **fallback_options)
                     if body:
                         captured["bytes"] = body
                         captured["url"] = final_url
@@ -845,6 +882,8 @@ class CaptureMixin:
                 self._event(result, "pdf_button_clicked", detail)
                 return True
             except Exception as exc:
+                if not _is_browser_error(exc):
+                    raise
                 self._event(result, "pdf_button_error", f"{type(exc).__name__}: {str(exc).splitlines()[0][:160] if str(exc) else ''}")
         return False
 
@@ -898,12 +937,14 @@ class CaptureMixin:
                 href = ""
                 try:
                     text = control.inner_text(timeout=1_000)
-                except Exception:
-                    pass
+                except Exception as exc:
+                    if not _is_browser_error(exc):
+                        raise
                 try:
                     href = control.get_attribute("href", timeout=1_000) or ""
-                except Exception:
-                    pass
+                except Exception as exc:
+                    if not _is_browser_error(exc):
+                        raise
                 if (self._is_supplementary_url(href)
                         or re.search(r"\b(?:supplement(?:ary|al)?|supporting information|disclosure form)\b", text, re.I)):
                     continue
@@ -913,6 +954,8 @@ class CaptureMixin:
                 self._event(result, "pdf_button_clicked", json.dumps(detail, ensure_ascii=False))
                 return True
             except Exception as exc:
+                if not _is_browser_error(exc):
+                    raise
                 # Kept, not dropped: three such 10 s click timeouts made Sage's silent
                 # 30 s before its PDF (a toolbar copy outside the viewport, 2026-11-02).
                 self._event(result, "pdf_button_error",
@@ -978,6 +1021,8 @@ class CaptureMixin:
                 self._event(result, "pdf_button_clicked", json.dumps(clicked, ensure_ascii=False))
                 return True
         except Exception as exc:
+            if not _is_browser_error(exc):
+                raise
             self._event(result, "pdf_button_error", f"{type(exc).__name__}: {exc}")
         return False
 
@@ -1256,7 +1301,7 @@ class CaptureMixin:
                 # Human verification gets its own budget: the user's time must
                 # not use up the PDF step, and once the check clears the PDF
                 # gets its full time again.
-                challenge_deadline = time.time() + max(self.SUSPECT_BLOCK_WAIT_SEC, timeout_sec)
+                challenge_deadline = time.time() + max(self.page_wait_sec, timeout_sec)
                 if not self._wait_for_challenge_with_deadline(page, result, challenge_deadline):
                     return None, str(getattr(page, "url", "") or "")
                 deadline = time.time() + timeout_sec

package/carsi/paper_search_carsi/engine/batch_common.py

@@ -8,12 +8,16 @@ import re
 import threading
 import time
 from dataclasses import dataclass, field
+from functools import wraps
 from pathlib import Path
 from typing import Any, Callable
 from urllib.parse import urlencode
 
 import requests
 
+from ..browser import watchdog
+from ..browser.challenges import looks_like_challenge
+
 
 EST_ISSN = "1520-5851"
 MIN_PDF_BYTES = 5_000
@@ -23,6 +27,15 @@ PDF_URL_RE = re.compile(r"https?://[^\s\"'<>]+", re.IGNORECASE)
 logger = logging.getLogger(__package__ + ".batch")  # the engine's one channel
 
 
+def page_operation(method):
+    """Bound an engine page operation by its existing page-wait setting."""
+    @wraps(method)
+    def bounded(self, *args, **kwargs):
+        with watchdog.operation(self.page_wait_sec, method.__name__):
+            return method(self, *args, **kwargs)
+    return bounded
+
+
 def _resolve_profile_dir(config: dict[str, Any], explicit: str | Path | None = None) -> Path:
     """Fixed browser profile dir; explicit/config wins, default is data-dir scoped.
 
@@ -121,6 +134,19 @@ def _is_browser_error(exc: BaseException, name: str = "") -> bool:
             and (not name or type(exc).__name__ == name))
 
 
+def page_title(page: Any) -> str:
+    """Read optional title text without consuming the whole page budget."""
+    try:
+        # Page.title has no timeout. A busy renderer held it for 30 s
+        # after access was ready, before the first PDF attempt (2026-11-04).
+        value = page.locator("head > title").first.text_content(timeout=1_000)
+        return re.sub(r"\s+", " ", str(value or "")).strip()
+    except Exception as exc:
+        if not _is_browser_error(exc):
+            raise
+        return ""
+
+
 # A browser call cut off by the page navigating: its document replaced, its frame gone.
 _NAVIGATION_CUT = ("Execution context was destroyed", "Cannot find context with specified id",
                    "Frame was detached", "because of a navigation")
@@ -181,19 +207,6 @@ def pump_until(page: Any, done: Callable[[], bool], timeout_ms: int, step_ms: in
     return bool(done())
 
 
-# A Cloudflare interstitial, recognised from its URL and title alone: scripts
-# reading the page (innerText, DOM queries) can make the check fail and come
-# back in a loop; an untouched page passes. "Attention Required! | Cloudflare" is
-# Cloudflare's block page, not a check.
-CHALLENGE_TITLES = ("just a moment", "请稍候")
-
-
-def looks_like_challenge(url: str, title: str) -> bool:
-    url = str(url or "").lower()
-    return ("__cf_chl" in url or "/cdn-cgi/challenge-platform/" in url
-            or str(title or "").strip().lower().startswith(CHALLENGE_TITLES))
-
-
 # The page's document can be read: parsed (DOMContentLoaded), or its body text has
 # stopped growing. pubs.acs.org showed its article and citation metadata 4.5 s
 # after the request and held DOMContentLoaded back until 33 s on an ad script
@@ -213,7 +226,7 @@ def open_page(page: Any, url: str, *, timeout_ms: int = 60_000, ready_ms: int =
 
     def readable() -> bool:
         try:
-            if looks_like_challenge(str(getattr(page, "url", "") or ""), str(page.title() or "")):
+            if looks_like_challenge(str(getattr(page, "url", "") or ""), page_title(page)):
                 return True
             state = page.evaluate(_DOCUMENT_STATE_JS)
         except Exception as exc: