papersearch 0.4.0 → 0.4.3

package/carsi/paper_search_carsi/adapters/platforms/figshare/__init__.py

@@ -0,0 +1,185 @@
+"""Resolve DOI-bound Figshare supplementary articles/collections via the public API.
+
+Works across publishers; never searches Figshare by title or accepts a reference
+as proof that an unrelated dataset belongs to the paper.
+"""
+from __future__ import annotations
+
+import re
+import time
+from typing import Any
+from urllib.parse import unquote, urlparse
+
+from ...base import SupplementLink
+from ....browser.watchdog import BrowserUnresponsive
+from ....entitlements import UserMarkedNoEntitlement
+
+_API = "https://api.figshare.com/v2"
+_MAX_RESOURCES = 5
+_MAX_ARTICLES = 50
+_MAX_BYTES = 200 * 1024 * 1024
+_BUDGET_SEC = 30
+_PAGE_LINKS = """() => ({
+ doi: document.querySelector('meta[name="citation_doi"]')?.content || '',
+ // A cited dataset is not a supplement of the article citing it.
+ links: [...document.querySelectorAll('a[href], iframe[src]')]
+   .filter(a => !a.closest('[role="doc-bibliography"], [role="doc-biblioentry"], '
+     + '#references, #bibliography, .ref-list, .article-section__references'))
+   .map(a => a.href || a.src)
+})"""
+
+
+def _doi(value: Any) -> str:
+    return re.sub(r"^(?:https?://(?:dx\.)?doi\.org/|doi:\s*)", "", str(value or "").strip(), flags=re.I).lower()
+
+
+def resource(url: str) -> tuple[str, str, str] | None:
+    """Return kind/id/version only for official Figshare URLs and DOI links."""
+    parsed = urlparse(url)
+    if parsed.scheme not in {"http", "https"} or parsed.username or parsed.password:
+        return None
+    host, path = (parsed.hostname or "").lower(), unquote(parsed.path)
+    if host in {"doi.org", "dx.doi.org"}:
+        # Publishers can mint their own Figshare DOI prefixes, e.g.
+        # 10.60893/figshare.adv.c.8713366 (AIP). Check the returned resource
+        # DOI below as well as the parent paper DOI; the numeric ID alone
+        # is never evidence that this landing link denotes that resource.
+        stem = r"/10\.\d{4,9}/(?:m9\.)?figshare(?:\.[a-z][a-z0-9_-]*)*"
+        for kind, suffix in (("collections", r"\.c\."), ("articles", r"\.")):
+            m = re.fullmatch(stem + suffix + r"(\d+)(?:\.v(\d+))?/?", path, re.I)
+            if m:
+                return kind, m[1], m[2] or ""
+        return None
+    if host != "figshare.com" and not host.endswith(".figshare.com"):
+        return None
+    if host == "widgets.figshare.com":
+        widget = re.fullmatch(r"/articles/(\d+)/embed/?", path)
+        if widget:
+            return "articles", widget[1], ""
+    m = re.fullmatch(r"/(articles|collections)/(?:[^/]+/)*(\d+)(?:/(\d+))?/?", path)
+    if not m:
+        return None
+    # A numeric last segment after the resource ID denotes a pinned version.
+    parts = path.strip('/').split('/')
+    if len(parts) >= 3 and parts[-1].isdigit() and parts[-2].isdigit():
+        return parts[0], parts[-2], parts[-1]
+    return m[1], m[2], m[3] or ""
+
+
+def _belongs(data: dict, doi: str) -> bool:
+    resource_doi = _doi(data.get("resource_doi"))
+    if resource_doi:
+        return resource_doi == doi
+    return any(isinstance(item, dict) and item.get("relation") == "IsSupplementTo"
+               and _doi(item.get("identifier")) == doi
+               for item in data.get("related_materials") or [])
+
+
+def supplement_links(page: Any, doi: str, *, clock=time.monotonic, pause=time.sleep) -> list[SupplementLink] | None:
+    """None means no Figshare route on this article; failures stay in the manifest."""
+    data = page.evaluate(_PAGE_LINKS)
+    if not isinstance(data, dict) or _doi(data.get("doi")) != _doi(doi):
+        return None
+    roots: dict[tuple[str, str, str], str] = {}
+    for url in data.get("links", []):
+        if isinstance(url, str) and (key := resource(url)):
+            roots.setdefault(key, url)
+    if not roots:
+        return None
+    started = clock()
+    links: list[SupplementLink] = []
+    fetched: dict[str, Any] = {}
+    last_request: float | None = None
+    doi = _doi(doi)
+
+    def get(path: str) -> Any:
+        nonlocal last_request
+        if path in fetched:
+            return fetched[path]
+        if last_request is not None:
+            pause(max(0, 1 - (clock() - last_request)))
+        remaining = _BUDGET_SEC - (clock() - started)
+        if remaining <= 0:
+            raise ValueError("figshare_time_budget")
+        last_request = clock()
+        response = page.context.request.get(_API + path, timeout=min(10_000, max(1, int(remaining * 1000))))
+        if not response.ok:
+            raise ValueError(f"figshare_http_{response.status}")
+        value = response.json()
+        fetched[path] = value
+        return value
+
+    for index, ((kind, ident, version), root) in enumerate(roots.items()):
+        if index >= _MAX_RESOURCES:
+            links.append(SupplementLink(root, "Figshare resources", error="figshare_resource_limit"))
+            break
+        try:
+            base = f"/{kind}/{ident}" + (f"/versions/{version}" if version else "")
+            detail = get(base)
+            if not isinstance(detail, dict) or str(detail.get("id")) != ident:
+                raise ValueError("figshare_invalid_metadata")
+            if urlparse(root).hostname in {"doi.org", "dx.doi.org"}:
+                requested = unquote(urlparse(root).path).strip("/").lower()
+                actual = _doi(detail.get("doi"))
+                if (requested != actual if version else
+                        re.sub(r"\.v\d+$", "", requested) != re.sub(r"\.v\d+$", "", actual)):
+                    raise ValueError("figshare_resource_doi_mismatch")
+            if not _belongs(detail, doi):
+                raise ValueError("figshare_article_mismatch")
+            articles = [detail]
+            if kind == "collections":
+                # Fetch one more than the collector limit to report truncation.
+                members = get(base + f"/articles?page_size={_MAX_ARTICLES + 1}&page=1")
+                if not isinstance(members, list):
+                    raise ValueError("figshare_invalid_metadata")
+                articles = []
+                for member in members[:_MAX_ARTICLES]:
+                    article_id = member.get("id") if isinstance(member, dict) else None
+                    if not isinstance(article_id, int) or article_id <= 0:
+                        raise ValueError("figshare_invalid_metadata")
+                    member_version = member.get("version")
+                    suffix = f"/versions/{member_version}" if isinstance(member_version, int) and member_version > 0 else ""
+                    item = get(f"/articles/{article_id}{suffix}")
+                    if not isinstance(item, dict) or item.get("id") != article_id:
+                        raise ValueError("figshare_invalid_metadata")
+                    # Collection association is sufficient only if the member does
+                    # not explicitly contradict it; its resource DOI takes precedence.
+                    if item.get("resource_doi") and _doi(item["resource_doi"]) != doi:
+                        raise ValueError("figshare_article_mismatch")
+                    if any(isinstance(rel, dict) and rel.get("relation") == "IsSupplementTo"
+                           and _doi(rel.get("identifier")) != doi
+                           for rel in item.get("related_materials") or []):
+                        raise ValueError("figshare_article_mismatch")
+                    articles.append(item)
+                if len(members) > _MAX_ARTICLES:
+                    links.append(SupplementLink(root, "Figshare collection", error="figshare_article_limit"))
+            count = 0
+            for article in articles:
+                files = article.get("files")
+                if not isinstance(files, list):
+                    raise ValueError("figshare_invalid_metadata")
+                for file in files:
+                    if not isinstance(file, dict) or not isinstance(file.get("id"), int) or file["id"] <= 0:
+                        raise ValueError("figshare_invalid_file")
+                    file_id = file["id"]
+                    # Construct the documented public download endpoint; never
+                    # follow arbitrary external URLs supplied by link-only datasets.
+                    url = f"https://api.figshare.com/v2/file/download/{file_id}"
+                    name = str(file.get("name") or f"Figshare file {file_id}")
+                    error = ""
+                    if file.get("is_link_only"):
+                        error = "figshare_link_only"
+                    elif not isinstance(file.get("size"), int) or file["size"] < 0:
+                        error = "figshare_invalid_file"
+                    elif file["size"] > _MAX_BYTES:
+                        error = "too_large"
+                    links.append(SupplementLink(url, name, error=error))
+                    count += 1
+            if count == 0:
+                raise ValueError("figshare_no_files")
+        except (BrowserUnresponsive, UserMarkedNoEntitlement):
+            raise
+        except Exception as exc:
+            reason = str(exc) if isinstance(exc, ValueError) and str(exc).startswith("figshare_") else "figshare_request_failed"
+            links.append(SupplementLink(root, "Figshare supplementary material", error=reason))
+    return list(dict.fromkeys(links))

package/carsi/paper_search_carsi/adapters/thieme/__init__.py

@@ -16,7 +16,7 @@ from typing import Any
 from urllib.parse import parse_qs, urlencode, urljoin, urlparse
 
 from ...engine.router import is_thieme_article_pdf_url
-from ...engine.batch_common import open_page
+from ...engine.batch_common import _is_browser_error, open_page
 from ..base import PublisherAdapter, register
 
 _HOSTS = frozenset({"thieme-connect.de", "www.thieme-connect.de", "thieme-connect.com", "www.thieme-connect.com"})
@@ -152,13 +152,14 @@ class ThiemeAdapter(PublisherAdapter):
             button = page.locator(
                 "button:has-text('Continue'), input[type='submit'][value='Continue']").first
             try:
-                if button.is_visible(timeout=1_500) is True:
-                    button.click(timeout=5_000, no_wait_after=True)
+                if (button.is_visible(timeout=1_500) is True
+                        and d._click_control(page, button, result, "Thieme federation Continue")):
                     result._federation_continue_at = time.time()
                     d._event(result, "institution_federation_continue", "carsi")
                     return True
-            except Exception:
-                pass
+            except Exception as exc:
+                if not _is_browser_error(exc):
+                    raise
         if not any(event["state"] == "institution_selection_required" for event in result.events):
             d._event(result, "institution_selection_required", "carsi")
         return changed

package/carsi/paper_search_carsi/browser/__init__.py

@@ -1 +1,8 @@
-"""browser tier of paper-search-carsi."""
+"""Browser support for paper-search-carsi.
+
+backend: browser selection, launch and context lifecycle.
+profiles: persistent session state and profile ownership.
+challenges: verification evidence predicates, with no browser or task state.
+watchdog: browser-call cancellation and generic operation budgets.
+engine/login/cookies/preview: browser operations and application integration.
+"""

package/carsi/paper_search_carsi/browser/backend.py

@@ -27,13 +27,27 @@ from __future__ import annotations
 import json
 import logging
 import os
-import re
 import shutil
 import sys
 import time
 from pathlib import Path
 from typing import Any
 
+# Compatibility exports: existing launchers and cold-start tooling use these names.
+# Their implementation is owned solely by profiles.py.
+from .profiles import (
+    PROFILE_RELEASE_TIMEOUT_SEC,
+    SESSION_COOKIES_FILE,
+    SESSION_COOKIES_MAX_AGE_SEC,
+    _lock_file_held,
+    _pid_alive,
+    _profile_owner_pid,
+    _restore_session_cookies,
+    _save_session_cookies,
+    profile_holder,
+    wait_for_profile_release,
+)
+
 logger = logging.getLogger(__name__)
 
 BACKEND_CAMOUFOX = "camoufox"
@@ -450,7 +464,14 @@ def _camoufox_options(headless: bool, proxy: Any, args: list[str] | None, humani
     # comes back as 304 Not Modified, without a body to capture (Springer,
     # 2026-10-31). The browser starts anew for every article, so its memory
     # cache never holds an earlier article's PDF.
-    opts["firefox_user_prefs"] = {"browser.cache.disk.enable": False, **(opts.get("firefox_user_prefs") or {})}
+    # Camoufox 156 with Fission loses cross-process Turnstile frames and sends
+    # input to their parent container. Keep CARSI's owned automation browser
+    # compatible with Juggler; explicit caller preferences still take priority.
+    opts["firefox_user_prefs"] = {
+        "browser.cache.disk.enable": False,
+        "fission.autostart": False,
+        **(opts.get("firefox_user_prefs") or {}),
+    }
     opts.update(headless=headless, humanize=humanize, proxy=proxy)
     pinned = pinned_camoufox_dir()
     if pinned is not None and "executable_path" not in opts:
@@ -535,54 +556,6 @@ def _launch_camoufox_persistent(user_data_dir: str, headless: bool, proxy: Any,
     return _close_with_driver(context, pw)
 
 
-SESSION_COOKIES_FILE = ".paper-search-session-cookies.json"
-SESSION_COOKIES_MAX_AGE_SEC = 12 * 3600
-
-
-def _save_session_cookies(context: Any, profile: Path) -> None:
-    """Keep session cookies (IdP and publisher logins) for the next launch.
-
-    Each paper runs in a new backend process and browser; Firefox drops
-    cookies without an expiry when it closes, so every paper asked for the
-    school login again. Only session cookies are kept: persistent ones are in
-    the profile already. The file is readable by the user only.
-    """
-    try:
-        cookies = [c for c in context.cookies() if not c.get("expires") or c.get("expires", -1) < 0]
-    except Exception:
-        return
-    path = profile / SESSION_COOKIES_FILE
-    try:
-        if not cookies:
-            path.unlink(missing_ok=True)
-            return
-        tmp = path.with_suffix(".tmp")
-        tmp.write_text(json.dumps({"saved_at": time.time(), "cookies": cookies}),
-                       encoding="utf-8")
-        os.chmod(tmp, 0o600)
-        os.replace(tmp, path)
-    except Exception as exc:
-        logger.debug("browser_backend: session cookies not saved: %s", type(exc).__name__)
-
-
-def _restore_session_cookies(context: Any, profile: Path) -> int:
-    path = profile / SESSION_COOKIES_FILE
-    try:
-        data = json.loads(path.read_text(encoding="utf-8"))
-    except Exception:
-        return 0
-    if time.time() - float(data.get("saved_at", 0)) > SESSION_COOKIES_MAX_AGE_SEC:
-        return 0
-    restored = 0
-    for cookie in data.get("cookies") or []:
-        try:
-            context.add_cookies([{k: v for k, v in cookie.items() if k != "expires" or v not in (None, -1)}])
-            restored += 1
-        except Exception:
-            continue
-    return restored
-
-
 def camoufox_profile_dir(user_data_dir: str | Path) -> Path:
     """Camoufox's own profile directory next to the given (Chromium-era) one."""
     path = Path(user_data_dir)
@@ -612,108 +585,11 @@ def launch(
     return _launch_camoufox(headless=headless, proxy=proxy, args=args, humanize=humanize, **kwargs)
 
 
-PROFILE_RELEASE_TIMEOUT_SEC = 20.0
 # The last Camoufox launch of this process, for the article's record: who held
 # the profile, how long the launch waited for it, whether it was released.
 LAST_LAUNCH: dict[str, Any] = {}
 
 
-def _profile_owner_pid(user_data_dir: str) -> int | None:
-    """PID of the browser holding this profile, if any.
-
-    Firefox (Camoufox) leaves a ``lock`` symlink to ``<host>:+<pid>``;
-    Chromium leaves ``SingletonLock`` to ``<host>-<pid>``.
-    """
-    for name in ("lock", "SingletonLock"):
-        try:
-            target = os.readlink(Path(user_data_dir) / name)
-        except OSError:
-            continue
-        match = re.search(r"[-+](\d+)$", target)
-        if match:
-            return int(match.group(1))
-    return None
-
-
-def _lock_file_held(path: Path) -> bool:
-    """Another process holds this lock file: Firefox's ``.parentlock`` on macOS
-    and Linux (an fcntl lock; macOS has no ``lock`` symlink), its ``parent.lock``
-    on Windows (opened without sharing). The lock is tested, never kept."""
-    if not path.exists():
-        return False
-    try:
-        fd = os.open(str(path), os.O_RDWR)
-    except PermissionError:
-        return True  # Windows: the running browser has it open
-    except OSError:
-        return False
-    try:
-        try:
-            import fcntl
-        except ImportError:
-            return False
-        try:
-            fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
-        except OSError:
-            return True
-        fcntl.lockf(fd, fcntl.LOCK_UN)
-        return False
-    finally:
-        os.close(fd)
-
-
-def profile_holder(user_data_dir: str) -> str:
-    """Who still holds this profile ("" = nobody): a live ``lock`` owner, or a
-    lock file another process holds."""
-    pid = _profile_owner_pid(user_data_dir)
-    if pid is not None and _pid_alive(pid):
-        return f"lock of pid {pid}"
-    for name in (".parentlock", "parent.lock"):
-        if _lock_file_held(Path(user_data_dir) / name):
-            return f"{name} held"
-    return ""
-
-
-def _pid_alive(pid: int) -> bool:
-    if pid <= 0 or pid == os.getpid():
-        return False
-    try:
-        os.kill(pid, 0)
-    except ProcessLookupError:
-        return False
-    except PermissionError:
-        return True
-    except OSError:
-        return False
-    return True
-
-
-def wait_for_profile_release(user_data_dir: str, timeout: float = PROFILE_RELEASE_TIMEOUT_SEC) -> bool:
-    """Wait while a previous browser process still owns this persistent profile.
-
-    Back-to-back downloads reuse one profile; the previous Chrome can take a
-    few seconds to exit after its context closed. Launching meanwhile fails,
-    and the fallback launcher then reports a misleading "executable doesn't
-    exist". Returns True when the profile is free (or never locked).
-    """
-    import time
-
-    deadline = time.monotonic() + max(0.0, timeout)
-    waited = False
-    while True:
-        if not profile_holder(user_data_dir):
-            if waited:
-                logger.info("browser_backend: persistent profile released by previous browser")
-            return True
-        if time.monotonic() >= deadline:
-            logger.info("browser_backend: persistent profile still in use by another browser process")
-            return False
-        if not waited:
-            logger.info("browser_backend: waiting for the previous browser to release the profile")
-            waited = True
-        time.sleep(0.5)
-
-
 def launch_persistent_context(
     user_data_dir: str,
     *,

package/carsi/paper_search_carsi/browser/challenge_interaction.py

@@ -0,0 +1,137 @@
+"""One bounded interaction with an observed Turnstile widget.
+
+No navigation, token injection, session transfer or success judgement. The
+engine owns waiting and decides whether verification has actually ended.
+"""
+from __future__ import annotations
+
+import time
+from collections.abc import Callable
+from typing import Any, Literal
+from urllib.parse import urlsplit
+
+
+_OBSERVE_SCRIPT = """() => {
+    const key = Symbol.for('papersearch.turnstile.ready');
+    if (window[key]) return;
+    const ready = window[key] = new WeakSet();
+    addEventListener('message', event => {
+        if (event.origin !== 'https://challenges.cloudflare.com' || !event.source
+            || event.data?.source !== 'cloudflare-challenge') return;
+        if (event.data.event === 'interactiveBegin') ready.add(event.source);
+        else if (['interactiveEnd', 'complete', 'fail', 'tokenExpired']
+                 .includes(event.data.event)) ready.delete(event.source);
+    });
+}"""
+
+
+def observe_checkbox(page: Any) -> None:
+    """Observe readiness only; never read or write the verification response."""
+    page.evaluate(_OBSERVE_SCRIPT)
+
+
+def prepare_context(context: Any) -> None:
+    """Observe each owned page before navigation, including its PDF popups.
+
+    The ego context filters its pages/events to CARSI-owned tabs. Install per
+    page, never on the shared context (which would also affect the user's tabs).
+    """
+    if getattr(context, "_papersearch_challenge_observer", False) is True:
+        return
+
+    def prepare(page: Any) -> None:
+        if getattr(page, "_papersearch_challenge_observer", False) is not True:
+            page.add_init_script("(" + _OBSERVE_SCRIPT + ")()")
+            page._papersearch_challenge_observer = True
+
+    open_page = context.new_page
+    def new_page(*args: Any, **kwargs: Any) -> Any:
+        page = open_page(*args, **kwargs)
+        prepare(page)
+        return page
+
+    context.new_page = new_page
+    context.on("page", prepare)
+    for page in context.pages:
+        prepare(page)
+    context._papersearch_challenge_observer = True
+
+
+def is_turnstile_frame(url: str) -> bool:
+    parsed = urlsplit(url)
+    return (parsed.scheme == "https" and parsed.hostname == "challenges.cloudflare.com"
+            and parsed.path.startswith("/cdn-cgi/challenge-platform/")
+            and "/turnstile/" in parsed.path)
+
+
+def click_checkbox(page: Any, *, timeout_ms: int, checkpoint: Callable[[], None]) -> Literal[
+    "absent", "clicked", "unavailable"
+]:
+    """Try one visible checkbox; never force a click through an overlay.
+
+    A closed shadow root hides the checkbox from locators. Only the normal,
+    compact-height Cloudflare widget with its identifying iframe title can
+    use its standard checkbox position. Expanded puzzles are left alone.
+    """
+    from playwright.sync_api import Error, TimeoutError
+
+    until = time.monotonic() + max(0, timeout_ms) / 1000
+    for frame in page.frames:
+        checkpoint()
+        if time.monotonic() >= until:
+            return "absent"
+        if not is_turnstile_frame(str(frame.url or "")):
+            continue
+        try:
+            # An iframe can be present for seconds while only a spinner is up.
+            # Start observing before inspecting it, so a later call can act.
+            observe_checkbox(page)
+            element = frame.frame_element()
+            if not element.is_visible():
+                continue
+            controls = frame.locator('input[type="checkbox"], [role="checkbox"]')
+            count = controls.count()
+            position = None
+            if count:
+                if count != 1:
+                    return "unavailable"
+                target = controls.first
+                if not target.is_visible() or not target.is_enabled() or target.is_checked():
+                    return "unavailable"
+            else:
+                title = (element.get_attribute("title") or "").lower()
+                box = element.bounding_box()
+                if ("cloudflare" not in title or "challenge" not in title or not box
+                        or not 250 <= box["width"] <= 350 or not 60 <= box["height"] <= 80):
+                    return "unavailable"
+                if not element.evaluate("""iframe =>
+                    window[Symbol.for('papersearch.turnstile.ready')]?.has(iframe.contentWindow) === true
+                """):
+                    return "absent"  # loading/automatic check; no click has been attempted
+                target = element
+                position = {"x": 28, "y": box["height"] / 2}
+            checkpoint()
+            # One readiness signal permits only one closed-shadow click. Wait
+            # for a fresh interactiveBegin before retrying; otherwise a stale
+            # signal could toggle an already checked or processing widget.
+            if position and not element.evaluate("""iframe =>
+                window[Symbol.for('papersearch.turnstile.ready')]?.delete(iframe.contentWindow) === true
+            """):
+                return "absent"
+            remaining_ms = int((until - time.monotonic()) * 1000)
+            if remaining_ms <= 0:
+                return "unavailable"
+            target.click(timeout=remaining_ms, **({"position": position} if position else {}))
+            return "clicked"
+        except TimeoutError:
+            return "unavailable"  # hidden/covered/stale; no repeated forced clicks
+        except Error as exc:
+            # Navigation can remove the widget before its action. Browser close,
+            # crash, watchdog and cancellation errors must still reach the caller.
+            if any(text in str(exc).lower() for text in (
+                "frame was detached", "frame has been detached", "element is not attached",
+                "execution context was destroyed",
+            )):
+                return "unavailable"
+            raise
+    return "absent"

package/carsi/paper_search_carsi/browser/challenges.py

@@ -0,0 +1,57 @@
+"""Browser-independent verification-page recognition from observed page data.
+
+These predicates do not read the browser, wait, report progress or decide
+article access. The caller checks explicit publisher blocks before the broader
+path/text predicates. No match means no recognised challenge, not verified
+access or a successfully completed check.
+"""
+from __future__ import annotations
+
+from urllib.parse import urlparse
+
+
+CHALLENGE_TITLES = ("just a moment", "请稍候")
+_TEXT_MARKERS = (
+    "just a moment",
+    "attention required",
+    "verify you are human",
+    "checking your browser",
+    "verifying your request",
+    "automated-traffic protection",
+    "are you a robot",
+    "please confirm you are a human",
+    "confirm that you are not a robot",
+)
+_CLOUDFLARE_MARKERS = (
+    "ray id:",
+    "security verification",
+    "security service",
+    "not a robot",
+    "瀹夊叏楠岃瘉",
+)
+
+
+def looks_like_challenge(url: str, title: str) -> bool:
+    """Recognise an interstitial without requiring a DOM/body read.
+
+    Reading the body of an active check can interfere with it. In particular,
+    'Attention Required! | Cloudflare' is not this lightweight shell: the engine
+    must first check whether it is an explicit block.
+    """
+    url = str(url or "").lower()
+    return ("__cf_chl" in url or "/cdn-cgi/challenge-platform/" in url
+            or str(title or "").strip().lower().startswith(CHALLENGE_TITLES))
+
+
+def is_challenge_path(url: str) -> bool:
+    """Verification routes, excluding links in queries and article topic paths."""
+    path = urlparse(url).path.lower()
+    return "/crawlprevention/" in path or path == "/captcha" or path.startswith("/captcha/")
+
+
+def has_challenge_text(title: str, text: str) -> bool:
+    """Recognise verification wording after explicit access blocks are excluded."""
+    haystack = f"{title} {text}".lower()
+    return (any(marker in haystack for marker in _TEXT_MARKERS)
+            or ("cloudflare" in haystack
+                and any(marker in haystack for marker in _CLOUDFLARE_MARKERS)))

package/carsi/paper_search_carsi/browser/engine.py

@@ -30,8 +30,8 @@ logger = logging.getLogger(__name__)
 # ---------------------------------------------------------------------------
 # Browser backend (Camoufox via Playwright; ego-lite has its own driver)
 # ---------------------------------------------------------------------------
-# Everything in this module drives the Playwright sync API, which only the
-# Camoufox backend provides. Backend selection lives in backend.py.
+# Everything in this module drives the Playwright sync API, shared by both
+# Camoufox and ego-lite. Backend selection lives in backend.py.
 # ---------------------------------------------------------------------------
 
 from .backend import (  # noqa: E402