papersearch 0.0.0-stage → 0.4.2

package/carsi/paper_search_carsi/adapters/acm/__init__.py

@@ -0,0 +1,13 @@
+"""On the Atypon platform (``platforms/atypon``): only its hosts are its own."""
+from __future__ import annotations
+
+from ..base import register
+from ..platforms.atypon import AtyponAdapter
+
+
+@register
+class ACMAdapter(AtyponAdapter):
+    """The ACM Digital Library on dl.acm.org: on Atypon's Literatum since 2017 (ACM, 2017-10-03)."""
+
+    publisher = "acm"
+    site_hosts = frozenset({"dl.acm.org"})

package/carsi/paper_search_carsi/adapters/acs/__init__.py

@@ -0,0 +1,47 @@
+"""ACS Publications: journals on pubs.acs.org.
+
+Article pages are ``/<journal>/article(-abstract)/<four numbers>(/<name>)``;
+an ``/article-abstract/`` page is the article without access and "Subscribed"
+by the title means access. Signed in, the header's institution dropdown
+(``button.at-institution``) names the school, without the older "access
+provided by" text: all three are the shared access rule's
+(engine/access_rules.py). The article PDF endpoint (``/article-pdf/``,
+``metadata_pdf_patterns``) is the page's ``citation_pdf_url`` when its
+``citation_doi`` is this DOI (the engine's one rule, batch_pdf_urls.py); once
+known, every other PDF response is refused.
+"""
+from __future__ import annotations
+
+import re
+from typing import Any
+from urllib.parse import urlparse
+
+from ..base import PublisherAdapter, SupplementLink, register
+from ..platforms import silverchair
+
+
+@register
+class ACSAdapter(PublisherAdapter):
+    publisher = "acs"
+    keeps_pdf_out_of_downloads = True
+    metadata_pdf_patterns = ("https://pubs.acs.org/*/article-pdf/*",)
+    # ACS's IP-block page names its contact mailbox (moved from the engine, review 2026-11-02).
+    block_page_markers = ("ipblock@acs.org",)
+
+    def login_url(self) -> str | None:
+        return "https://pubs.acs.org/"
+
+    def supplement_links(self, page: Any, doi: str) -> list[SupplementLink] | None:
+        # The Supporting Information: the Silverchair platform's rule (ACS's site is
+        # Silverchair's since 2026; its PDF rules stay its own above).
+        return silverchair.supplement_links(page, doi, frozenset({"pubs.acs.org"}))
+
+    def is_success_article_url(self, url: str) -> bool | None:
+        parsed = urlparse(url.lower())
+        if parsed.hostname != "pubs.acs.org":
+            return False
+        # Volume/issue/page article ids, or the DOI form of the 2026 pages
+        # ("/jacsat/article-abstract/doi/10.1021/jacs.6c12304/5461066/Investigation-…", 2026-11-02).
+        if re.fullmatch(r"/[^/]+/article(?:-abstract)?/(?:(?:\d+/){3}\d+|doi/10\.\d{4,9}/[^/]+/\d+)(?:/[^/]+)?/?", parsed.path):
+            return True
+        return None

package/carsi/paper_search_carsi/adapters/aha_journals/__init__.py

@@ -0,0 +1,13 @@
+"""On the Atypon platform (``platforms/atypon``): only its hosts are its own."""
+from __future__ import annotations
+
+from ..base import register
+from ..platforms.atypon import AtyponAdapter
+
+
+@register
+class AHAJournalsAdapter(AtyponAdapter):
+    """AHA journals (Circulation, Stroke, ...) on ahajournals.org."""
+
+    publisher = "aha journals"
+    site_hosts = frozenset({"ahajournals.org", "www.ahajournals.org"})

package/carsi/paper_search_carsi/adapters/ams/__init__.py

@@ -0,0 +1,19 @@
+"""American Meteorological Society: journals on journals.ametsoc.org.
+
+The PDF of an article page ``/view/<path>.xml`` is ``/downloadpdf/view/<path>.pdf``.
+"""
+from __future__ import annotations
+
+from ...engine.router import _append_ams_candidates
+from ..base import PublisherAdapter, register
+
+
+@register
+class AMSAdapter(PublisherAdapter):
+    publisher = "ams"
+
+    @classmethod
+    def profile_pdf_candidates(cls, doi: str, source_url: str) -> list[str]:
+        urls: list[str] = []
+        _append_ams_candidates(urls, source_url)
+        return urls

package/carsi/paper_search_carsi/adapters/annual_reviews/__init__.py

@@ -0,0 +1,154 @@
+"""Annual Reviews: journals on annualreviews.org.
+
+Outside CARSI its institution login is on its own pages: ``/session/ext/shib``
+(Shibboleth) and ``/session/ext/athens`` ("Option 2: Sign-in with OpenAthens").
+The OpenAthens page is taken: from the Shibboleth page its address is opened
+with the configured school as the search (``openathens_entry``), and there the
+school is typed into the organisation search and chosen
+(``fill_institution_search``). Moved from the engine (review 2026-11-02); the
+steps are as before, and a browser error on them is recorded instead of passed
+over in silence.
+"""
+from __future__ import annotations
+
+import json
+from typing import Any
+from urllib.parse import parse_qs, urlencode, urlparse
+
+from ...engine.batch_common import _is_browser_error, open_page, pump_until
+from ..base import PublisherAdapter, register
+
+_OPENATHENS_BOX = ("xpath=(//*[contains(normalize-space(.), 'Option 2: Sign-in with OpenAthens')]"
+                   "/following::input[contains(@placeholder, 'organization')])[1]")
+# The page's own controls, by their words: the school's result, "Go To Sign-in",
+# else the organisation search and "Find Your Organization".
+_OPENATHENS_STEP_JS = """
+                (query) => {
+                  const needle = (query || '').toLowerCase();
+                  if (!needle) return null;
+                  const visible = (el) => {
+                    const rect = el.getBoundingClientRect();
+                    const style = window.getComputedStyle(el);
+                    return rect.width > 0 && rect.height > 0 && style.visibility !== 'hidden' && style.display !== 'none';
+                  };
+                  const textOf = (el) => [
+                    el.innerText || '',
+                    el.textContent || '',
+                    el.value || '',
+                    el.getAttribute('aria-label') || '',
+                    el.getAttribute('title') || ''
+                  ].join(' ').trim();
+                  const controls = [...document.querySelectorAll('a,button,input[type="button"],input[type="submit"]')].filter(visible);
+                  const goButtons = controls.filter((el) => textOf(el).toLowerCase().includes('go to sign-in'));
+                  const resultEls = [...document.querySelectorAll('a,button,li,div,span,option')]
+                    .filter(visible)
+                    .filter((el) => textOf(el).toLowerCase().includes(needle));
+                  if (resultEls.length) {
+                    resultEls[resultEls.length - 1].click();
+                    if (goButtons.length) {
+                      goButtons[goButtons.length - 1].click();
+                      return {action: 'openathens_go', result: textOf(resultEls[resultEls.length - 1]).slice(0, 160)};
+                    }
+                    return {action: 'openathens_result', result: textOf(resultEls[resultEls.length - 1]).slice(0, 160)};
+                  }
+                  const inputs = [...document.querySelectorAll('input')]
+                    .filter(visible)
+                    .filter((el) => {
+                      const haystack = [el.placeholder || '', el.name || '', el.id || '', el.getAttribute('aria-label') || ''].join(' ').toLowerCase();
+                      return haystack.includes('organization') || haystack.includes('institution');
+                    });
+                  if (inputs.length) {
+                    const input = inputs[inputs.length - 1];
+                    input.focus();
+                    input.value = query;
+                    input.dispatchEvent(new Event('input', {bubbles: true}));
+                    input.dispatchEvent(new Event('change', {bubbles: true}));
+                    const findButtons = controls.filter((el) => textOf(el).toLowerCase().includes('find your organization'));
+                    if (findButtons.length) {
+                      findButtons[findButtons.length - 1].click();
+                    }
+                    return {action: 'openathens_search', query};
+                  }
+                  if (goButtons.length) {
+                    goButtons[goButtons.length - 1].click();
+                    return {action: 'openathens_go'};
+                  }
+                  return null;
+                }
+                """
+
+
+def _error(exc: Exception) -> str:
+    return f"{type(exc).__name__}: {str(exc).splitlines()[0][:160] if str(exc) else ''}"
+
+
+@register
+class AnnualReviewsAdapter(PublisherAdapter):
+    publisher = "annual reviews"
+
+    def openathens_entry(self, page: Any, result: Any) -> bool | None:
+        """From the Shibboleth page, its OpenAthens page with the school as the
+        search; on the OpenAthens page nothing more (the school is chosen there)."""
+        d = self.d
+        parsed = urlparse(str(getattr(page, "url", "") or ""))
+        if not parsed.netloc.endswith("annualreviews.org"):
+            return None
+        path = parsed.path.lower()
+        if path.startswith("/session/ext/athens"):
+            return False
+        if not path.startswith("/session/ext/shib"):
+            return None
+        target_url = parse_qs(parsed.query).get("url", ["/"])[0] or "/"
+        openathens_url = (f"{parsed.scheme or 'https'}://{parsed.netloc}/session/ext/athens?"
+                          f"{urlencode({'url': target_url, 'athensWayfSearch': d.institution_query})}")
+        try:
+            open_page(page, openathens_url, timeout_ms=30_000)
+        except Exception as exc:
+            if not _is_browser_error(exc):
+                raise
+            d._event(result, "openathens_entry_error", _error(exc))
+            return False
+        d._event(result, "openathens_entry", openathens_url)
+        return True
+
+    def fill_institution_search(self, page: Any, result: Any) -> bool:
+        """On the OpenAthens page: the school in "Option 2"'s organisation search,
+        its result, then "Go To Sign-in"; else the page's own controls by their words."""
+        d = self.d
+        if "annualreviews.org/session/ext/athens" not in str(getattr(page, "url", "") or "").lower():
+            return False
+        try:
+            input_box = page.locator(_OPENATHENS_BOX).first
+            if input_box.is_visible(timeout=1_500):
+                input_box.fill(d.institution_query)
+                page.locator("text=Find Your Organization").last.click(timeout=5_000, no_wait_after=True)
+                d._event(result, "institution_search", f"OpenAthens: {d.institution_query}")
+                pump_until(page, lambda: d._any_visible(page, d._institution_result_selectors()), 2_000)
+                for selector in d._institution_result_selectors():
+                    try:
+                        option = page.locator(selector).last
+                        if option.is_visible(timeout=1_500):
+                            option.click(timeout=5_000, no_wait_after=True)
+                            page.locator("text=Go To Sign-in").last.click(timeout=5_000, no_wait_after=True)
+                            d._event(result, "institution_selected", selector)
+                            break
+                    except Exception as exc:
+                        if not _is_browser_error(exc):
+                            raise
+                        continue  # the next result selector
+                return True
+        except Exception as exc:
+            if not _is_browser_error(exc):
+                raise
+            d._event(result, "institution_search_error", f"OpenAthens organisation search: {_error(exc)}")
+        try:
+            action = page.evaluate(_OPENATHENS_STEP_JS, d.institution_query)
+        except Exception as exc:
+            if not _is_browser_error(exc):
+                raise
+            d._event(result, "institution_select_error", _error(exc))
+            return False
+        if action:
+            d._event(result, "institution_selected", json.dumps(action, ensure_ascii=False))
+            return True
+        return False

package/carsi/paper_search_carsi/adapters/aps/__init__.py

@@ -0,0 +1,127 @@
+"""American Physical Society: journals on journals.aps.org (and link.aps.org DOIs).
+
+Article pages are ``/<journal>/abstract/<DOI>`` and ``/pdf/<DOI>``, and
+link.aps.org's ``/doi/`` addresses. Without access the article shows an
+access box (``#inline-unauthorized-page``, a paywall box of the shared access
+rule) offering APS member, APS account and institution-provided logins. The institution's own login is "Sign in via
+your Institution" in the header's Sign-in menu (OpenAthens). A login that
+ends on another APS page goes back to this record's article.
+"""
+from __future__ import annotations
+
+import json
+from typing import Any
+from urllib.parse import urlparse
+
+from ...engine.router import _append_aps_candidates
+from ...engine.batch_common import open_page
+from ..base import PublisherAdapter, register
+
+
+# APS resolves article pages per journal code; the DOI suffix carries the journal.
+def aps_article_url(doi: str) -> str:
+    suffix = doi.split("/", 1)[-1] if "/" in doi else doi
+    journal_codes = (
+        ("PhysRevLett.", "prl"),
+        ("PhysRevA.", "pra"),
+        ("PhysRevB.", "prb"),
+        ("PhysRevC.", "prc"),
+        ("PhysRevD.", "prd"),
+        ("PhysRevE.", "pre"),
+        ("PhysRevX.", "prx"),
+        ("PhysRevApplied.", "prapplied"),
+        ("PhysRevResearch.", "prresearch"),
+        ("RevModPhys.", "rmp"),
+        ("PRXQuantum.", "prxquantum"),
+    )
+    for prefix, journal_code in journal_codes:
+        if suffix.startswith(prefix):
+            return f"https://journals.aps.org/{journal_code}/abstract/{doi}"
+    return ""
+
+
+@register
+class APSAdapter(PublisherAdapter):
+    publisher = "aps"
+    returns_to_record_article = True
+
+    def article_url(self, doi: str) -> str | None:
+        return aps_article_url(doi) or None
+
+    @classmethod
+    def profile_pdf_candidates(cls, doi: str, source_url: str) -> list[str]:
+        urls: list[str] = []
+        _append_aps_candidates(urls, source_url)
+        return urls
+
+    @classmethod
+    def belongs_to_article(cls, url: str, doi: str, source_url: str) -> bool | None:
+        lower = url.lower()
+        host = (urlparse(url).hostname or "").lower()
+        if host.endswith("journals.aps.org") or host.endswith("link.aps.org"):
+            if "/accepted" in lower:
+                return False
+            doi_lower = doi.lower()
+            doi_escaped = doi_lower.replace("/", "%2f")
+            if "/pdf/" in lower or "/doi/" in lower:
+                return doi_lower in lower or doi_escaped in lower
+        return True
+
+    def is_success_article_url(self, url: str) -> bool | None:
+        parsed = urlparse(url.lower())
+        host = parsed.netloc
+        path = parsed.path
+        if host.endswith("link.aps.org") and path.startswith("/doi/"):
+            return True
+        return host.endswith("journals.aps.org") and ("/abstract/" in path or "/pdf/" in path)
+
+    def open_institution_entry(self, page: Any, result: Any) -> bool | None:
+        host = (urlparse(str(getattr(page, "url", "") or "")).hostname or "").lower()
+        if host == "openathens.net" or host.endswith(".openathens.net"):
+            return None  # a federation page: the engine clicks nothing there
+        if self.institution_sign_in(page, result):
+            return True
+        if result is not None:
+            self.d._event(result, "aps_institution_entry_missing", getattr(page, "url", ""))
+        return False
+
+    def institution_sign_in(self, page: Any, result: Any = None) -> bool:
+        """APS "Sign in via your Institution" (OpenAthens federation).
+
+        It sits in the header's Sign-in menu. The access box's
+        "username/password provided by your institution" link is an APS
+        account form, not the institution's own login, and is never used.
+        """
+        d = self.d
+        current_url = str(getattr(page, "url", "") or "").lower()
+        if "/auth/openathens/login" in current_url or "/login_inst_user" in current_url:
+            return False
+        try:
+            detail = page.evaluate(
+                """
+                () => {
+                  const norm = (value) => (value || '').toString().replace(/\\s+/g, ' ').trim();
+                  const link = [...document.querySelectorAll('a[href]')].find((el) => {
+                    let url;
+                    try { url = new URL(el.getAttribute('href'), location.href); } catch (e) { return false; }
+                    return url.protocol === 'https:' && url.hostname === 'journals.aps.org'
+                      && url.pathname === '/auth/openathens/login' && url.searchParams.get('rt');
+                  });
+                  if (!link) return null;
+                  return {
+                    text: norm(link.innerText || link.textContent || '').slice(0, 200),
+                    href: link.href || link.getAttribute('href') || ''
+                  };
+                }
+                """
+            )
+            if not isinstance(detail, dict) or not detail.get("href"):
+                return False
+            open_page(page, str(detail["href"]), timeout_ms=30_000)
+            if result is not None:
+                d._event(result, "sso_entry_clicked", json.dumps(detail, ensure_ascii=False))
+            return True
+        except Exception as exc:
+            if result is not None:
+                d._event(result, "sso_entry_error", f"{type(exc).__name__}: {exc}")
+            return False

package/carsi/paper_search_carsi/adapters/asce/__init__.py

@@ -0,0 +1,13 @@
+"""On the Atypon platform (``platforms/atypon``): only its hosts are its own."""
+from __future__ import annotations
+
+from ..base import register
+from ..platforms.atypon import AtyponAdapter
+
+
+@register
+class ASCEAdapter(AtyponAdapter):
+    """ASCE Library (ascelibrary.org), an Atypon site: the shared rules, nothing of its own (2026-11-02)."""
+
+    publisher = "asce"
+    site_hosts = frozenset({"ascelibrary.org", "www.ascelibrary.org"})