reference-audit 0.1.0__py3-none-any.whl

reference_audit/sources/openlibrary.py

@@ -0,0 +1,158 @@
+"""Open Library adapter — books / ISBN backfill (gavrilets2004, mabook).
+
+Matches on title+author (the pilot books have a publisher typo, so publisher is never a feature).
+`fetch_editions` additionally enumerates a book's concrete editions (year/publisher/ISBN each) for
+the edition-aware book check.
+"""
+
+from __future__ import annotations
+
+import re
+
+from reference_audit.models import (
+    BibEntry,
+    EntryType,
+    Identifiers,
+    SourceQueryResult,
+    SourceRecord,
+)
+from reference_audit.parsing.entities import decode_html_entities
+from reference_audit.sources.base import SourceAdapter
+from reference_audit.sources.http import TransientHTTPError, get_json, new_client
+from reference_audit.sources.normalize import (
+    openlibrary_doc_to_record,
+    openlibrary_edition_to_record,
+)
+
+_SEARCH = "https://openlibrary.org/search.json"
+_FIELDS = "key,title,author_name,first_publish_year,isbn,edition_count,publisher"
+
+_OL = "https://openlibrary.org"
+_EDITIONS_LIMIT = 50  # editions fetched per work
+_MAX_WORKS = 6        # Open Library often splits one book across several "work" records
+
+
+def _fold(s: str | None) -> str:
+    # `doc` titles are raw API text, not a decoded `SourceRecord`: "&" is no word "amp".
+    return re.sub(r"[^a-z0-9]+", " ", decode_html_entities(s or "").lower()).strip()
+
+
+def _title_compatible(entry_title: str, work_title: str | None) -> bool:
+    """A work is the cited book if the folded titles are equal or one contains the other (Open
+    Library appends series subtitles like '(Frontiers in Physics, No. 46)' to some records)."""
+    a, b = _fold(entry_title), _fold(work_title)
+    return bool(a and b and (a == b or a in b or b in a))
+
+
+def _author_query(authors: list[str]) -> str | None:
+    """A surname to constrain the title search ('Ma, Shang-Keng'/'Shang-Keng Ma' → 'Ma')."""
+    if not authors:
+        return None
+    first = authors[0].strip()
+    return (first.split(",")[0] if "," in first else first.split()[-1]).strip() or None
+
+
+class OpenLibraryAdapter(SourceAdapter):
+    name = "openlibrary"
+    handles = {EntryType.BOOK, EntryType.INCOLLECTION}
+    rate_per_sec = 5.0
+
+    def __init__(self, *, email: str | None = None, client=None, limiter=None):
+        # Open Library asks API clients to identify themselves with a contact email *in the
+        # User-Agent header* (not a query param like Crossref's `mailto`); see
+        # https://openlibrary.org/developers/api#rate-limits. When configured and no client is
+        # injected (tests pass their own), build the client with an email-bearing User-Agent.
+        if client is None and email:
+            client = new_client(user_agent=f"reference-audit/0.1 (mailto:{email})")
+        super().__init__(client=client, limiter=limiter)
+        self.email = email
+
+    async def lookup_by_id(self, ids: Identifiers) -> SourceQueryResult:
+        if not ids.isbn13:
+            return SourceQueryResult(source=self.name, query_kind="id", records=[])
+        params = {"isbn": ids.isbn13, "fields": _FIELDS, "limit": 1}
+        try:
+            _status, data = await get_json(self.client, self.rate_limiter, _SEARCH, params=params)
+        except TransientHTTPError as exc:
+            return SourceQueryResult(source=self.name, query_kind="id", error=str(exc))
+        docs = (data or {}).get("docs") or []
+        records = [openlibrary_doc_to_record(d) for d in docs[:1]]
+        return SourceQueryResult(source=self.name, query_kind="id", records=records)
+
+    async def search_by_metadata(self, entry: BibEntry, limit: int = 10) -> SourceQueryResult:
+        if not entry.title:
+            return SourceQueryResult(source=self.name, query_kind="metadata", records=[])
+        params: dict[str, str | int] = {"title": entry.title, "fields": _FIELDS, "limit": limit}
+        if entry.authors:
+            params["author"] = entry.authors[0]
+        try:
+            _status, data = await get_json(self.client, self.rate_limiter, _SEARCH, params=params)
+        except TransientHTTPError as exc:
+            return SourceQueryResult(source=self.name, query_kind="metadata", error=str(exc))
+        docs = (data or {}).get("docs") or []
+        records: list[SourceRecord] = [openlibrary_doc_to_record(d) for d in docs]
+        return SourceQueryResult(source=self.name, query_kind="metadata", records=records)
+
+    async def fetch_editions(self, entry: BibEntry) -> SourceQueryResult:
+        """Every concrete Open Library edition of the cited book (one SourceRecord per edition).
+
+        Works keys come from both the ISBN search (anchors the cited edition) and a title+author
+        search (reaches editions Open Library scattered across sibling work records — needed to find
+        the latest one). A transport/HTTP failure is surfaced as `error` (never an empty 'no
+        editions'), so the caller reports the gap rather than silently passing the book unverified.
+        """
+        kind = "editions"
+        try:
+            keys = await self._work_keys(entry)
+        except TransientHTTPError as exc:
+            return SourceQueryResult(source=self.name, query_kind=kind, error=str(exc))
+
+        records: list[SourceRecord] = []
+        seen: set[str] = set()
+        errors: list[str] = []
+        for key in keys[:_MAX_WORKS]:
+            try:
+                editions = await self._editions_of_work(key)
+            except TransientHTTPError as exc:
+                errors.append(f"{key}: {exc}")
+                continue
+            for rec in editions:
+                dedup = rec.ids.isbn13 or rec.source_native_id or f"{rec.year}|{_fold(rec.publisher)}"
+                if dedup in seen:
+                    continue
+                seen.add(dedup)
+                records.append(rec)
+        if errors and not records:
+            return SourceQueryResult(source=self.name, query_kind=kind, error="; ".join(errors))
+        return SourceQueryResult(source=self.name, query_kind=kind, records=records)
+
+    async def _work_keys(self, entry: BibEntry) -> list[str]:
+        keys: list[str] = []
+        # Try every ISBN the entry carries: one book registers several (print/electronic, per
+        # edition) and Open Library indexes only some, so a single ISBN can miss a book it does hold.
+        for isbn in sorted(entry.ids.all_isbn13()):
+            params: dict[str, str | int] = {"isbn": isbn, "fields": "key,title", "limit": 5}
+            _status, data = await get_json(self.client, self.rate_limiter, _SEARCH, params=params)
+            for doc in (data or {}).get("docs") or []:
+                key = (doc.get("key") or "").strip()
+                if key:
+                    keys.append(key)
+        if entry.title:
+            params = {"title": entry.title, "fields": "key,title", "limit": 10}
+            author = _author_query(entry.authors)
+            if author:
+                params["author"] = author
+            _status, data = await get_json(self.client, self.rate_limiter, _SEARCH, params=params)
+            for doc in (data or {}).get("docs") or []:
+                key = (doc.get("key") or "").strip()
+                if key and _title_compatible(entry.title, doc.get("title")):
+                    keys.append(key)
+        return list(dict.fromkeys(keys))
+
+    async def _editions_of_work(self, work_key: str) -> list[SourceRecord]:
+        url = f"{_OL}{work_key}/editions.json"
+        _status, data = await get_json(
+            self.client, self.rate_limiter, url, params={"limit": _EDITIONS_LIMIT}
+        )
+        entries = (data or {}).get("entries") or []
+        return [openlibrary_edition_to_record(e) for e in entries]

reference_audit/sources/publisher.py

@@ -0,0 +1,186 @@
+"""Publisher-of-record resolver — the DOI's landing-page citation export.
+
+The authority of record. Aggregators (Crossref/OpenAlex/Semantic Scholar) sometimes lack fields the
+publisher exposes only through its own "Cite" export — notably pages/article numbers for
+article-numbered venues (the MIT Press/Silverchair proceedings whose Crossref `page` is null but
+whose BibTeX export carries `pages={131}`).
+
+Resolution is deliberately conservative and reliability-first:
+  1. dereference the DOI to its landing URL (a redirect — works even when the page body is
+     bot-protected, because we read only the final URL);
+  2. derive the platform's citation-export URL (Silverchair: `/Citation/Download?...` keyed by the
+     numeric resource id in the landing path);
+  3. fetch + parse the BibTeX.
+The export fetch (step 3) goes through a browser-TLS-impersonating client (`get_text_impersonate`),
+because these platforms sit behind Cloudflare, which fingerprints httpx's TLS ClientHello and 403s it
+even with browser headers; replaying a real browser's ClientHello passes. If any step fails — unknown
+platform, an escalated JS/interactive bot-challenge instead of BibTeX, a transport error — the result
+is a reported gap (empty records, or an `error` for a transient block), never a guess and never
+cached as "absent".
+"""
+
+from __future__ import annotations
+
+from urllib.parse import quote, urlparse
+
+import httpx
+
+from reference_audit.models import EntryType, Identifiers, SourceQueryResult
+from reference_audit.sources.base import SourceAdapter
+from reference_audit.sources.http import (
+    TransientHTTPError,
+    get_text_impersonate,
+    new_impersonate_session,
+)
+from reference_audit.sources.normalize import publisher_bibtex_to_record
+
+_DOI_RESOLVER = "https://doi.org/"
+_HANDLE_API = "https://doi.org/api/handles/"
+
+# Silverchair-hosted publishers expose a citation download at
+#   {scheme}://{host}/Citation/Download?resourceId={N}&resourceType=3&citationFormat=2  (BibTeX)
+# where {N} is the trailing numeric segment of the article landing path
+#   .../isal2023/35/131/116921  ->  resourceId=116921
+_SILVERCHAIR_HOSTS = ("direct.mit.edu",)
+
+# Atypon-hosted publishers (Sage, …) expose a citation download keyed directly by the DOI:
+#   {scheme}://{host}/action/downloadCitation?doi={DOI}&format=bibtex&include=cit  (BibTeX)
+# Unlike Silverchair, no landing-path resource id is needed — the DOI alone derives the URL.
+# Like Silverchair, these endpoints are Cloudflare-walled, so the export fetch goes through the
+# browser-impersonating client; an *escalated* interactive challenge still surfaces a reported error.
+_ATYPON_HOSTS = ("journals.sagepub.com",)
+
+
+def _silverchair_export_url(landing: str) -> str | None:
+    parts = urlparse(landing)
+    if not any(parts.netloc == h or parts.netloc.endswith("." + h) for h in _SILVERCHAIR_HOSTS):
+        return None
+    segments = [s for s in parts.path.split("/") if s]
+    if not segments or not segments[-1].isdigit():
+        return None
+    return (
+        f"{parts.scheme}://{parts.netloc}/Citation/Download"
+        f"?resourceId={segments[-1]}&resourceType=3&citationFormat=2"
+    )
+
+
+def _atypon_export_url(landing: str, doi: str) -> str | None:
+    parts = urlparse(landing)
+    if not any(parts.netloc == h or parts.netloc.endswith("." + h) for h in _ATYPON_HOSTS):
+        return None
+    return (
+        f"{parts.scheme}://{parts.netloc}/action/downloadCitation"
+        f"?doi={quote(doi, safe='')}&format=bibtex&include=cit"
+    )
+
+
+class PublisherAdapter(SourceAdapter):
+    """Fetch the DOI publisher's own citation export (Silverchair and Atypon platforms)."""
+
+    name = "publisher"
+    handles = {
+        EntryType.ARTICLE,
+        EntryType.INPROCEEDINGS,
+        EntryType.BOOK,
+        EntryType.INCOLLECTION,
+        EntryType.MISC,
+    }
+    rate_per_sec = 2.0  # be gentle on publisher sites
+
+    def __init__(self, *, export_fetch=None, **kwargs):
+        """`export_fetch` (async `(url) -> (status, text)`) overrides the citation-export fetcher;
+        injected by tests. In production it defaults to a lazily-created browser-impersonating
+        session (curl_cffi), kept separate from `self.client` (httpx) which still serves the DOI
+        resolution. DOI resolution stays on httpx because it reads only the redirect target, never a
+        Cloudflare-walled body.
+        """
+        super().__init__(**kwargs)
+        self._export_fetch = export_fetch
+        self._imp_session = None
+
+    async def _fetch_export(self, url: str) -> tuple[int, str]:
+        if self._export_fetch is not None:
+            return await self._export_fetch(url)
+        if self._imp_session is None:
+            self._imp_session = new_impersonate_session()
+        return await get_text_impersonate(self._imp_session, self.rate_limiter, url)
+
+    async def aclose(self) -> None:
+        if self._imp_session is not None:
+            await self._imp_session.close()
+        await super().aclose()
+
+    async def _landing_url(self, doi: str) -> str:
+        """Final URL the DOI resolves to (redirect only; the page body may be bot-protected)."""
+        await self.rate_limiter.acquire()
+        try:
+            resp = await self.client.get(f"{_DOI_RESOLVER}{doi}")
+        except httpx.TransportError as exc:
+            raise TransientHTTPError(f"transport: {exc}") from exc
+        return str(resp.url)
+
+    async def doi_registered(self, doi: str) -> bool | None:
+        """Whether the handle system (doi.org) actually knows this DOI.
+
+        A *backfilled* DOI is only useful if it resolves: authors sometimes record a bogus DOI in
+        their arXiv/preprint metadata (e.g. an ACM `10.5555/...` placeholder for a venue that mints
+        no real DOI) which the aggregators then echo; a cited DOI may be invented outright.
+
+        The Handle REST API answers this authoritatively, and is asked instead of the resolver: the
+        resolver answers HTTP 500, not 404, for a DOI under a prefix nobody registered
+        (`10.8888/...`), which read as an outage. The API's `responseCode` decides: 1 (found) or 200
+        (handle found, no URL value) ⇒ registered; 100 (handle not found) ⇒ not registered; 301
+        ("That prefix doesn't live here": a handle prefix the DOI system does not serve, e.g.
+        `10.77771/...`) ⇒ not registered, since every real DOI prefix is homed there. A transport
+        error, a 429/5xx or a body without a code returns None — an outage is never read as
+        'invalid'.
+        """
+        await self.rate_limiter.acquire()
+        try:
+            resp = await self.client.get(f"{_HANDLE_API}{doi}", params={"type": "URL"})
+            data = resp.json()
+        except (httpx.HTTPError, ValueError):
+            return None
+        code = data.get("responseCode") if isinstance(data, dict) else None
+        if code in (1, 200):
+            return True
+        if code in (100, 301):
+            return False
+        return None  # undetermined; never asserted invalid
+
+    async def lookup_by_id(self, ids: Identifiers) -> SourceQueryResult:
+        if not ids.doi:
+            return SourceQueryResult(source=self.name, query_kind="id", records=[])
+        try:
+            landing = await self._landing_url(ids.doi)
+        except TransientHTTPError as exc:
+            return SourceQueryResult(source=self.name, query_kind="id", error=str(exc))
+
+        export_url = _silverchair_export_url(landing) or _atypon_export_url(landing, ids.doi)
+        if export_url is None:
+            # Unknown publisher platform — nothing we know how to fetch. A clean "not found" here
+            # (not an error): field checks then report the gap against the sources we did consult.
+            return SourceQueryResult(source=self.name, query_kind="id", records=[])
+
+        try:
+            status, text = await self._fetch_export(export_url)
+        except TransientHTTPError as exc:
+            # Most commonly a Cloudflare/JS bot-challenge (HTTP 403). Surfaced as an error — a
+            # human-retrievable datum we could not auto-fetch — so it is reported, not treated as
+            # absent, and retried next run rather than cached.
+            return SourceQueryResult(
+                source=self.name,
+                query_kind="id",
+                error=f"publisher citation export not retrievable ({exc}): {export_url}",
+            )
+        if status == 404:
+            return SourceQueryResult(source=self.name, query_kind="id", records=[])
+
+        record = publisher_bibtex_to_record(text)
+        if record is None:
+            return SourceQueryResult(
+                source=self.name,
+                query_kind="id",
+                error=f"publisher response was not a citation export (bot-challenge?): {export_url}",
+            )
+        return SourceQueryResult(source=self.name, query_kind="id", records=[record])

reference_audit/sources/registry.py

@@ -0,0 +1,117 @@
+"""Adapter construction + per-entry routing (entry_type × available identifiers).
+
+Builds the default adapter set from config, and decides which adapters to query by id vs by
+metadata for a given entry. Metadata search ALWAYS runs (even when an id is present) so it can
+backfill missing DOIs/ISBNs and supply corroborating candidates for the equivalence step.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from reference_audit.config import AuditConfig
+from reference_audit.models import BibEntry, EntryType
+from reference_audit.sources.arxiv import ArxivAdapter
+from reference_audit.sources.base import SourceAdapter
+from reference_audit.sources.clickhouse import build_clickhouse_adapters
+from reference_audit.sources.crossref import CrossrefAdapter
+from reference_audit.sources.dblp import DblpAdapter
+from reference_audit.sources.google_books import GoogleBooksAdapter
+from reference_audit.sources.openalex import OpenAlexAdapter
+from reference_audit.sources.openlibrary import OpenLibraryAdapter
+from reference_audit.sources.publisher import PublisherAdapter
+from reference_audit.sources.render import ChromiumRenderer, find_browser
+from reference_audit.sources.semantic_scholar import SemanticScholarAdapter
+from reference_audit.sources.web import WebAdapter
+
+# Reputable venues that legitimately mint no DOI — a no-DOI match here is not a defect (README 1.1).
+VENUE_ALLOWLIST_NO_DOI = ("tmlr", "transactions on machine learning research")
+
+
+def build_web_renderer(config: AuditConfig) -> ChromiumRenderer | None:
+    """A headless renderer for SPA shells, or None when rendering is disabled. The renderer self-
+    reports 'unavailable' at call time when no browser is found, so the funnel leaves such pages
+    unresolved rather than crying hallucination (see `sources.render`)."""
+    if not config.web_render_enabled:
+        return None
+    return ChromiumRenderer(
+        find_browser(config.web_render_browser_path),
+        timeout=config.web_render_timeout,
+        virtual_time_ms=config.web_render_virtual_time_ms,
+    )
+
+
+def build_default_adapters(config: AuditConfig) -> list[SourceAdapter]:
+    mailto = config.resolved_mailto()
+    # Semantic Scholar, OpenAlex and DBLP come from their public APIs, or — with
+    # source_backend="clickhouse" — from a local ClickHouse mirror of the same databases.
+    if config.source_backend == "clickhouse":
+        mirrored: list[SourceAdapter] = build_clickhouse_adapters(config)
+    else:
+        mirrored = [
+            OpenAlexAdapter(mailto=mailto),
+            SemanticScholarAdapter(api_key=config.s2_api_key),
+            DblpAdapter(),
+        ]
+    return [
+        CrossrefAdapter(mailto=mailto),
+        *mirrored,
+        ArxivAdapter(),
+        OpenLibraryAdapter(email=config.openlibrary_email),
+        GoogleBooksAdapter(api_key=config.google_books_api_key),
+        PublisherAdapter(),
+        WebAdapter(render=build_web_renderer(config)),
+    ]
+
+
+@dataclass
+class Route:
+    id_adapters: list[SourceAdapter]
+    metadata_adapters: list[SourceAdapter]
+
+
+def route_entry(entry: BibEntry, adapters: list[SourceAdapter]) -> Route:
+    """Pick adapters to query by-id and by-metadata for this entry."""
+    by_name = {a.name: a for a in adapters}
+
+    def present(*names: str) -> list[SourceAdapter]:
+        return [by_name[n] for n in names if n in by_name]
+
+    id_adapters: list[SourceAdapter] = []
+    if entry.ids.doi:
+        id_adapters += present("crossref", "openalex", "semantic_scholar")
+    if entry.ids.arxiv_id:
+        id_adapters += present("arxiv", "openalex", "semantic_scholar")
+    if entry.ids.isbn13:
+        id_adapters += present("openlibrary", "google_books")
+    # A cited Google Books volume id (books.google.…/books?id=…) resolves to exactly that volume —
+    # the authoritative key for a trade/book title both the article-centric search and Open
+    # Library's strict title match miss.
+    if entry.ids.google_books:
+        id_adapters += present("google_books")
+    # A cited OpenAlex Work id resolves to exactly that Work — the authoritative key for entries
+    # (notably books/trade titles) the article-centric metadata search and Crossref/Open Library
+    # miss. Routed for every entry type, since OpenAlex indexes books too.
+    if entry.ids.openalex:
+        id_adapters += present("openalex")
+    # NOTE: `publisher` (the DOI landing-page citation export) is deliberately NOT an identity
+    # source. It is queried only in the advisory enrichment pass — a bot-walled/blocked publisher
+    # must never set the `errored` flag and mask a hallucinated DOI as 'unresolved' vs 'no match'.
+
+    if entry.entry_type in (EntryType.BOOK, EntryType.INCOLLECTION):
+        metadata_adapters = present("openlibrary", "google_books", "crossref")
+    elif entry.entry_type == EntryType.MISC:
+        metadata_adapters = present("arxiv", "openalex", "crossref", "semantic_scholar", "dblp")
+    else:  # ARTICLE / INPROCEEDINGS / UNKNOWN
+        # DBLP is the authority for the premier CS/ML venues (NeurIPS/ICLR/ICML), which mint no DOI
+        # and are thinly covered by the article-centric aggregators — decisive for @inproceedings.
+        metadata_adapters = present("crossref", "openalex", "semantic_scholar", "dblp")
+
+    # de-dup while preserving order
+    id_adapters = list(dict.fromkeys(id_adapters))
+    return Route(id_adapters=id_adapters, metadata_adapters=metadata_adapters)
+
+
+def venue_allows_no_doi(venue: str) -> bool:
+    v = (venue or "").lower()
+    return any(allowed in v for allowed in VENUE_ALLOWLIST_NO_DOI)

reference_audit/sources/render.py

@@ -0,0 +1,115 @@
+"""Headless-browser rendering for client-side-rendered (single-page-app) web pages.
+
+Some cited URLs are JavaScript single-page apps: a plain HTTP GET returns only an empty app shell
+(a spinner plus a ``<script>`` bundle), and the page's real title and text exist only *after* the
+JavaScript runs. Reading such a shell as 'a different page' would be a false hallucination
+(see `matching.webcheck`). This module renders the page in a headless Chromium and returns the
+post-JavaScript DOM so the page's own metadata/text can be read like any other page.
+
+Reliability-first (per the project's two-level reporting goal):
+  * if no browser is available, rendering is *unavailable* — the caller leaves the entry unresolved
+    and reports that a browser is needed, never guesses 'different page' from the empty shell;
+  * a render failure (timeout, crash, non-zero exit) is an *error* — reported, retried next run,
+    never cached and never read as 'absent'.
+
+The renderer shells out to a system Chromium/Chrome binary (``--headless=new --dump-dom``) rather
+than pulling in a Python browser-automation package and its ~150 MB browser download: the only thing
+needed is a chromium-family binary on PATH (or an explicit path via config), and everything degrades
+gracefully to 'unavailable' when none is present.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import shutil
+
+# Candidate binary names searched on PATH, in preference order, when no explicit path is configured.
+_BROWSER_NAMES = (
+    "chromium",
+    "chromium-browser",
+    "google-chrome",
+    "google-chrome-stable",
+    "chrome",
+)
+
+
+class RenderUnavailable(Exception):
+    """No headless browser is available to render the page (a configuration gap, not a failure)."""
+
+
+class RenderError(Exception):
+    """A headless render was attempted but failed (timeout, crash, non-zero exit, empty output)."""
+
+
+def find_browser(explicit_path: str | None = None) -> str | None:
+    """Resolve a chromium-family binary: the configured path if runnable, else the first on PATH."""
+    if explicit_path:
+        return explicit_path if shutil.which(explicit_path) or _is_executable(explicit_path) else None
+    for name in _BROWSER_NAMES:
+        found = shutil.which(name)
+        if found:
+            return found
+    return None
+
+
+def _is_executable(path: str) -> bool:
+    import os
+
+    return os.path.isfile(path) and os.access(path, os.X_OK)
+
+
+class ChromiumRenderer:
+    """An injectable async renderer: ``await renderer(url) -> (status, final_url, html)``.
+
+    Constructed once per run with a resolved browser path (or ``None`` — then every call raises
+    `RenderUnavailable`, so the funnel reports 'a browser is needed' rather than crashing). Matches
+    the ``fetch=`` stub contract of `WebAdapter`, so tests inject a plain callable instead.
+    """
+
+    def __init__(
+        self,
+        browser_path: str | None,
+        *,
+        timeout: float = 30.0,
+        virtual_time_ms: int = 15000,
+    ):
+        self.browser_path = browser_path
+        self.timeout = timeout
+        self.virtual_time_ms = virtual_time_ms
+
+    async def __call__(self, url: str) -> tuple[int, str, str]:
+        if not self.browser_path:
+            raise RenderUnavailable("no chromium-family browser found (set web_render_browser_path)")
+        cmd = [
+            self.browser_path,
+            "--headless=new",
+            "--disable-gpu",
+            "--no-sandbox",
+            "--hide-scrollbars",
+            "--disable-dev-shm-usage",
+            f"--virtual-time-budget={self.virtual_time_ms}",
+            "--dump-dom",
+            url,
+        ]
+        try:
+            proc = await asyncio.create_subprocess_exec(
+                *cmd,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+            )
+        except (OSError, ValueError) as exc:  # binary vanished / not executable
+            raise RenderError(f"could not launch browser: {exc}") from exc
+        try:
+            stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=self.timeout)
+        except (asyncio.TimeoutError, TimeoutError) as exc:
+            proc.kill()
+            await proc.wait()
+            raise RenderError(f"render timed out after {self.timeout}s") from exc
+        if proc.returncode != 0:
+            tail = (stderr or b"").decode("utf-8", "replace").strip().splitlines()[-1:] or [""]
+            raise RenderError(f"browser exited {proc.returncode}: {tail[0]}")
+        html = (stdout or b"").decode("utf-8", "replace")
+        if not html.strip():
+            raise RenderError("browser produced empty output")
+        # --dump-dom does not report redirects; the cited URL is the best 'final' URL we have.
+        return 200, url, html

reference_audit/sources/semantic_scholar.py

@@ -0,0 +1,68 @@
+"""Semantic Scholar adapter — strong for CS/ML (zhang2018 CVPR, fu2023 NeurIPS, kumar2024)."""
+
+from __future__ import annotations
+
+from reference_audit.models import (
+    BibEntry,
+    EntryType,
+    Identifiers,
+    SourceQueryResult,
+    SourceRecord,
+)
+from reference_audit.sources.base import SourceAdapter
+from reference_audit.sources.http import TransientHTTPError, get_json
+from reference_audit.sources.normalize import s2_paper_to_record
+
+_BASE = "https://api.semanticscholar.org/graph/v1/paper"
+_FIELDS = "title,year,venue,authors,externalIds,citationCount,abstract"
+
+
+class SemanticScholarAdapter(SourceAdapter):
+    name = "semantic_scholar"
+    handles = {EntryType.ARTICLE, EntryType.INPROCEEDINGS, EntryType.MISC}
+    rate_per_sec = 1.0  # public limit is strict; key (if set) is sent as a header
+
+    def __init__(self, api_key: str | None = None, **kw):
+        super().__init__(**kw)
+        self.api_key = api_key
+
+    def _headers(self) -> dict | None:
+        return {"x-api-key": self.api_key} if self.api_key else None
+
+    async def lookup_by_id(self, ids: Identifiers) -> SourceQueryResult:
+        key = None
+        if ids.doi:
+            key = f"DOI:{ids.doi}"
+        elif ids.arxiv_id:
+            key = f"ARXIV:{ids.arxiv_id}"
+        elif ids.pmid:
+            key = f"PMID:{ids.pmid}"
+        if key is None:
+            return SourceQueryResult(source=self.name, query_kind="id", records=[])
+        try:
+            status, data = await get_json(
+                self.client, self.rate_limiter, f"{_BASE}/{key}",
+                params={"fields": _FIELDS}, headers=self._headers(),
+            )
+        except TransientHTTPError as exc:
+            return SourceQueryResult(source=self.name, query_kind="id", error=str(exc))
+        if status == 404 or not data:
+            return SourceQueryResult(source=self.name, query_kind="id", records=[])
+        return SourceQueryResult(
+            source=self.name, query_kind="id", records=[s2_paper_to_record(data)]
+        )
+
+    async def search_by_metadata(self, entry: BibEntry, limit: int = 10) -> SourceQueryResult:
+        if not entry.title:
+            return SourceQueryResult(source=self.name, query_kind="metadata", records=[])
+        params = {"query": entry.title, "limit": limit, "fields": _FIELDS}
+        try:
+            _status, data = await get_json(
+                self.client, self.rate_limiter, f"{_BASE}/search",
+                params=params, headers=self._headers(),
+            )
+        except TransientHTTPError as exc:
+            return SourceQueryResult(source=self.name, query_kind="metadata", error=str(exc))
+        papers = (data or {}).get("data") or []
+        records: list[SourceRecord] = [s2_paper_to_record(p) for p in papers]
+        return SourceQueryResult(source=self.name, query_kind="metadata", records=records)

reference_audit/sources/titlewords.py

@@ -0,0 +1,31 @@
+"""The words of a cited title that a word index can match.
+
+Shared by every title search that requires *all* words to match: DBLP's SPARQL word index and the
+local ClickHouse full-text indexes. Under all-words matching a single word the index spells
+differently hides the real record, so only words that are certain to be indexed the same way are
+kept.
+"""
+
+from __future__ import annotations
+
+import html
+import re
+
+# LaTeX that survives .bib decoding and is no word of the indexed title: inline math (`$\epsilon$`,
+# which the databases store as "ε") and bare commands.
+_LATEX_MATH_RE = re.compile(r"\$[^$]*\$")
+_LATEX_CMD_RE = re.compile(r"\\[A-Za-z]+")
+_WORD_RE = re.compile(r"[^\W_]+")
+
+
+def title_search_words(title: str) -> list[str]:
+    """The searchable words of `title`, lowercased, in order.
+
+    HTML entities are decoded first (`Don't` would otherwise yield a phantom word "apos"), then
+    LaTeX math and commands are dropped. Only plain ASCII alphanumeric words are kept: a word with a
+    non-ASCII letter ("Kübler") or a LaTeX remnant could be indexed differently from how the `.bib`
+    spells it.
+    """
+    text = _LATEX_CMD_RE.sub(" ", _LATEX_MATH_RE.sub(" ", html.unescape(title)))
+    words = (w.lower() for w in _WORD_RE.findall(text))
+    return [w for w in words if w.isascii() and w.isalnum()]