reference-audit 0.1.0__py3-none-any.whl

reference_audit/sources/dblp.py

@@ -0,0 +1,140 @@
+"""DBLP adapter — authoritative coverage of the premier CS/ML venues.
+
+NeurIPS, ICLR, and ICML/PMLR mint **no DOI** and are thinly or ambiguously indexed by the
+article-centric aggregators (Crossref/OpenAlex/S2), so a real conference paper cited only by its
+proceedings URL can otherwise fall through to "unable to verify". DBLP indexes exactly these venues
+with exact titles, full author lists, year, venue, and the proceedings landing page — the record
+needed to confirm such an entry.
+
+DBLP is queried through its **SPARQL endpoint** (`sparql.dblp.org`, a QLever instance), not the
+`/search/publ/api` search API. The search API and its mirrors now answer automated clients with a
+bot-challenge HTML page instead of JSON, which made every DBLP query an error and left every
+unmatched entry unresolved. The SPARQL endpoint serves the same records as JSON.
+
+A search is two requests:
+1. A title search over QLever's word index. A publication matches when its title contains **every**
+   searchable word of the cited title (the search API also required every word). Hits are ordered
+   shortest-title-first, which puts the exact title ahead of the many longer titles that quote it
+   ("Attention Is All You Need" vs "Not All Attention Is All You Need").
+2. The properties and ordered author signatures of the top `limit` hits.
+
+The records then go to central matching, which decides.
+
+This is a metadata-search recall source (there is no DBLP-native id in a typical `.bib`); a DOI it
+carries is still surfaced for backfill. A 429/5xx, a non-JSON body, or a title with no searchable
+word surfaces as `error` (retry next run), never a silent "not found".
+"""
+
+from __future__ import annotations
+
+import re
+
+from reference_audit.models import (
+    BibEntry,
+    EntryType,
+    SourceQueryResult,
+    SourceRecord,
+)
+from reference_audit.sources.base import SourceAdapter
+from reference_audit.sources.http import TransientHTTPError, get_json
+from reference_audit.sources.normalize import dblp_sparql_to_records
+from reference_audit.sources.titlewords import title_search_words
+
+_ENDPOINT = "https://sparql.dblp.org/sparql"
+
+_PREFIXES = """\
+PREFIX dblp: <https://dblp.org/rdf/schema#>
+PREFIX ql: <http://qlever.cs.uni-freiburg.de/builtin-functions/>
+"""
+
+# Step 1: the candidate publications. Kept separate from step 2 because QLever plans the two joined
+# into one query poorly (seconds instead of tens of milliseconds).
+_SEARCH = _PREFIXES + """\
+SELECT DISTINCT ?publ ?title WHERE {{
+  ?publ dblp:title ?title .
+  ?text ql:contains-entity ?title .
+  ?text ql:contains-word "{words}" .
+}}
+ORDER BY STRLEN(?title) LIMIT {limit}
+"""
+
+# Step 2: the properties of those publications, one UNION branch per property so no branch multiplies
+# another's rows. The normalizer folds the rows back into one record per publication.
+_DETAILS = _PREFIXES + """\
+SELECT ?publ ?type ?year ?venue ?pages ?doi ?ee ?ordinal ?name WHERE {{
+  VALUES ?publ {{ {iris} }}
+  {{ ?publ a ?type }}
+  UNION {{ ?publ dblp:yearOfPublication ?year }}
+  UNION {{ ?publ dblp:publishedIn ?venue }}
+  UNION {{ ?publ dblp:pagination ?pages }}
+  UNION {{ ?publ dblp:doi ?doi }}
+  UNION {{ ?publ dblp:primaryDocumentPage ?ee }}
+  UNION {{
+    ?publ dblp:hasSignature ?sig .
+    ?sig a dblp:AuthorSignature ; dblp:signatureOrdinal ?ordinal ; dblp:signatureDblpName ?name .
+  }}
+}}
+"""
+
+# A publication IRI as the endpoint returns it; anything else is not interpolated into a query.
+_PUBL_IRI_RE = re.compile(r"^https://dblp\.org/rec/[^\s<>\"{}|\\^`]+$")
+_ACCEPT = {"Accept": "application/sparql-results+json"}
+
+
+def build_search_query(words: list[str], limit: int) -> str:
+    # `words` are [a-z0-9]+ only (title_search_words), so they are safe inside the string literal.
+    return _SEARCH.format(words=" ".join(words), limit=int(limit))
+
+
+def build_details_query(iris: list[str]) -> str:
+    return _DETAILS.format(iris=" ".join(f"<{iri}>" for iri in iris))
+
+
+class _UnexpectedResponse(Exception):
+    """A SPARQL response without the result shape the query asked for."""
+
+
+def _bindings(data: dict | None) -> list[dict]:
+    bindings = ((data or {}).get("results") or {}).get("bindings")
+    if not isinstance(bindings, list):
+        raise _UnexpectedResponse(f"unexpected SPARQL response: {str(data)[:200]}")
+    return bindings
+
+
+class DblpAdapter(SourceAdapter):
+    name = "dblp"
+    handles = {EntryType.ARTICLE, EntryType.INPROCEEDINGS, EntryType.MISC}
+    rate_per_sec = 2.0
+
+    async def search_by_metadata(self, entry: BibEntry, limit: int = 10) -> SourceQueryResult:
+        if not entry.title:
+            return SourceQueryResult(source=self.name, query_kind="metadata", records=[])
+        words = title_search_words(entry.title)
+        if not words:
+            # Nothing the word index can match. An empty result here would read as "DBLP has no
+            # such paper" when DBLP was never actually asked.
+            return SourceQueryResult(
+                source=self.name, query_kind="metadata",
+                error=f"title has no searchable word for DBLP: {entry.title!r}",
+            )
+        try:
+            _status, found = await get_json(
+                self.client, self.rate_limiter, _ENDPOINT,
+                params={"query": build_search_query(words, limit)}, headers=_ACCEPT,
+            )
+            hits = _bindings(found)
+            iris = list(dict.fromkeys(r.get("publ", {}).get("value", "") for r in hits))
+            if not iris:
+                return SourceQueryResult(source=self.name, query_kind="metadata", records=[])
+            bad = [iri for iri in iris if not _PUBL_IRI_RE.match(iri)]
+            if bad:
+                raise _UnexpectedResponse(f"unexpected publication IRI(s) from DBLP: {bad[:3]}")
+            _status, details = await get_json(
+                self.client, self.rate_limiter, _ENDPOINT,
+                params={"query": build_details_query(iris)}, headers=_ACCEPT,
+            )
+            rows = hits + _bindings(details)
+        except (TransientHTTPError, _UnexpectedResponse) as exc:
+            return SourceQueryResult(source=self.name, query_kind="metadata", error=str(exc))
+        records: list[SourceRecord] = dblp_sparql_to_records(rows)
+        return SourceQueryResult(source=self.name, query_kind="metadata", records=records)

reference_audit/sources/google_books.py

@@ -0,0 +1,101 @@
+"""Google Books adapter — broad book coverage where Open Library's strict title search misses.
+
+Open Library's `title=` search is exact-ish: a cited title carrying its subtitle ("Why Nations
+Fail: The Origins of Power, Prosperity, and Poverty") returns nothing, and a single off-by-one ISBN
+anchors nothing — so a real, famous book can fall through to "not found". Google Books is far more
+forgiving (`q=intitle:… inauthor:…`, `q=isbn:…`) and, crucially, a .bib often carries a Google Books
+*volume id* in its URL (books.google.…/books?id=…) — the authoritative key for that exact volume,
+resolvable with no fuzzy matching at all.
+
+Quota note (the reliability contract): the keyless endpoint shares a global anonymous daily quota
+that is routinely exhausted (HTTP 429); an API key gives a per-project quota. Either way a 429/5xx
+surfaces as `error` (retry next run) via `get_json`, never a silent "not found".
+"""
+
+from __future__ import annotations
+
+from reference_audit.models import (
+    BibEntry,
+    EntryType,
+    Identifiers,
+    SourceQueryResult,
+    SourceRecord,
+)
+from reference_audit.sources.base import SourceAdapter
+from reference_audit.sources.http import TransientHTTPError, get_json
+from reference_audit.sources.normalize import google_books_volume_to_record
+
+_BASE = "https://www.googleapis.com/books/v1/volumes"
+
+
+def _author_surname(authors: list[str]) -> str | None:
+    """A surname to constrain the title search ('Acemoglu, D.'/'Daron Acemoglu' → 'Acemoglu')."""
+    if not authors:
+        return None
+    first = authors[0].strip()
+    return (first.split(",")[0] if "," in first else first.split()[-1]).strip() or None
+
+
+class GoogleBooksAdapter(SourceAdapter):
+    name = "google_books"
+    handles = {EntryType.BOOK, EntryType.INCOLLECTION}
+    rate_per_sec = 5.0
+
+    def __init__(self, *, api_key: str | None = None, client=None, limiter=None):
+        super().__init__(client=client, limiter=limiter)
+        self.api_key = api_key
+
+    def _params(self, **extra: str | int) -> dict:
+        params: dict[str, str | int] = dict(extra)
+        if self.api_key:
+            params["key"] = self.api_key
+        return params
+
+    async def lookup_by_id(self, ids: Identifiers) -> SourceQueryResult:
+        # The cited volume id is the authoritative key for that exact volume — no fuzzy matching.
+        # Otherwise fall back to an ISBN query (one volume).
+        try:
+            if ids.google_books:
+                status, data = await get_json(
+                    self.client,
+                    self.rate_limiter,
+                    f"{_BASE}/{ids.google_books}",
+                    params=self._params(),
+                )
+                if status == 404 or not data:
+                    return SourceQueryResult(source=self.name, query_kind="id", records=[])
+                return SourceQueryResult(
+                    source=self.name,
+                    query_kind="id",
+                    records=[google_books_volume_to_record(data)],
+                )
+            if ids.isbn13:
+                _status, data = await get_json(
+                    self.client,
+                    self.rate_limiter,
+                    _BASE,
+                    params=self._params(q=f"isbn:{ids.isbn13}", maxResults=1),
+                )
+                items = (data or {}).get("items") or []
+                records = [google_books_volume_to_record(v) for v in items[:1]]
+                return SourceQueryResult(source=self.name, query_kind="id", records=records)
+        except TransientHTTPError as exc:
+            return SourceQueryResult(source=self.name, query_kind="id", error=str(exc))
+        return SourceQueryResult(source=self.name, query_kind="id", records=[])
+
+    async def search_by_metadata(self, entry: BibEntry, limit: int = 10) -> SourceQueryResult:
+        if not entry.title:
+            return SourceQueryResult(source=self.name, query_kind="metadata", records=[])
+        q = f'intitle:"{entry.title}"'
+        surname = _author_surname(entry.authors)
+        if surname:
+            q += f" inauthor:{surname}"
+        try:
+            _status, data = await get_json(
+                self.client, self.rate_limiter, _BASE, params=self._params(q=q, maxResults=limit)
+            )
+        except TransientHTTPError as exc:
+            return SourceQueryResult(source=self.name, query_kind="metadata", error=str(exc))
+        items = (data or {}).get("items") or []
+        records: list[SourceRecord] = [google_books_volume_to_record(v) for v in items]
+        return SourceQueryResult(source=self.name, query_kind="metadata", records=records)

reference_audit/sources/http.py

@@ -0,0 +1,260 @@
+"""Shared async HTTP utilities: rate limiting + transient-error retry.
+
+Modeled on `sciwrite-lint/rate_limiter.py`. `error` from a transport failure or 429/5xx is
+surfaced distinctly so the pipeline never treats an outage as 'not found'.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from collections.abc import AsyncIterator, Awaitable, Callable
+from contextlib import asynccontextmanager
+
+import httpx
+from curl_cffi.requests import AsyncSession
+from curl_cffi.requests.exceptions import RequestException as ImpersonateRequestError
+from tenacity import (
+    retry,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_exponential,
+)
+
+DEFAULT_TIMEOUT = 30.0
+DEFAULT_USER_AGENT = "reference-audit/0.1 (https://github.com/; mailto:reference-audit@example.org)"
+
+# Some publisher platforms (Silverchair direct.mit.edu, Atypon journals.sagepub.com) sit behind
+# Cloudflare, which fingerprints the TLS ClientHello (JA3/JA4). httpx's fingerprint is flagged as a
+# bot and 403'd even with a full browser header set + HTTP/2; curl/Firefox pass. curl_cffi replays a
+# real browser's ClientHello, so the export resolves. Used ONLY for those publisher fetches —
+# the clean-API sources stay on httpx.
+DEFAULT_IMPERSONATE = "chrome"
+
+
+class TransientHTTPError(Exception):
+    """A retryable failure: transport error or 429/5xx response."""
+
+
+class MonotonicRateLimiter:
+    """Token-free min-interval limiter shared across coroutines (monotonic clock), with an optional
+    cap on requests in flight.
+
+    The interval spaces request *starts*, which alone does not bound concurrency: a source whose
+    searches take seconds (Crossref) piles up dozens of open requests at 10 starts/s and answers 429.
+    `max_in_flight` caps them; a slot is held for a whole request, retries and backoff included.
+    """
+
+    def __init__(self, rate_per_sec: float, max_in_flight: int | None = None):
+        self._min_interval = 1.0 / rate_per_sec if rate_per_sec > 0 else 0.0
+        self._last = 0.0
+        self._lock = asyncio.Lock()
+        self.max_in_flight = max_in_flight
+        self._slots = asyncio.Semaphore(max_in_flight) if max_in_flight else None
+
+    async def acquire(self) -> None:
+        if self._min_interval <= 0:
+            return
+        async with self._lock:
+            now = time.monotonic()
+            wait = self._last + self._min_interval - now
+            if wait > 0:
+                await asyncio.sleep(wait)
+            self._last = time.monotonic()
+
+    @asynccontextmanager
+    async def slot(self) -> AsyncIterator[None]:
+        """Hold one in-flight slot for the duration of a request (a no-op without a cap)."""
+        if self._slots is None:
+            yield
+            return
+        async with self._slots:
+            yield
+
+
+def new_client(user_agent: str = DEFAULT_USER_AGENT, timeout: float = DEFAULT_TIMEOUT) -> httpx.AsyncClient:
+    return httpx.AsyncClient(timeout=timeout, headers={"User-Agent": user_agent}, follow_redirects=True)
+
+
+def is_transient_status(resp: httpx.Response) -> bool:
+    """The default definition of a retryable response: rate-limited, or a server-side failure."""
+    return resp.status_code == 429 or resp.status_code >= 500
+
+
+@retry(
+    stop=stop_after_attempt(3),
+    wait=wait_exponential(multiplier=0.5, min=0.5, max=8),
+    retry=retry_if_exception_type(TransientHTTPError),
+    reraise=True,
+)
+async def _send_with_retry(
+    send: Callable[[], Awaitable[httpx.Response]],
+    transient: Callable[[httpx.Response], bool] = is_transient_status,
+) -> httpx.Response:
+    """Verb-agnostic retry core: one definition of "what is transient" for every request we make.
+
+    `transient` is injectable because not every 5xx means "the server is unwell". GROBID reports a
+    *per-document* parse failure through a 500 body, and retrying that three times is pure latency
+    while reporting it as an outage would mislabel an unparseable PDF as a service problem.
+    """
+    try:
+        resp = await send()
+    except httpx.TransportError as exc:  # network/DNS/timeout
+        raise TransientHTTPError(f"transport: {exc}") from exc
+    if transient(resp):
+        raise TransientHTTPError(f"http {resp.status_code}")
+    return resp
+
+
+async def _request_with_retry(
+    client: httpx.AsyncClient,
+    limiter: MonotonicRateLimiter,
+    url: str,
+    params: dict | None,
+    headers: dict | None,
+) -> httpx.Response:
+    """Every attempt, retries included, waits its turn at the limiter. The in-flight slot is held
+    across the backoff, so a source that answered 429 is not handed the next request meanwhile."""
+
+    async def send() -> httpx.Response:
+        await limiter.acquire()
+        return await client.get(url, params=params, headers=headers)
+
+    async with limiter.slot():
+        return await _send_with_retry(send)
+
+
+async def post_multipart(
+    client: httpx.AsyncClient,
+    url: str,
+    *,
+    files: dict,
+    data: dict | None = None,
+    timeout: float | None = None,
+    transient: Callable[[httpx.Response], bool] = is_transient_status,
+) -> httpx.Response:
+    """Multipart POST under the same transient-retry policy as the GET helpers.
+
+    Returns the raw response — unlike `get_json`/`get_text` the caller classifies non-2xx itself,
+    because a multipart-POST service (GROBID) encodes distinct per-request failure modes in the body
+    and they need different handling. Not rate-limited: the only caller talks to a single-tenant local
+    service one request per run.
+    """
+    return await _send_with_retry(
+        lambda: client.post(url, files=files, data=data, timeout=timeout), transient
+    )
+
+
+async def get_json(
+    client: httpx.AsyncClient,
+    limiter: MonotonicRateLimiter,
+    url: str,
+    *,
+    params: dict | None = None,
+    headers: dict | None = None,
+) -> tuple[int, dict | None]:
+    """Rate-limited GET with retry. Returns (status_code, json-or-None).
+
+    Raises TransientHTTPError after exhausting retries (caller maps to SourceQueryResult.error).
+    A 404 returns (404, None) — a genuine 'absent', distinct from an error.
+    """
+    resp = await _request_with_retry(client, limiter, url, params, headers)
+    if resp.status_code == 404:
+        return 404, None
+    if resp.status_code >= 400:
+        # non-retryable client error (e.g. 400/403) — treat as a hard error, not 'absent'
+        raise TransientHTTPError(f"http {resp.status_code}")
+    try:
+        return resp.status_code, resp.json()
+    except ValueError as exc:
+        raise TransientHTTPError(f"invalid json: {exc}") from exc
+
+
+async def get_text(
+    client: httpx.AsyncClient,
+    limiter: MonotonicRateLimiter,
+    url: str,
+    *,
+    params: dict | None = None,
+    headers: dict | None = None,
+) -> tuple[int, str]:
+    """Rate-limited GET with retry for text/XML payloads (e.g. the arXiv Atom API).
+
+    Mirrors `get_json`: exponential backoff on 429/5xx via `_request_with_retry`, then raises
+    TransientHTTPError after exhausting retries. A 404 returns (404, "").
+    """
+    resp = await _request_with_retry(client, limiter, url, params, headers)
+    if resp.status_code == 404:
+        return 404, ""
+    if resp.status_code >= 400:
+        # non-retryable client error (e.g. 400/403) — treat as a hard error, not 'absent'
+        raise TransientHTTPError(f"http {resp.status_code}")
+    return resp.status_code, resp.text
+
+
+async def get_html(
+    client: httpx.AsyncClient,
+    limiter: MonotonicRateLimiter,
+    url: str,
+    *,
+    headers: dict | None = None,
+) -> tuple[int, str, str]:
+    """Rate-limited GET of an HTML page, returning (status, final_url, text).
+
+    Unlike `get_text`, the *final* URL (after redirects) is returned too, so a web-artifact check can
+    record where the citation actually landed. A dead link — HTTP 404 or 410 (Gone) — returns
+    (status, final_url, "") so the caller can report it as a dead link rather than an outage; any
+    other 4xx (e.g. a 403 bot-wall) raises TransientHTTPError so it is reported as a block, never a
+    false 'page absent'.
+    """
+    resp = await _request_with_retry(client, limiter, url, None, headers)
+    if resp.status_code in (404, 410):
+        return resp.status_code, str(resp.url), ""
+    if resp.status_code >= 400:
+        raise TransientHTTPError(f"http {resp.status_code}")
+    return resp.status_code, str(resp.url), resp.text
+
+
+def new_impersonate_session(timeout: float = DEFAULT_TIMEOUT) -> AsyncSession:
+    """A browser-TLS-impersonating session (libcurl via curl_cffi) for Cloudflare-fingerprint-walled
+    endpoints. Reused across calls so the issued `__cf_bm` cookie persists. See DEFAULT_IMPERSONATE.
+    """
+    return AsyncSession(timeout=timeout, impersonate=DEFAULT_IMPERSONATE)
+
+
+@retry(
+    stop=stop_after_attempt(3),
+    wait=wait_exponential(multiplier=0.5, min=0.5, max=8),
+    retry=retry_if_exception_type(TransientHTTPError),
+    reraise=True,
+)
+async def _impersonate_get_with_retry(
+    session: AsyncSession, limiter: MonotonicRateLimiter, url: str
+):
+    await limiter.acquire()  # per attempt, as in `_request_with_retry`
+    try:
+        resp = await session.get(url)
+    except ImpersonateRequestError as exc:  # network/DNS/timeout/TLS
+        raise TransientHTTPError(f"transport: {exc}") from exc
+    if resp.status_code == 429 or resp.status_code >= 500:
+        raise TransientHTTPError(f"http {resp.status_code}")
+    return resp
+
+
+async def get_text_impersonate(
+    session: AsyncSession,
+    limiter: MonotonicRateLimiter,
+    url: str,
+) -> tuple[int, str]:
+    """Browser-impersonating counterpart of `get_text` for Cloudflare-fingerprint-walled publisher
+    exports. Same contract: rate-limited, exponential backoff on transport/429/5xx, then raises
+    TransientHTTPError after exhausting retries; a 404 returns (404, ""); any other 4xx (e.g. a 403
+    bot-wall) raises TransientHTTPError so the caller reports a block rather than a false 'absent'.
+    """
+    async with limiter.slot():
+        resp = await _impersonate_get_with_retry(session, limiter, url)
+    if resp.status_code == 404:
+        return 404, ""
+    if resp.status_code >= 400:
+        raise TransientHTTPError(f"http {resp.status_code}")
+    return resp.status_code, resp.text