scholarmend 0.1.0__py3-none-any.whl

scholarmend/__init__.py

@@ -0,0 +1,3 @@
+from ._version import __version__
+
+__all__ = ["__version__"]

scholarmend/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

scholarmend/cache.py

@@ -0,0 +1,80 @@
+"""A content-addressed response store, committed to the repository.
+
+Reproducibility is the point. A systematic review is reported once and defended
+for years; a rerun has to produce what the first run produced. Every cached file
+records the key alongside the payload so that the store stays auditable by
+reading it, rather than only by replaying it.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import sys
+import tempfile
+from collections.abc import Callable
+from pathlib import Path
+
+
+class CacheMiss(RuntimeError):
+    """Offline mode was asked for something the cache does not hold."""
+
+
+class Cache:
+    def __init__(self, root: Path, offline: bool = False) -> None:
+        self.root = Path(root)
+        self.offline = offline
+
+    def _path(self, key: str) -> Path:
+        # Hashing keeps arbitrary keys -- titles, URLs -- safe as filenames.
+        digest = hashlib.sha256(key.encode("utf-8")).hexdigest()
+        return self.root / digest[:2] / f"{digest}.json"
+
+    def get(self, key: str) -> dict | None:
+        path = self._path(key)
+        if not path.exists():
+            return None
+        try:
+            return json.loads(path.read_text(encoding="utf-8"))["payload"]
+        except (json.JSONDecodeError, KeyError, UnicodeDecodeError) as error:
+            # A damaged entry is a miss, loudly. Raising instead would abort a
+            # 2,413-record run over one unreadable file; staying silent would
+            # hide a cache the repository is supposed to be able to audit.
+            print(f"cache entry {path} is unreadable ({error}); treating as a miss",
+                  file=sys.stderr)
+            return None
+
+    def put(self, key: str, payload: dict) -> None:
+        path = self._path(key)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        document = {"key": key, "payload": payload}
+        text = json.dumps(document, indent=1, sort_keys=True, ensure_ascii=False)
+        # Write-then-rename, because a run interrupted mid-write used to leave
+        # truncated JSON behind, and the next run raised JSONDecodeError on a
+        # file it had written itself. The temporary file shares the directory
+        # so that the replace is atomic on one filesystem.
+        descriptor, temporary = tempfile.mkstemp(
+            dir=path.parent, prefix=path.name, suffix=".tmp"
+        )
+        try:
+            with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
+                handle.write(text)
+            os.replace(temporary, path)
+        except BaseException:
+            Path(temporary).unlink(missing_ok=True)
+            raise
+
+    def fetch(self, key: str, loader: Callable[[], dict]) -> dict:
+        """Cached value, or ``loader()`` stored and returned."""
+        hit = self.get(key)
+        if hit is not None:
+            return hit
+        if self.offline:
+            raise CacheMiss(
+                f"{key!r} is not cached and --offline forbids fetching it. "
+                f"Run once without --offline to populate the cache."
+            )
+        payload = loader()
+        self.put(key, payload)
+        return payload

scholarmend/cli.py

@@ -0,0 +1,211 @@
+"""Command line entry point."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+from .cache import Cache, CacheMiss
+from .emit import project_ris, to_json
+from .http import HttpError
+from .parse import parse_file
+from .pipeline import RESOLVED_FIELDS, resolve_record
+from .resolvers.openreview import AuthError, OpenReviewResolver, login
+from .resolvers.pmc import PmcResolver
+from .resolvers.pmlr_index import PmlrIndexResolver
+from .resolvers.proceedings_page import ProceedingsPageResolver
+from .resolvers.semanticscholar import SemanticScholarResolver
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="scholarmend",
+        description="Recover true venue, year and track for Google Scholar RIS exports.",
+    )
+    parser.add_argument("--input", required=True, type=Path,
+                        help="a .ris file, or a directory of them")
+    parser.add_argument("--out", required=True, type=Path, help="directory to write outputs to")
+    parser.add_argument("--cache", default=Path(".scholarmend-cache"), type=Path,
+                        help="response cache; commit it for reproducibility")
+    parser.add_argument("--offline", action="store_true",
+                        help="never call the network; fail on a cache miss")
+    parser.add_argument("--abstracts", action="store_true",
+                        help="replace Scholar's AB snippet with the full abstract, from the "
+                             "proceedings page, OpenReview or Semantic Scholar")
+    return parser
+
+
+def _report(documents: list[dict], unresolved: list[tuple[str, list[str]]],
+            abstracts_requested: bool = False) -> str:
+    """What a human still has to look at, with the structurally impossible removed.
+
+    A field no source in this run could supply -- doi, which nothing resolved
+    for any of the 2,413 records -- accused every record of a failure no human
+    could act on, and buried the genuinely unresolved venues and years under
+    1,854 lines of noise. Emptiness across the whole corpus is the evidence
+    that no source exists for it, so the field is named once and then left out
+    of the per-record list.
+    """
+    supplied = {field for field in RESOLVED_FIELDS
+                if any(doc["fields"][field].get("value") for doc in documents)}
+    absent = [f for f in RESOLVED_FIELDS if f not in supplied]
+
+    rows = [(title, [f for f in missing if f in supplied]) for title, missing in unresolved]
+    rows = [(title, missing) for title, missing in rows if missing]
+    # Venue and year are the deliverable; a record missing one of them comes
+    # first, so that truncating the list cannot hide it.
+    rows.sort(key=lambda row: not ({"venue", "year"} & set(row[1])))
+
+    counts = " ".join(
+        f"{field}={sum(1 for _, missing in rows if field in missing)}" for field in RESOLVED_FIELDS
+    )
+    lines = [
+        f"records: {len(documents)}",
+        f"records with unresolved fields: {len(rows)}",
+        f"unresolved by field: {counts}",
+    ]
+    if absent:
+        lines.append(
+            f"no source in this run could supply: {','.join(absent)} (excluded above and below)"
+        )
+    # Scholar's AB always "resolves", to its own snippet, so the count above
+    # cannot show these. They are the records a screener reads a fragment for.
+    snippets = [doc["title"] for doc in documents
+                if abstracts_requested and doc["fields"]["abstract"].get("source") == "scholar"]
+    if abstracts_requested:
+        lines.append(f"abstract still Scholar's snippet: {len(snippets)}")
+    lines.append("")
+    for title, missing in rows[:200]:
+        lines.append(f"  unresolved {','.join(missing):28s} {title[:70]}")
+    if snippets:
+        lines.append("")
+    for title in snippets:
+        lines.append(f"  snippet    {title[:70]}")
+    return "\n".join(lines) + "\n"
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _build_parser().parse_args(argv)
+
+    # A single file is how venuetriage's clean.ris -- the Covidence upload,
+    # already deduplicated and triaged -- gets its abstracts: its directory
+    # also holds removed.ris, which must not be merged into the upload.
+    files = [args.input] if args.input.is_file() else sorted(args.input.glob("*.ris"))
+    if not files:
+        print(f"no .ris files in {args.input}", file=sys.stderr)
+        return 2
+
+    cache = Cache(args.cache, offline=args.offline)
+    degraded = False
+
+    user = os.environ.get("SCHOLARMEND_OPENREVIEW_USER")
+    password = os.environ.get("SCHOLARMEND_OPENREVIEW_PASSWORD")
+    credentialed = bool(user and password) and not args.offline
+    warm = args.cache.exists()
+    token: dict[str, str | None] = {}
+
+    def openreview_token() -> str | None:
+        """Log in on the first genuine cache miss, and never before.
+
+        A committed cache holds every answer tier 2 needs, so a warm run must
+        make no API call at all -- including the login the run used to perform
+        unconditionally. Deferring it here is what makes the README's "makes no
+        API calls" true of the path a third party actually runs.
+        """
+        nonlocal degraded
+        if not credentialed:
+            return None
+        if "value" not in token:
+            try:
+                token["value"] = login(str(user), str(password))
+            except Exception as error:  # noqa: BLE001 - degrade, never crash the run
+                print(f"OpenReview login failed, continuing on tier 1: {error}", file=sys.stderr)
+                degraded = True
+                token["value"] = None
+        return token["value"]
+
+    if not credentialed and not args.offline:
+        if warm:
+            # The reproducibility path: a committed cache and no account. Tier 2
+            # is not disabled here -- only a forum the cache does not hold will
+            # degrade this run, one record at a time.
+            print("no OpenReview credentials; tier 2 served from the cache", file=sys.stderr)
+        else:
+            print("no OpenReview credentials and no cache; tier 2 disabled", file=sys.stderr)
+            degraded = True
+
+    # Gate on whether an answer is reachable at all, not on credentials alone.
+    openreview = (
+        OpenReviewResolver(cache, openreview_token)
+        if (credentialed or warm or args.offline)
+        else None
+    )
+    pmlr_index = PmlrIndexResolver(cache)
+    pmc = PmcResolver(cache)
+    semanticscholar = SemanticScholarResolver(cache, os.environ.get("SCHOLARMEND_S2_KEY"))
+    proceedings_page = ProceedingsPageResolver(cache)
+
+    abstract_errors: list[str] = []
+
+    def abstract_failed(error: Exception) -> None:
+        """An abstract lookup failed: keep the snippet, and say so once."""
+        nonlocal degraded
+        if not abstract_errors:
+            print(f"some abstracts not recovered, Scholar's snippet kept: {error}",
+                  file=sys.stderr)
+        abstract_errors.append(str(error))
+        degraded = True
+
+    documents, projections, unresolved = [], [], []
+    for path in files:
+        for record in parse_file(path):
+            try:
+                ledger = resolve_record(record, openreview=openreview,
+                                        pmlr_index=pmlr_index, pmc=pmc,
+                                        semanticscholar=semanticscholar,
+                                        proceedings_page=proceedings_page,
+                                        abstracts=args.abstracts,
+                                        on_error=abstract_failed)
+            except (AuthError, CacheMiss, HttpError) as error:
+                # Degrade, never abort. One unreachable lookup must not cost
+                # the other 2,412 records their tier-1 resolution, which needs
+                # no network at all. The run reports itself partial via exit 1.
+                if not degraded:
+                    print(f"falling back to tier 1 for some records: {error}", file=sys.stderr)
+                degraded = True
+                # The proceedings page needs no key the failed lookup could
+                # have supplied, so the abstract is still worth trying.
+                ledger = resolve_record(record, openreview=None, pmlr_index=None,
+                                        pmc=None, semanticscholar=None,
+                                        proceedings_page=proceedings_page,
+                                        abstracts=args.abstracts,
+                                        on_error=abstract_failed)
+            documents.append(to_json(record, ledger))
+            projections.append(project_ris(record, ledger))
+            unresolved.append((record.title, [f for f in RESOLVED_FIELDS
+                                              if ledger.is_unresolved(f)]))
+
+    args.out.mkdir(parents=True, exist_ok=True)
+    (args.out / "resolved.json").write_text(
+        json.dumps(documents, indent=1, sort_keys=True, ensure_ascii=False), encoding="utf-8"
+    )
+    # utf-8-sig, because parse_file reads utf-8-sig: without the BOM the file
+    # round-trips record by record but not as a file.
+    (args.out / "mended.ris").write_text("".join(projections), encoding="utf-8-sig")
+
+    (args.out / "report.txt").write_text(_report(documents, unresolved, args.abstracts), encoding="utf-8")
+
+    recovered = sum(1 for doc in documents
+                    if doc["fields"]["abstract"].get("source") not in (None, "scholar"))
+    print(f"wrote {len(documents)} records to {args.out}")
+    if args.abstracts:
+        print(f"{recovered} with a recovered abstract; "
+              f"{len(abstract_errors)} abstract lookups failed")
+    return 1 if degraded else 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

scholarmend/emit.py

@@ -0,0 +1,129 @@
+"""Canonical JSON, and the RIS projection derived from it.
+
+The JSON is the record. RIS cannot represent provenance, so it is emitted as a
+projection for the tools that need it -- Covidence, and venuetriage -- by
+rewriting only the lines whose values scholarmend corrected, and inserting a
+line for a corrected field the record never carried. Every other byte of the
+original record survives, which is what makes the projection safe to feed to a
+parser that was written against Scholar's exact output.
+"""
+
+from __future__ import annotations
+
+import re
+
+from .ledger import PRECEDENCE, Ledger
+from .models import Record
+from .pipeline import RESOLVED_FIELDS
+
+# Which RIS tag carries each resolved field. Only these are ever rewritten.
+#
+# Authors are deliberately absent. Rewriting AU means deleting N lines and
+# inserting M -- restructuring a repeated field -- which is a different act from
+# substituting one line or inserting one missing line, and puts the round-trip
+# guarantee -- the projection's whole safety argument -- at risk. Resolved
+# authors live in the canonical JSON, which is the record; RIS is a projection
+# for tools that cannot read it.
+#
+# Abstracts are present because AB is not that case: every corpus record has
+# exactly one AB line, and every abstract claim is collapsed to one line, so a
+# recovered abstract is a one-line substitution like PY or JF. Screening in
+# Covidence reads AB, and Scholar's is a snippet on every record.
+_TAG_FOR = {"year": "PY", "venue": "JF", "abstract": "AB"}
+
+
+def to_json(record: Record, ledger: Ledger) -> dict:
+    """The canonical record: winning values, and every claim behind them."""
+    fields: dict[str, dict[str, object]] = {}
+    for field in RESOLVED_FIELDS:
+        claim = ledger.resolve(field)
+        if claim is None:
+            fields[field] = {"value": None, "unresolved": True}
+        else:
+            fields[field] = {
+                "value": claim.value,
+                "source": claim.source,
+                "tier": claim.tier,
+                "confidence": claim.confidence,
+                "evidence": claim.evidence,
+            }
+    # Every field the precedence table knows about, not a hand-listed subset.
+    # The subset dropped pmlr_volume, pmc_id and arxiv_id from the canonical
+    # JSON entirely -- losing the evidence on exactly the records a human has
+    # to adjudicate, which are the ones no resolver settled.
+    claims = [
+        {"field": c.field, "value": c.value, "source": c.source,
+         "tier": c.tier, "confidence": c.confidence, "evidence": c.evidence}
+        for field in sorted(set(PRECEDENCE) | set(RESOLVED_FIELDS))
+        for c in ledger.claims(field)
+    ]
+    return {
+        "title": record.title,
+        "source_file": record.source_file,
+        "fields": fields,
+        "claims": claims,
+        "conflicts": ledger.conflicts(),
+    }
+
+
+def _line(tag: str, value: str) -> str:
+    """One RIS line, preserving Scholar's ``2025///`` year shape."""
+    if tag == "PY":
+        return f"PY  - {value}///"
+    # One value, one line, whatever a resolver handed over: a newline here
+    # would split the field into a continuation line no downstream parser
+    # agrees on how to read.
+    return f"{tag}  - {' '.join(value.split())}"
+
+
+def _insert_at(lines: list[str]) -> int:
+    """Where a missing tag's line goes: immediately before ``ER  - ``.
+
+    A record with no ``ER`` line at all is malformed, but refusing to emit it
+    would be a false drop -- the failure mode the design calls silent and
+    unrecoverable -- so the line is appended after the last non-blank line
+    instead, which keeps the record's trailing newline where it was.
+    """
+    for index, line in enumerate(lines):
+        if line.startswith("ER  -"):
+            return index
+    last = max((i for i, line in enumerate(lines) if line.strip()), default=-1)
+    return last + 1
+
+
+def project_ris(record: Record, ledger: Ledger) -> str:
+    """``record.raw`` with corrected fields rewritten and nothing else touched.
+
+    A tag that already has a line is substituted in place. A tag that has none
+    is inserted immediately before ``ER  - ``: 331 corpus records -- every one
+    of them from OpenReview -- carry no ``PY`` line at all, which is precisely
+    the population tier 2 exists to supply a year for, and substitution alone
+    would drop every year it resolved for them.
+
+    Inserting one line is not the same as restructuring a multi-line field.
+    ``AU`` stays out of ``_TAG_FOR`` for exactly that reason.
+    """
+    corrections = {}
+    for field, tag in _TAG_FOR.items():
+        claim = ledger.resolve(field)
+        if claim is not None and claim.source != "scholar":
+            corrections[tag] = claim.value
+    if not corrections:
+        return record.raw
+
+    out = []
+    substituted = set()
+    for line in record.raw.split("\n"):
+        match = re.match(r"^([A-Z][A-Z0-9])  - ", line)
+        line_tag: str | None = match.group(1) if match else None
+        if line_tag in corrections:
+            substituted.add(line_tag)
+            out.append(_line(line_tag, corrections[line_tag]))
+        else:
+            out.append(line)
+
+    missing = [tag for tag in _TAG_FOR.values() if tag in corrections and tag not in substituted]
+    if missing:
+        index = _insert_at(out)
+        out[index:index] = [_line(tag, corrections[tag]) for tag in missing]
+    return "\n".join(out)

scholarmend/http.py

@@ -0,0 +1,133 @@
+"""A small JSON-over-HTTP client.
+
+stdlib only, matching refaudit's decision to carry no runtime dependencies:
+this gets installed in a hurry, close to a deadline, often on a machine someone
+else administers.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+import urllib.error
+import urllib.request
+
+
+class HttpError(RuntimeError):
+    """A request that failed. ``status`` and ``body`` are set for an HTTP
+    refusal, so a resolver can tell "not on this API" from "not visible to
+    you" without parsing the message."""
+
+    def __init__(self, message: str, status: int | None = None, body: str = "") -> None:
+        super().__init__(message)
+        self.status = status
+        self.body = body
+
+
+# The largest window any server here advertises is one hour. Cap the wait so a
+# malformed or hostile header cannot park a run indefinitely.
+MAX_RATELIMIT_WAIT = 3700.0
+
+
+def _advertised_wait(headers) -> float | None:
+    """The pause the server itself asked for, or ``None`` if it asked for none.
+
+    ``ratelimit-reset`` is what OpenReview sends; ``retry-after`` is what
+    several other servers send instead, in its seconds form. Either is a far
+    better estimate than exponential backoff, which on a real 429 burns three
+    attempts in three seconds and then gives up while the window still has
+    fifty minutes to run.
+    """
+    if headers is None:
+        return None
+    for name in ("ratelimit-reset", "retry-after"):
+        raw = headers.get(name)
+        if raw is None:
+            continue
+        text = str(raw).strip()
+        if text.replace(".", "", 1).isdigit():
+            return min(max(float(text), 0.0), MAX_RATELIMIT_WAIT) + 1.0
+    return None
+
+
+def _respect_rate_limit(response, sleep=time.sleep) -> None:
+    """Pause when the server says the budget is spent.
+
+    OpenReview advertises ``ratelimit-policy: 500;w=3600`` -- 500 requests an
+    hour, per token. Tier 2 needs 527 forum lookups, so an unbroken run runs
+    out 27 calls from the end. Waiting for the window to roll over turns that
+    from a failed run into a slow one, and because every response is cached,
+    the wait is paid once ever rather than once per run.
+    """
+    remaining = response.headers.get("ratelimit-remaining")
+    if remaining is None or not remaining.strip().isdigit():
+        return
+    if int(remaining) > 0:
+        return
+    reset = response.headers.get("ratelimit-reset", "")
+    seconds = float(reset) if reset.strip().replace(".", "", 1).isdigit() else 60.0
+    sleep(min(max(seconds, 0.0), MAX_RATELIMIT_WAIT) + 1.0)
+
+
+def _read_body(error: urllib.error.HTTPError) -> str:
+    try:
+        return error.read(2000).decode("utf-8", "replace")
+    except (OSError, AttributeError):  # a closed or absent response body
+        return ""
+
+
+def get_json(
+    url: str,
+    headers: dict[str, str] | None = None,
+    timeout: float = 20.0,
+    attempts: int = 3,
+    sleep=time.sleep,
+) -> dict:
+    """GET ``url`` and parse JSON, retrying on transient failure. See ``get_text``."""
+    return json.loads(get_text(url, headers, timeout, attempts, sleep, parse=json.loads))
+
+
+def get_text(
+    url: str,
+    headers: dict[str, str] | None = None,
+    timeout: float = 20.0,
+    attempts: int = 3,
+    sleep=time.sleep,
+    parse=None,
+) -> str:
+    """GET ``url`` as text, retrying on transient failure.
+
+    429 and 5xx are retried; 4xx other than 429 is raised at once, because
+    retrying a refusal only spends someone else's rate limit. A 429 waits for
+    as long as the server's own headers say the window has left, rather than
+    for the exponential backoff the other failures get: OpenReview's window is
+    an hour wide, so backing off for one second and then two and then giving up
+    fails the run while doing nothing to let the budget recover.
+
+    ``parse``, if given, is applied inside the retry loop only to validate the
+    body, so that a truncated JSON response is retried like any other
+    transient failure.
+    """
+    last: Exception | None = None
+    for attempt in range(attempts):
+        wait: float | None = None
+        request = urllib.request.Request(url, headers=headers or {})
+        try:
+            with urllib.request.urlopen(request, timeout=timeout) as response:
+                body = response.read().decode("utf-8")
+                if parse is not None:
+                    parse(body)
+                _respect_rate_limit(response, sleep)
+                return body
+        except urllib.error.HTTPError as error:
+            last = error
+            if error.code != 429 and error.code < 500:
+                raise HttpError(f"{error.code} from {url}", status=error.code,
+                                body=_read_body(error)) from error
+            if error.code == 429:
+                wait = _advertised_wait(getattr(error, "headers", None))
+        except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as error:
+            last = error
+        if attempt < attempts - 1:
+            sleep(2.0**attempt if wait is None else wait)
+    raise HttpError(f"giving up on {url}: {last!r}")

scholarmend/ledger.py

@@ -0,0 +1,88 @@
+"""Claim accumulation and resolution.
+
+The precedence table is the whole policy, and it is data. A source listed for
+a field may answer it, best first; a source *absent* from a field's tuple is
+excluded from answering it at all. That single mechanism expresses both
+ordering and exclusion -- notably that preprint-derived sources may supply an
+abstract and a full author list, which preprints genuinely have, but must never
+supply a venue or a year, which they get wrong by construction.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+
+from .models import Claim
+
+PRECEDENCE: dict[str, tuple[str, ...]] = {
+    # Scholar loses all 1,264 year disagreements measured on the corpus.
+    "year": ("proceedings_url", "openreview_api", "pmlr_index", "semanticscholar", "scholar"),
+    # The OpenReview venueid was correct in 92 of 92 hand-checked cases.
+    "venue": ("openreview_api", "proceedings_url", "pmlr_index", "semanticscholar", "scholar"),
+    "track": ("openreview_api", "proceedings_url"),
+    # pmlr_index emits "PMLR v318" as a venue_id too. Listing only
+    # openreview_api here made resolve("venue_id") return None for every
+    # PMLR record, discarding the identity evidence a human adjudicating
+    # one of them needs.
+    "venue_id": ("openreview_api", "pmlr_index"),
+    "version": ("proceedings_url", "openreview_api", "pmlr_index", "arxiv_url"),
+    # Preprint sources are welcome here: Scholar truncates 71% of author lists.
+    "authors": ("openreview_api", "semanticscholar", "openalex", "arxiv_url", "scholar"),
+    # Scholar's AB is a search snippet on every record, so it answers last.
+    # The proceedings page is the published abstract itself; it outranks the
+    # OpenReview note, which a camera-ready revision can leave behind.
+    "abstract": ("proceedings_page", "openreview_api", "semanticscholar", "openalex",
+                 "arxiv_url", "scholar"),
+    "doi": ("openreview_api", "semanticscholar", "openalex", "arxiv_url"),
+    # Keys, not answers: these carry an identifier from tier 1 to tier 2.
+    "forum_id": ("openreview_url",),
+    "pmlr_volume": ("pmlr_url", "pmc_api"),
+    "arxiv_id": ("arxiv_url",),
+    "pmc_id": ("pmc_url",),
+}
+
+
+class Ledger:
+    """Every claim made about one record, and the policy for choosing between them."""
+
+    def __init__(self) -> None:
+        self._claims: dict[str, list[Claim]] = defaultdict(list)
+
+    def add(self, claim: Claim) -> None:
+        if claim not in self._claims[claim.field]:
+            self._claims[claim.field].append(claim)
+
+    def claims(self, field: str) -> list[Claim]:
+        """Every claim for ``field``, including ones excluded by precedence."""
+        return list(self._claims.get(field, []))
+
+    def resolve(self, field: str) -> Claim | None:
+        """The winning claim, or ``None`` if no permitted source answered."""
+        order = PRECEDENCE.get(field)
+        if not order:
+            return None
+        for source in order:
+            for claim in self._claims.get(field, []):
+                if claim.source == source:
+                    return claim
+        return None
+
+    def is_unresolved(self, field: str) -> bool:
+        return self.resolve(field) is None
+
+    def conflicts(self) -> list[str]:
+        """Fields where two sources *at the same tier* disagree.
+
+        Cross-tier disagreement is the normal case -- Scholar against a mined
+        URL happens 1,264 times in the corpus -- so flagging it would drown the
+        report. Same-tier disagreement means two equally trusted sources cannot
+        both be right, which is worth a human's attention.
+        """
+        out = []
+        for field, claims in self._claims.items():
+            by_tier: dict[int, set[str]] = defaultdict(set)
+            for claim in claims:
+                by_tier[claim.tier].add(claim.value)
+            if any(len(values) > 1 for values in by_tier.values()):
+                out.append(field)
+        return sorted(out)

scholarmend/miners/__init__.py

@@ -0,0 +1,29 @@
+"""The miner registry.
+
+Order matters only for readability; miners are disjoint by host, so at most one
+answers any given URL.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable
+
+from ..models import Claim
+from . import arxiv, openreview, pmc, pmlr, proceedings
+
+ALL: tuple[Callable[[str], list[Claim]], ...] = (
+    proceedings.mine,
+    openreview.mine,
+    pmlr.mine,
+    arxiv.mine,
+    pmc.mine,
+)
+
+
+def mine_all(urls: Iterable[str]) -> list[Claim]:
+    """Every claim every miner can make about every URL on a record."""
+    claims: list[Claim] = []
+    for url in urls:
+        for miner in ALL:
+            claims.extend(miner(url))
+    return claims