ghostcite 0.1.0__py3-none-any.whl

ghostcite/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

ghostcite/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from ghostcite.cli import main
+
+sys.exit(main())

ghostcite/_pace.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import re
+import time
+from collections.abc import Mapping
+
+_INTERVAL_RE = re.compile(r"(\d+)\s*s")
+
+
+class _Pacer:
+    """Client-side request pacer enforcing a minimum interval between calls.
+
+    The floor is the larger of an explicit ``--max-rps`` ceiling and whatever
+    the server advertises via ``x-rate-limit-limit`` / ``x-rate-limit-interval``
+    response headers, so the more conservative of the two always wins.
+    """
+
+    def __init__(self, max_rps: float | None) -> None:
+        if max_rps is not None and max_rps <= 0:
+            raise ValueError("max_rps must be positive")
+        self._min_interval = 1.0 / max_rps if max_rps else 0.0
+        self._last: float | None = None
+
+    def update_from_headers(self, headers: Mapping[str, str]) -> None:
+        """Raise the floor from CrossRef rate headers. Malformed/missing → no-op."""
+        raw_limit = headers.get("x-rate-limit-limit")
+        raw_interval = headers.get("x-rate-limit-interval")
+        if raw_limit is None or raw_interval is None:
+            return
+        try:
+            limit = int(raw_limit)
+        except (TypeError, ValueError):
+            return
+        if limit <= 0:
+            return
+        m = _INTERVAL_RE.fullmatch(raw_interval.strip())
+        if not m:
+            return
+        interval_sec = int(m.group(1))
+        self._min_interval = max(self._min_interval, interval_sec / limit)
+
+    def wait(self) -> None:
+        """Block until at least ``_min_interval`` has elapsed since the last call."""
+        now = time.monotonic()
+        if self._last is None:
+            # First call: prime the clock, don't sleep.
+            self._last = now
+            return
+        delay = self._min_interval - (now - self._last)
+        if delay > 0:
+            time.sleep(delay)
+        self._last = time.monotonic()

ghostcite/cli.py

@@ -0,0 +1,174 @@
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from contextlib import ExitStack
+
+from ghostcite import __version__
+from ghostcite.compare import cross_check_pubmed, evaluate
+from ghostcite.crossref import CrossRefClient
+from ghostcite.models import Finding, Tier
+from ghostcite.parsers import parse
+from ghostcite.pubmed import PubMedClient
+from ghostcite.report import render_json, render_text
+
+_TIER_BY_NAME = {
+    "author": Tier.AUTHOR,
+    "year": Tier.YEAR,
+    "retraction": Tier.RETRACTION,
+}
+
+
+def _parse_args(argv):
+    p = argparse.ArgumentParser(
+        prog="ghostcite",
+        description="Catch ghost citations: cross-check claimed author/year against CrossRef.",
+    )
+    p.add_argument("--version", action="version", version=f"ghostcite {__version__}")
+    p.add_argument(
+        "file",
+        help="bibliography file (.bib, markdown refs, or DOI list), or '-' for stdin",
+    )
+    p.add_argument("--format", choices=["auto", "bibtex", "markdown", "doi"], default="auto")
+    p.add_argument("--json", action="store_true", help="machine-readable output")
+    p.add_argument("--dry-run", action="store_true", help="parse + count only, no network")
+    p.add_argument(
+        "--fail-on",
+        default="author,year,retraction",
+        help="comma list of tiers that cause exit 1, or 'none' "
+        "(choices: author,year,retraction,none)",
+    )
+    p.add_argument(
+        "--max-rps",
+        type=float,
+        default=None,
+        help="cap outbound requests per second (proactive rate pacing)",
+    )
+    p.add_argument(
+        "--color",
+        choices=["auto", "always", "never"],
+        default="auto",
+        help="colorize tier glyphs (default auto; honors NO_COLOR)",
+    )
+    p.add_argument(
+        "--cross-check",
+        choices=["none", "pubmed"],
+        default="none",
+        help="second source to corroborate findings against (default none)",
+    )
+    p.add_argument(
+        "--ncbi-email",
+        default=os.environ.get("NCBI_EMAIL"),
+        help="contact email for NCBI E-utilities (or set NCBI_EMAIL)",
+    )
+    p.add_argument(
+        "--ncbi-api-key",
+        default=os.environ.get("NCBI_API_KEY"),
+        help="NCBI API key for higher rate limits (or set NCBI_API_KEY)",
+    )
+    args = p.parse_args(argv)
+    if args.max_rps is not None and args.max_rps <= 0:
+        p.error("--max-rps must be > 0")
+    return args
+
+
+def _want_color(mode: str) -> bool:
+    """Resolve --color {auto,always,never} against NO_COLOR + TTY state.
+
+    NO_COLOR (presence, any value) disables color even when ``always`` is set,
+    per https://no-color.org/.
+    """
+    if "NO_COLOR" in os.environ:
+        return False
+    if mode == "always":
+        return True
+    if mode == "never":
+        return False
+    return sys.stdout.isatty() and os.environ.get("TERM") != "dumb"
+
+
+def main(argv=None) -> int:
+    args = _parse_args(argv if argv is not None else sys.argv[1:])
+    if args.file == "-":
+        text = sys.stdin.read()
+        if not text.strip():
+            print("ghostcite: no input on stdin", file=sys.stderr)
+            return 2
+    else:
+        try:
+            with open(args.file, encoding="utf-8") as fh:
+                text = fh.read()
+        except OSError as e:
+            print(f"ghostcite: cannot read {args.file}: {e}", file=sys.stderr)
+            return 2
+
+    try:
+        citations = parse(text, fmt=args.format)
+    except ValueError as e:
+        print(f"ghostcite: {e}", file=sys.stderr)
+        return 2
+
+    color = False if args.json else _want_color(args.color)
+    with_doi = sum(1 for c in citations if c.doi)
+    if args.dry_run:
+        print(
+            f"ghostcite: would check {len(citations)} entries "
+            f"({with_doi} via DOI, {len(citations) - with_doi} via search)."
+        )
+        return 0
+
+    findings: list[Finding] = []
+    use_pubmed = args.cross_check == "pubmed"
+    try:
+        with ExitStack() as stack:
+            client = stack.enter_context(CrossRefClient(max_rps=args.max_rps))
+            pmclient = None
+            if use_pubmed:
+                pmclient = stack.enter_context(
+                    PubMedClient(
+                        max_rps=args.max_rps,
+                        email=args.ncbi_email,
+                        api_key=args.ncbi_api_key,
+                    )
+                )
+            for c in citations:
+                if c.doi:
+                    rec = client.lookup_by_doi(c.doi)
+                else:
+                    rec = client.search_bibliographic(
+                        c.claimed_first_author, c.claimed_year, c.claimed_title
+                    )
+                cite_findings = evaluate(c, rec)
+                if pmclient is not None:
+                    if c.doi:
+                        pm = pmclient.lookup_by_doi(c.doi)
+                    else:
+                        pm = pmclient.lookup_by_doi_meta(
+                            c.claimed_first_author, c.claimed_year, c.claimed_title
+                        )
+                    cross_check_pubmed(c, rec, cite_findings, pm)
+                findings.extend(cite_findings)
+    except Exception as e:  # fail-loud: surface, keep partial findings
+        print(f"ghostcite: cross-check error: {e}", file=sys.stderr)
+        out = (
+            render_json(findings, len(citations), with_doi)
+            if args.json
+            else render_text(findings, len(citations), with_doi, color=color)
+        )
+        print(out)
+        return 2
+
+    out = (
+        render_json(findings, len(citations), with_doi)
+        if args.json
+        else render_text(findings, len(citations), with_doi, color=color)
+    )
+    print(out)
+
+    if args.fail_on.strip().lower() == "none":
+        return 0
+    fail_tiers = {
+        _TIER_BY_NAME[n.strip()] for n in args.fail_on.split(",") if n.strip() in _TIER_BY_NAME
+    }
+    return 1 if any(f.tier in fail_tiers for f in findings) else 0

ghostcite/compare.py

@@ -0,0 +1,321 @@
+from __future__ import annotations
+
+import unicodedata
+
+from ghostcite.models import CanonicalRecord, Citation, Finding, PubMedRecord, Tier
+
+
+def normalize_surname(name: str | None) -> str:
+    """Fold to a comparable key: strip diacritics, lowercase, keep only letters."""
+    if not name:
+        return ""
+    decomposed = unicodedata.normalize("NFKD", name)
+    ascii_only = "".join(c for c in decomposed if not unicodedata.combining(c))
+    return "".join(c for c in ascii_only.lower() if c.isalpha())
+
+
+def _is_initials(token: str) -> bool:
+    """True if a token is an initials cluster, e.g. "J", "J.", "JA", "J.A.".
+    Single letter, or all-uppercase letters after stripping dots — a real
+    surname ("Li", "Berg") is mixed-case, not all-caps."""
+    bare = token.replace(".", "")
+    if not bare or not bare.isalpha():
+        return False
+    return len(bare) == 1 or bare.isupper()
+
+
+def _surname_tokens(name: str) -> list[str]:
+    # A lone token is always the surname, never an initials cluster — keep it
+    # even if it looks like initials (e.g. an all-caps "LI"). Only strip
+    # initials-looking tokens when there is at least one other token to fall
+    # back on ("Smith J" → ["Smith"], "J Smith" → ["Smith"]).
+    tokens = name.split()
+    if len(tokens) <= 1:
+        return tokens
+    return [t for t in tokens if not _is_initials(t)]
+
+
+def _surname_key(name: str | None) -> str:
+    """Surname comparison key: drop standalone initial tokens
+    (e.g. "Smith J", "J Smith", "Smith JA") before normalizing, so a claimed
+    author carrying initials still matches a bare CrossRef family name."""
+    if not name:
+        return ""
+    return normalize_surname(" ".join(_surname_tokens(name)))
+
+
+def _surname_raw(name: str | None) -> str:
+    """Initial-stripped surname, diacritics PRESERVED, lowercased, despaced.
+    For the CLAIMED author: drops initials ("Smith J" → "smith") so an
+    initials-only difference doesn't read as a diacritic difference once
+    `_surname_key` has matched the keys."""
+    if not name:
+        return ""
+    return "".join(_surname_tokens(name)).lower()
+
+
+def _canonical_raw(name: str | None) -> str:
+    """Like `_surname_raw` but for a CANONICAL family name (CrossRef/PubMed):
+    diacritics preserved, lowercased, despaced, NO initials-stripping — an
+    all-caps family ("BURGER") is a real surname, not an initials cluster."""
+    if not name:
+        return ""
+    return "".join(name.split()).lower()
+
+
+def _title_tokens(title: str) -> set[str]:
+    decomposed = unicodedata.normalize("NFKD", title)
+    ascii_only = "".join(c for c in decomposed if not unicodedata.combining(c)).lower()
+    words = "".join(c if c.isalnum() else " " for c in ascii_only).split()
+    stop = {
+        "the",
+        "a",
+        "an",
+        "of",
+        "and",
+        "in",
+        "for",
+        "on",
+        "to",
+        "with",
+        "by",
+        "reveals",
+    }
+    return {w for w in words if len(w) > 2 and w not in stop}
+
+
+def title_similar(a: str | None, b: str | None, threshold: float = 0.4) -> bool:
+    """Jaccard token overlap >= threshold. Used to tell wrong-author from wrong-DOI."""
+    if not a or not b:
+        return False
+    ta, tb = _title_tokens(a), _title_tokens(b)
+    if not ta or not tb:
+        return False
+    inter = len(ta & tb)
+    union = len(ta | tb)
+    return (inter / union) >= threshold
+
+
+def evaluate(citation: Citation, canonical: CanonicalRecord | None) -> list[Finding]:
+    """Compare a claimed citation against the canonical record. Empty list = OK."""
+    if canonical is None:
+        return [Finding(citation, Tier.UNRESOLVABLE, None, "DOI not found / unresolvable")]
+
+    findings: list[Finding] = []
+
+    # Retraction is orthogonal — fires regardless of author/year.
+    if canonical.retracted:
+        findings.append(Finding(citation, Tier.RETRACTION, canonical, "RETRACTED per CrossRef"))
+    elif canonical.eoc:
+        findings.append(
+            Finding(
+                citation,
+                Tier.RETRACTION,
+                canonical,
+                "Expression of concern per CrossRef",
+            )
+        )
+
+    # Author/year only when the input actually claimed an author (not DOI-list mode).
+    if citation.claimed_first_author:
+        findings.extend(_author_year(citation, canonical))
+
+    return findings
+
+
+def _author_year(citation: Citation, canonical: CanonicalRecord) -> list[Finding]:
+    families_raw = canonical.authors or []
+    if not families_raw:
+        # DOI resolved but CrossRef has no author array (some preprints,
+        # datasets, protocols). Author can't be verified → warn, don't fail CI.
+        return [
+            Finding(
+                citation,
+                Tier.UNRESOLVABLE,
+                canonical,
+                "CrossRef record has no author data — author not verifiable",
+            )
+        ]
+    if not citation.claimed_first_author:
+        # No claimed author to verify against (caller already guards this, but
+        # keep the precondition explicit so the type is narrowed).
+        return []
+    claimed_raw = citation.claimed_first_author.strip()
+    # Asymmetric normalization: the claimed author is free text that may carry
+    # initials ("Smith JA") → initials-strip it. A CrossRef `family` field is
+    # ALWAYS a pure surname (even when ALL CAPS) → diacritic-fold only, never
+    # initials-strip (else "TURING" → "" and a correct claim is flagged Tier A).
+    claimed = _surname_key(claimed_raw)
+    first_raw = families_raw[0] if families_raw else ""
+    first = normalize_surname(first_raw)
+    all_norm = [normalize_surname(a) for a in families_raw]
+
+    conf = " (low-confidence match)" if canonical.low_confidence else ""
+
+    if claimed == first:
+        # First-author matches after normalization.
+        if _surname_raw(claimed_raw) != _canonical_raw(first_raw):
+            return [
+                Finding(
+                    citation,
+                    Tier.COSMETIC,
+                    canonical,
+                    f'diacritic/spelling: CrossRef has "{first_raw}"{conf}',
+                )
+            ]
+        # True match → check year.
+        if citation.claimed_year and canonical.year and citation.claimed_year != canonical.year:
+            return [
+                Finding(
+                    citation,
+                    Tier.YEAR,
+                    canonical,
+                    f"CrossRef year is {canonical.year}{conf}",
+                )
+            ]
+        return []
+
+    # First-author mismatch.
+    if claimed in all_norm:
+        idx = all_norm.index(claimed)
+        return [
+            Finding(
+                citation,
+                Tier.AUTHOR,
+                canonical,
+                f"claimed first author is actually author #{idx + 1}; "
+                f"CrossRef first author is {first_raw}{conf}",
+            )
+        ]
+
+    # Not in author list at all — wrong author, or wrong DOI entirely.
+    if not title_similar(citation.claimed_title, canonical.title):
+        return [
+            Finding(
+                citation,
+                Tier.AUTHOR,
+                canonical,
+                f"DOI resolves to {first_raw} ({canonical.year}) — possibly wrong DOI{conf}",
+            )
+        ]
+    return [
+        Finding(
+            citation,
+            Tier.AUTHOR,
+            canonical,
+            f"CrossRef first author is {first_raw}{conf}",
+        )
+    ]
+
+
+def cross_check_pubmed(
+    citation: Citation,
+    canonical: CanonicalRecord | None,
+    findings: list[Finding],
+    pubmed: PubMedRecord | None,
+) -> None:
+    """Reconcile CrossRef-derived ``findings`` against a PubMed record in place.
+
+    Annotates existing findings with a ``cross_check`` note and/or appends NEW
+    findings (author/year disagreements PubMed sees that CrossRef didn't,
+    retractions PubMed flags). Pure aside from mutating ``findings``; no I/O.
+
+    Semantics:
+      * CrossRef had no record (UNRESOLVABLE) + PubMed resolves → annotate the
+        unresolvable finding: evaluable via PubMed (low confidence).
+      * An AUTHOR/YEAR finding that PubMed *corroborates* (PubMed agrees with
+        CrossRef, not the claim) → note "corroborated by PubMed", tier kept.
+      * An AUTHOR/YEAR finding that PubMed *contradicts* (PubMed agrees with the
+        cited claim) → conflict note, tier kept (manual review).
+      * No CrossRef author/year finding but PubMed disagrees with the claim →
+        NEW AUTHOR/YEAR finding "raised by PubMed".
+      * Retraction is OR-combined: PubMed-flagged retraction/EoC appends a
+        RETRACTION finding if CrossRef didn't already.
+    """
+    if pubmed is None:
+        return
+
+    claimed_author = _surname_key(citation.claimed_first_author)
+    # PubMed's surname field is a pure surname (possibly ALL CAPS) — fold only,
+    # don't initials-strip it (see _author_year for the same asymmetry).
+    pm_author = normalize_surname(pubmed.first_author_surname)
+    claimed_year = citation.claimed_year
+
+    # 1. Retraction / expression of concern — OR-combine with CrossRef.
+    if (pubmed.retracted or pubmed.eoc) and not any(f.tier is Tier.RETRACTION for f in findings):
+        label = "RETRACTED" if pubmed.retracted else "Expression of concern"
+        findings.append(
+            Finding(
+                citation,
+                Tier.RETRACTION,
+                canonical,
+                f"{label} per PubMed",
+                cross_check="raised by PubMed",
+            )
+        )
+
+    # 2. CrossRef had no usable record → PubMed makes it evaluable.
+    if canonical is None:
+        for f in findings:
+            if f.tier is Tier.UNRESOLVABLE:
+                f.cross_check = "resolved via PubMed; CrossRef had no record (low confidence)"
+        return
+
+    author_year_findings = [f for f in findings if f.tier in (Tier.AUTHOR, Tier.YEAR)]
+
+    # 3. Existing AUTHOR/YEAR findings — does PubMed back CrossRef or the claim?
+    no_data_note = "PubMed record had no author/year data — not verifiable"
+    for f in author_year_findings:
+        if f.tier is Tier.AUTHOR and pm_author and claimed_author:
+            if pm_author == claimed_author:
+                f.cross_check = (
+                    "⚠ PubMed agrees with the cited author — CrossRef and PubMed "
+                    "conflict; verify manually"
+                )
+            else:
+                f.cross_check = "corroborated by PubMed"
+        elif f.tier is Tier.YEAR and pubmed.year and claimed_year:
+            if pubmed.year == claimed_year:
+                f.cross_check = (
+                    "⚠ PubMed agrees with the cited year — CrossRef and PubMed "
+                    "conflict; verify manually"
+                )
+            else:
+                f.cross_check = "corroborated by PubMed"
+        else:
+            # PubMed was consulted but lacks the usable author/year needed to
+            # corroborate this finding — say so instead of leaving it silent.
+            f.cross_check = no_data_note
+
+    # 4. CrossRef raised no author finding but PubMed disagrees with the claim.
+    has_author_finding = any(f.tier is Tier.AUTHOR for f in findings)
+    if not has_author_finding and claimed_author and pm_author and pm_author != claimed_author:
+        findings.append(
+            Finding(
+                citation,
+                Tier.AUTHOR,
+                canonical,
+                f"PubMed first author is {pubmed.first_author_surname}",
+                cross_check="raised by PubMed",
+            )
+        )
+
+    # 5. CrossRef raised no year finding but PubMed disagrees with the claimed year.
+    has_year_finding = any(f.tier is Tier.YEAR for f in findings)
+    if (
+        not has_year_finding
+        and claimed_year
+        and pubmed.year
+        and pubmed.year != claimed_year
+        # Only when the author lines up — otherwise the author finding carries it.
+        and (not pm_author or not claimed_author or pm_author == claimed_author)
+    ):
+        findings.append(
+            Finding(
+                citation,
+                Tier.YEAR,
+                canonical,
+                f"PubMed year is {pubmed.year}",
+                cross_check="raised by PubMed",
+            )
+        )

ghostcite/crossref.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+import time
+
+import httpx
+
+from ghostcite import __version__
+from ghostcite._pace import _Pacer
+from ghostcite.models import CanonicalRecord
+
+_BASE = "https://api.crossref.org"
+_UA = f"ghostcite/{__version__} (https://github.com/musharna/ghostcite)"
+
+
+def _retraction_flags(message: dict) -> tuple[bool, bool]:
+    """Return (retracted, expression_of_concern) from a CrossRef work message.
+
+    CrossRef exposes retraction/EoC three ways, and the live schema (verified
+    against the Wakefield 1998 Lancet DOI, 2026-06-08) uses ``updated-by``:
+      - ``updated-by``: items ON THE RETRACTED WORK pointing forward to the
+        notice; each has ``type`` ∈ {retraction, expression_of_concern, ...}.
+        This is the canonical "this paper was retracted" signal.
+      - ``update-to``: the inverse — items on the NOTICE pointing back to what
+        it updates (kept for completeness; often null on the retracted work).
+      - ``relation``: keys like ``is-retracted-by`` (rarely populated).
+    """
+    retracted = eoc = False
+    relation = message.get("relation") or {}
+    for key in relation:
+        k = key.lower()
+        if "retract" in k:
+            retracted = True
+        if "concern" in k:
+            eoc = True
+    for field in ("updated-by", "update-to"):
+        for upd in message.get(field) or []:
+            t = str(upd.get("type", "")).lower()
+            if "retract" in t:
+                retracted = True
+            if "concern" in t:
+                eoc = True
+    return retracted, eoc
+
+
+def _year(message: dict) -> int | None:
+    for key in ("published", "published-print", "published-online", "issued"):
+        parts = (message.get(key) or {}).get("date-parts") or []
+        if parts and parts[0] and parts[0][0]:
+            return int(parts[0][0])
+    return None
+
+
+def _record_from_message(message: dict, low_confidence: bool = False) -> CanonicalRecord:
+    retracted, eoc = _retraction_flags(message)
+    authors = [a.get("family", "").strip() for a in message.get("author") or [] if a.get("family")]
+    title = (message.get("title") or [None])[0]
+    journal = (message.get("container-title") or [None])[0]
+    return CanonicalRecord(
+        doi=(message.get("DOI") or "").lower() or None,
+        authors=authors,
+        year=_year(message),
+        title=title,
+        journal=journal,
+        retracted=retracted,
+        eoc=eoc,
+        low_confidence=low_confidence,
+    )
+
+
+class CrossRefClient:
+    def __init__(self, timeout: float = 20.0, max_rps: float | None = None):
+        self._client = httpx.Client(
+            timeout=timeout,
+            headers={"User-Agent": _UA},
+            follow_redirects=True,
+        )
+        self._pacer = _Pacer(max_rps)
+
+    def __enter__(self) -> CrossRefClient:
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self._client.close()
+
+    def _get(self, url: str, **kw) -> httpx.Response:
+        """GET with a single retry on a transient 429/503, honoring Retry-After
+        (capped at 60s). The caller handles status codes (e.g. 404)."""
+        self._pacer.wait()
+        r = self._client.get(url, **kw)
+        self._pacer.update_from_headers(r.headers)
+        if r.status_code in (429, 503):
+            delay = min(int(r.headers.get("Retry-After", "5")), 60)
+            time.sleep(delay)
+            self._pacer.wait()
+            r = self._client.get(url, **kw)
+            self._pacer.update_from_headers(r.headers)
+        return r
+
+    def lookup_by_doi(self, doi: str) -> CanonicalRecord | None:
+        r = self._get(f"{_BASE}/works/{doi}")
+        if r.status_code == 404:
+            return None
+        r.raise_for_status()
+        return _record_from_message(r.json()["message"])
+
+    def search_bibliographic(
+        self, author: str | None, year: int | None, title: str | None
+    ) -> CanonicalRecord | None:
+        query = " ".join(str(x) for x in (author, year, title) if x).strip()
+        if not query:
+            return None
+        r = self._get(f"{_BASE}/works", params={"query.bibliographic": query, "rows": 1})
+        r.raise_for_status()
+        items = r.json().get("message", {}).get("items") or []
+        if not items:
+            return None
+        return _record_from_message(items[0], low_confidence=True)