sciwrite-lint 0.2.0__py3-none-any.whl

sciwrite_lint/checks/reference_accuracy.py

@@ -0,0 +1,277 @@
+"""Check: reference-accuracy — manuscript metadata vs canonical API records.
+
+Runs after reference-exists confirms a reference is real. Compares title,
+authors, year, and venue in the bibitem against stored canonical data from
+CrossRef / OpenAlex / Semantic Scholar.
+
+Uses metadata already persisted in references/metadata/{key}.json — no new
+API calls.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from sciwrite_lint.checks.registry import check
+from sciwrite_lint.config import LintConfig
+from sciwrite_lint.models import CitationMetadata, Finding
+
+
+# ---------------------------------------------------------------------------
+# Field-level comparison helpers
+# ---------------------------------------------------------------------------
+
+
+def _check_title(
+    key: str, bibitem: dict, canonical: dict, api_source: str = ""
+) -> Finding | None:
+    """Title: fuzzy match, ERROR if below threshold (possible fabricated title)."""
+    from sciwrite_lint.references.matching import TITLE_THRESHOLD, title_similarity
+
+    tex_title = bibitem.get("title", "")
+    api_title = canonical.get("title", "")
+    if not tex_title or not api_title:
+        return None
+
+    sim = title_similarity(tex_title, api_title)
+    if sim >= TITLE_THRESHOLD:
+        return None
+
+    return Finding(
+        level="error",
+        rule_id="reference-accuracy",
+        message=(
+            f"{key}: Title mismatch (similarity={sim:.2f}): "
+            f"tex='{tex_title[:80]}', canonical='{api_title[:80]}'"
+        ),
+        context=f"Source: {api_source}" if api_source else "",
+    )
+
+
+def _check_authors(
+    key: str, bibitem: dict, canonical: dict, api_source: str = ""
+) -> Finding | None:
+    """Authors: set comparison on normalized last names."""
+    from sciwrite_lint.references.matching import AUTHOR_THRESHOLD, author_similarity
+
+    tex_authors = bibitem.get("authors", [])
+    api_authors = canonical.get("authors", [])
+    if not tex_authors or not api_authors:
+        return None
+
+    # Handle case where tex_authors is a single joined string
+    if len(tex_authors) == 1 and "," in tex_authors[0]:
+        tex_authors = [a.strip() for a in tex_authors[0].split(",") if a.strip()]
+
+    sim = author_similarity(tex_authors, api_authors)
+    if sim >= AUTHOR_THRESHOLD:
+        return None
+
+    tex_str = ", ".join(str(a) for a in tex_authors[:3])
+    api_str = ", ".join(str(a) for a in api_authors[:3])
+    suffix = "..." if len(api_authors) > 3 else ""
+    return Finding(
+        level="warning",
+        rule_id="reference-accuracy",
+        message=(
+            f"{key}: Author mismatch (similarity={sim:.2f}): "
+            f"tex='{tex_str}', canonical='{api_str}{suffix}'"
+        ),
+        context=f"Source: {api_source}" if api_source else "",
+    )
+
+
+def _check_year(
+    key: str, bibitem: dict, canonical: dict, api_source: str = ""
+) -> Finding | None:
+    """Year: exact match, flag if off by >1 year."""
+    tex_year = bibitem.get("year", "")
+    api_year = canonical.get("year")
+    if not tex_year or not api_year:
+        return None
+
+    try:
+        ty = int(tex_year)
+        ay = int(api_year)
+    except (ValueError, TypeError):
+        return None
+
+    diff = abs(ty - ay)
+    if diff <= 1:
+        return None
+
+    return Finding(
+        level="warning",
+        rule_id="reference-accuracy",
+        message=f"{key}: Year mismatch: tex={ty}, canonical={ay} (off by {diff})",
+        context=f"Source: {api_source}" if api_source else "",
+    )
+
+
+def _check_venue(
+    key: str, bibitem: dict, canonical: dict, api_source: str = ""
+) -> tuple[Finding | None, tuple[str, str] | None]:
+    """Venue: fuzzy match on normalized name.
+
+    Returns (finding_or_none, venue_pair_needing_llm_confirmation_or_none).
+    When fuzzy score is below threshold, returns the finding AND the venue
+    pair so the caller can optionally confirm via vLLM before emitting.
+    """
+    from sciwrite_lint.references.matching import VENUE_THRESHOLD, venue_similarity
+
+    tex_venue = bibitem.get("venue", "")
+    api_venue = canonical.get("venue", "")
+    if not tex_venue or not api_venue:
+        return None, None
+
+    sim = venue_similarity(tex_venue, api_venue)
+    if sim >= VENUE_THRESHOLD:
+        return None, None
+
+    finding = Finding(
+        level="warning",
+        rule_id="reference-accuracy",
+        message=(
+            f"{key}: Venue mismatch (similarity={sim:.2f}): "
+            f"tex='{tex_venue[:60]}', canonical='{api_venue[:60]}'"
+        ),
+        context=f"Source: {api_source}" if api_source else "",
+    )
+    return finding, (tex_venue, api_venue)
+
+
+# ---------------------------------------------------------------------------
+# Main check: run on all stored metadata
+# ---------------------------------------------------------------------------
+
+
+def check_reference_accuracy_from_metadata(
+    all_metadata: dict[str, CitationMetadata],
+) -> list[Finding]:
+    """Compare bibitem vs canonical for all verified references.
+
+    Only checks references where api_match is "verified" or "mismatch"
+    (i.e., reference-exists already passed).
+
+    Synchronous version — venue mismatches use fuzzy matching only.
+    For vLLM-confirmed venue matching, use the async variant.
+    """
+    findings: list[Finding] = []
+
+    for key, meta in sorted(all_metadata.items()):
+        # Skip unverified, not-found, and web resources
+        if meta.api_match not in ("verified", "mismatch"):
+            continue
+
+        # Skip manually overridden references
+        if meta.manual_override and meta.manual_override.get("skip_accuracy"):
+            continue
+
+        canonical = meta.canonical
+        bibitem = meta.bibitem
+        if not canonical or not bibitem:
+            continue
+
+        src = meta.api_source
+        for checker in (_check_title, _check_authors, _check_year):
+            finding = checker(key, bibitem, canonical, api_source=src)
+            if finding:
+                findings.append(finding)
+
+        venue_finding, _venue_pair = _check_venue(
+            key, bibitem, canonical, api_source=src
+        )
+        if venue_finding:
+            findings.append(venue_finding)
+
+    return findings
+
+
+async def check_reference_accuracy_from_metadata_async(
+    all_metadata: dict[str, CitationMetadata],
+    config: Any | None = None,
+) -> list[Finding]:
+    """Compare bibitem vs canonical, with vLLM venue confirmation.
+
+    Same as sync version, but venue mismatches below the fuzzy threshold
+    are sent to vLLM for confirmation before flagging. If vLLM says the
+    venues match, the finding is suppressed.
+    """
+    from sciwrite_lint.references.matching import venue_match_llm
+
+    findings: list[Finding] = []
+    venue_candidates: list[tuple[Finding, str, str]] = []
+
+    for key, meta in sorted(all_metadata.items()):
+        if meta.api_match not in ("verified", "mismatch"):
+            continue
+        if meta.manual_override and meta.manual_override.get("skip_accuracy"):
+            continue
+
+        canonical = meta.canonical
+        bibitem = meta.bibitem
+        if not canonical or not bibitem:
+            continue
+
+        src = meta.api_source
+        for checker in (_check_title, _check_authors, _check_year):
+            finding = checker(key, bibitem, canonical, api_source=src)
+            if finding:
+                findings.append(finding)
+
+        venue_finding, venue_pair = _check_venue(
+            key, bibitem, canonical, api_source=src
+        )
+        if venue_finding and venue_pair:
+            venue_candidates.append((venue_finding, venue_pair[0], venue_pair[1]))
+
+    # Confirm venue mismatches via vLLM (concurrent, bounded)
+    if venue_candidates:
+        import asyncio
+
+        sem = asyncio.Semaphore(50)
+
+        async def _confirm(finding: Finding, tex_v: str, api_v: str) -> Finding | None:
+            async with sem:
+                same = await venue_match_llm(tex_v, api_v, config=config)
+            if same is True:
+                return None  # vLLM says same venue — suppress
+            return finding
+
+        results = await asyncio.gather(
+            *[_confirm(f, tv, av) for f, tv, av in venue_candidates]
+        )
+        for r in results:
+            if r is not None:
+                findings.append(r)
+
+    return findings
+
+
+@check(
+    id="reference-accuracy",
+    category="reference-db",
+    severity="warning",
+    description="Manuscript metadata vs canonical API records (title, authors, year, venue).",
+)
+def check_reference_accuracy(tex_path: Path, config: LintConfig) -> list[Finding]:
+    """Load stored metadata and compare bibitem fields against canonical records.
+
+    No API calls — uses references/metadata/{key}.json persisted by verify.
+    """
+    from sciwrite_lint.references.workspace_db import get_db, query_refs_by_match
+
+    refs_dir = config.effective_references_dir()
+
+    if not refs_dir.exists():
+        return []
+
+    # Only load refs that have been verified (the only ones with accuracy data)
+    with get_db(refs_dir) as conn:
+        candidates = query_refs_by_match(conn, "verified")
+        candidates.update(query_refs_by_match(conn, "mismatch"))
+    if not candidates:
+        return []
+
+    return check_reference_accuracy_from_metadata(candidates)

sciwrite_lint/checks/reference_exists.py

@@ -0,0 +1,119 @@
+"""Check: reference-exists — citation found in academic APIs or URL alive.
+
+Runs during `sciwrite-lint verify`. Checks stored metadata to determine
+whether each reference was found in CrossRef, OpenAlex, or Semantic Scholar
+(academic) or has a live URL (web resources).
+
+Uses metadata already persisted in references/metadata/{key}.json — the
+actual API calls happen in the verify pipeline stage, not here.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from sciwrite_lint.checks.registry import check
+from sciwrite_lint.config import LintConfig
+from sciwrite_lint.models import CitationMetadata, Finding
+
+
+def _searched_ids_context(meta: CitationMetadata) -> str:
+    """Build a human-readable summary of identifiers available for lookup."""
+    bib = meta.bibitem or {}
+    ids: list[str] = []
+    if bib.get("doi"):
+        ids.append(f"doi={bib['doi']}")
+    if bib.get("arxiv_id"):
+        ids.append(f"arxiv={bib['arxiv_id']}")
+    if bib.get("pmid"):
+        ids.append(f"pmid={bib['pmid']}")
+    if bib.get("isbn"):
+        ids.append(f"isbn={bib['isbn']}")
+    if bib.get("lccn"):
+        ids.append(f"lccn={bib['lccn']}")
+    if bib.get("url"):
+        ids.append(f"url={bib['url']}")
+    return f"Searched with: {', '.join(ids)}" if ids else "No identifiers in bib entry"
+
+
+def check_reference_exists_from_metadata(
+    all_metadata: dict[str, CitationMetadata],
+) -> list[Finding]:
+    """Check that all references were found in APIs.
+
+    Flags:
+    - not_found: ERROR — reference not in any API
+    - dead URL: ERROR — web resource URL is dead
+    - content extraction failed: WARNING — URL alive but content not extracted
+    """
+    findings: list[Finding] = []
+
+    for key, meta in sorted(all_metadata.items()):
+        # Skip manually overridden references
+        if meta.manual_override and meta.manual_override.get("skip_exists"):
+            continue
+
+        if meta.api_match == "not_found":
+            findings.append(
+                Finding(
+                    level="error",
+                    rule_id="reference-exists",
+                    message=f"{key}: Not found in CrossRef, OpenAlex, Semantic Scholar, Open Library, or Library of Congress",
+                    context=_searched_ids_context(meta),
+                )
+            )
+            continue
+
+        # Check web resource issues stored in the issues list
+        for issue in meta.issues:
+            issue_lower = issue.lower()
+            if "dead url" in issue_lower:
+                findings.append(
+                    Finding(
+                        level="error",
+                        rule_id="reference-exists",
+                        message=f"{key}: {issue}",
+                        context=f"Source: {meta.api_source}" if meta.api_source else "",
+                    )
+                )
+            elif "content extraction failed" in issue_lower:
+                findings.append(
+                    Finding(
+                        level="warning",
+                        rule_id="reference-exists",
+                        message=f"{key}: {issue}",
+                        context=f"Source: {meta.api_source}" if meta.api_source else "",
+                    )
+                )
+
+    return findings
+
+
+@check(
+    id="reference-exists",
+    category="reference-db",
+    severity="error",
+    description="Reference not found in CrossRef, OpenAlex, Semantic Scholar, Open Library, or Library of Congress.",
+)
+def check_reference_exists(tex_path: Path, config: LintConfig) -> list[Finding]:
+    """Load stored metadata and check existence status.
+
+    No API calls — uses references/metadata/{key}.json persisted by verify.
+    """
+    from sciwrite_lint.references.workspace_db import get_db, query_refs_by_match
+
+    refs_dir = config.effective_references_dir()
+
+    if not refs_dir.exists():
+        return []
+
+    # Only load refs that could produce findings (not_found, web_dead, or with issues)
+    with get_db(refs_dir) as conn:
+        candidates = query_refs_by_match(conn, "not_found")
+        candidates.update(query_refs_by_match(conn, "web_dead"))
+        # Web-verified refs may have content extraction issues
+        candidates.update(query_refs_by_match(conn, "web_verified"))
+    if not candidates:
+        return []
+
+    return check_reference_exists_from_metadata(candidates)

sciwrite_lint/checks/reference_unreliable.py

@@ -0,0 +1,244 @@
+"""Check: reference-unreliable — per-reference aggregate reliability.
+
+Aggregates signals from whatever checks have run (metadata, claim verdicts,
+chain verification) into a single per-reference reliability score. Fires a
+warning when convergent evidence says a reference is unreliable.
+
+Two entry points:
+- metadata_to_unreliable_findings: fast path (after verify), metadata only
+- claims_to_unreliable_findings: deep path (after verify-claims), all signals
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from sciwrite_lint.models import CheckMeta, CitationMetadata, Finding
+
+if TYPE_CHECKING:
+    from sciwrite_lint.scoring.chain import RefBibCheck
+
+# ---------------------------------------------------------------------------
+# Threshold
+# ---------------------------------------------------------------------------
+
+UNRELIABLE_THRESHOLD = 0.4
+
+
+# ---------------------------------------------------------------------------
+# Signal descriptions (for human-readable context)
+# ---------------------------------------------------------------------------
+
+
+def _describe_metadata(meta: CitationMetadata) -> list[str]:
+    """Build a list of human-readable signal descriptions from metadata."""
+    from sciwrite_lint.references.metadata import compute_tier
+
+    parts: list[str] = []
+    if meta.canonical.get("retracted"):
+        parts.append("retracted")
+        return parts  # retraction is definitive
+
+    tier = meta.access.get("tier") or compute_tier(meta)
+    if tier == "T3":
+        parts.append("not found in APIs")
+    if meta.api_match == "mismatch":
+        parts.append("API metadata mismatch")
+    if meta.mismatches:
+        parts.append(f"{len(meta.mismatches)} field mismatch(es)")
+    return parts
+
+
+def _describe_claims(claims: list[dict[str, Any]]) -> list[str]:
+    """Build signal descriptions from claim verification results."""
+    parts: list[str] = []
+    active = [c for c in claims if not c.get("dismissed")]
+    not_sup = sum(1 for c in active if c.get("verdict") == "NOT_SUPPORTED")
+    partial = sum(1 for c in active if c.get("verdict") == "PARTIALLY_SUPPORTS")
+    if not_sup:
+        parts.append(f"{not_sup} claim(s) not supported")
+    if partial:
+        parts.append(f"{partial} claim(s) partially supported")
+    return parts
+
+
+# ---------------------------------------------------------------------------
+# Scoring (reuses scilint_score constants)
+# ---------------------------------------------------------------------------
+
+
+def _score_from_metadata(meta: CitationMetadata) -> float:
+    """Compute reliability score from metadata signals only."""
+    from sciwrite_lint.scoring.scilint_score import _score_metadata
+
+    return _score_metadata(meta)
+
+
+def _score_from_claims(claims: list[dict[str, Any]]) -> float:
+    """Compute verification score from claim verdicts."""
+    from sciwrite_lint.scoring.scilint_score import VERDICT_SCORES
+
+    active = [c for c in claims if not c.get("dismissed")]
+    if not active:
+        return 1.0  # no claims to verify — no negative signal
+    total = sum(
+        VERDICT_SCORES.get(c.get("verdict", "CANNOT_DETERMINE"), 0.25) for c in active
+    )
+    return total / len(active)
+
+
+def _describe_bib_checks(bib_checks: list[RefBibCheck], key: str) -> list[str]:
+    """Build signal descriptions from standalone bibliography checks."""
+    parts: list[str] = []
+    for rb in bib_checks:
+        if rb.key == key and rb.total_entries > 0:
+            if rb.not_found > 0:
+                parts.append(
+                    f"{rb.not_found}/{rb.total_entries} bibliography entries "
+                    f"not found in APIs ({rb.hallucination_rate:.0%})"
+                )
+            if rb.retracted > 0:
+                parts.append(f"{rb.retracted} retracted entries in bibliography")
+            if rb.metadata_mismatches > 0:
+                parts.append(
+                    f"{rb.metadata_mismatches} bibliography metadata mismatches "
+                    f"({rb.mismatch_rate:.0%} of found)"
+                )
+    return parts
+
+
+def _score_from_bib_checks(bib_checks: list[RefBibCheck], key: str) -> float:
+    """Compute score from standalone bibliography checks."""
+    for rb in bib_checks:
+        if rb.key == key and rb.total_entries > 0:
+            existence_score = rb.found / rb.total_entries
+            mismatch_penalty = min(rb.metadata_mismatches * 0.05, 0.3)
+            retraction_penalty = min(rb.retracted * 0.15, 0.3)
+            return max(existence_score - mismatch_penalty - retraction_penalty, 0.0)
+    return 1.0
+
+
+# ---------------------------------------------------------------------------
+# Finding generators
+# ---------------------------------------------------------------------------
+
+
+def _make_finding(
+    key: str,
+    reliability: float,
+    signals: list[str],
+    tex_path: Path,
+    line: int | None = None,
+) -> Finding:
+    return Finding(
+        level="warning",
+        rule_id="reference-unreliable",
+        message=f"{key}: Reference has low reliability ({reliability:.2f})",
+        file=tex_path.name,
+        line=line,
+        context=", ".join(signals) if signals else "",
+    )
+
+
+def metadata_to_unreliable_findings(
+    metadata_map: dict[str, CitationMetadata],
+    tex_path: Path,
+    bib_checks: list[RefBibCheck] | None = None,
+) -> list[Finding]:
+    """Fast path: flag unreliable references from metadata signals only.
+
+    Call after ``verify`` when metadata is available but claim verification
+    has not run. Bibliography checks from the pipeline are included when
+    available.
+    """
+    findings: list[Finding] = []
+    for key, meta in metadata_map.items():
+        meta_score = _score_from_metadata(meta)
+        bib_score = _score_from_bib_checks(bib_checks, key) if bib_checks else 1.0
+        reliability = meta_score * bib_score
+        if reliability < UNRELIABLE_THRESHOLD:
+            signals = _describe_metadata(meta)
+            if bib_checks:
+                signals.extend(_describe_bib_checks(bib_checks, key))
+            findings.append(_make_finding(key, reliability, signals, tex_path))
+    return findings
+
+
+def claims_to_unreliable_findings(
+    claim_results: list[dict[str, Any]],
+    tex_path: Path,
+    metadata_map: dict[str, CitationMetadata] | None = None,
+    bib_checks: list[RefBibCheck] | None = None,
+) -> list[Finding]:
+    """Deep path: flag unreliable references using all available signals.
+
+    Call after ``verify-claims``. Combines metadata, claim verdicts, and
+    bibliography checks (existence + metadata + retraction from pipeline).
+    """
+    # Group claims by reference key
+    by_key: dict[str, list[dict[str, Any]]] = {}
+    for c in claim_results:
+        k = c.get("key", "")
+        if k:
+            by_key.setdefault(k, []).append(c)
+
+    findings: list[Finding] = []
+    seen_keys = set(by_key.keys())
+
+    for key, claims in by_key.items():
+        # Start with metadata if available
+        if metadata_map and key in metadata_map:
+            meta_score = _score_from_metadata(metadata_map[key])
+        else:
+            meta_score = 1.0  # no metadata → no negative signal
+
+        claim_score = _score_from_claims(claims)
+
+        # Bibliography checks (from pipeline Stage 4.6)
+        bib_score = _score_from_bib_checks(bib_checks, key) if bib_checks else 1.0
+
+        reliability = meta_score * claim_score * bib_score
+
+        if reliability < UNRELIABLE_THRESHOLD:
+            signals: list[str] = []
+            if metadata_map and key in metadata_map:
+                signals.extend(_describe_metadata(metadata_map[key]))
+            signals.extend(_describe_claims(claims))
+            if bib_checks:
+                signals.extend(_describe_bib_checks(bib_checks, key))
+
+            # Use line from first claim for location
+            line = claims[0].get("line") if claims else None
+            findings.append(_make_finding(key, reliability, signals, tex_path, line))
+
+    # Also check metadata-only keys not in claims (e.g., T3 refs with no T1 source)
+    if metadata_map:
+        for key, meta in metadata_map.items():
+            if key in seen_keys:
+                continue
+            meta_score = _score_from_metadata(meta)
+            bib_score = _score_from_bib_checks(bib_checks, key) if bib_checks else 1.0
+            reliability = meta_score * bib_score
+            if reliability < UNRELIABLE_THRESHOLD:
+                signals = _describe_metadata(meta)
+                if bib_checks:
+                    signals.extend(_describe_bib_checks(bib_checks, key))
+                findings.append(_make_finding(key, reliability, signals, tex_path))
+
+    return findings
+
+
+# ---------------------------------------------------------------------------
+# Registry
+# ---------------------------------------------------------------------------
+
+
+# Metadata for `sciwrite-lint checks` listing.
+# This check runs as a pipeline stage (not from the check registry).
+REFERENCE_UNRELIABLE_META = CheckMeta(
+    id="reference-unreliable",
+    severity="warning",
+    category="reference-db",
+    description="Reference has low aggregate reliability across multiple signals.",
+)