sciwrite-lint 0.2.0__py3-none-any.whl

sciwrite_lint/references/matching.py

@@ -0,0 +1,273 @@
+"""Fuzzy comparison for citation verification.
+
+Compares parsed bibitem data against API-returned canonical data using
+calibrated similarity thresholds inspired by bibtex-updater's algorithms.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from anyascii import anyascii
+
+from sciwrite_lint.models import Citation
+from sciwrite_lint.schemas import VenueMatch, vllm_schema
+
+# Thresholds
+TITLE_THRESHOLD = 0.80  # rapidfuzz ratio
+AUTHOR_THRESHOLD = 0.60  # combined Jaccard + sequence
+YEAR_TOLERANCE = 1  # ±1 year
+
+# Try rapidfuzz first; use basic implementation if unavailable
+try:
+    from rapidfuzz import fuzz as _rf_fuzz
+
+    def _fuzzy_ratio(a: str, b: str) -> float:
+        return _rf_fuzz.ratio(a, b) / 100.0
+except ImportError:
+
+    def _fuzzy_ratio(a: str, b: str) -> float:
+        """Basic sequence similarity when rapidfuzz not installed."""
+        if not a or not b:
+            return 0.0
+        a, b = a.lower(), b.lower()
+        if a == b:
+            return 1.0
+        # Simple word overlap ratio
+        wa, wb = set(a.split()), set(b.split())
+        if not wa or not wb:
+            return 0.0
+        overlap = len(wa & wb)
+        return (2 * overlap) / (len(wa) + len(wb))
+
+
+def _normalize(text: str) -> str:
+    """Normalize text for comparison: transliterate, lowercase, strip LaTeX, collapse whitespace."""
+    text = anyascii(text)  # Cyrillic→Latin, Unicode hyphens→ASCII, etc.
+    text = text.lower()
+    text = re.sub(r"\\[a-zA-Z]+\{([^}]*)\}", r"\1", text)  # \cmd{arg} -> arg
+    text = re.sub(r"[{}~\\]", " ", text)
+    text = re.sub(r"['\"`]", "", text)
+    text = re.sub(r"\s+", " ", text).strip()
+    return text
+
+
+def title_similarity(tex_title: str, api_title: str) -> float:
+    """Fuzzy title similarity score (0.0 to 1.0).
+
+    Handles subtitle variations: books often have subtitles in one source
+    but not the other (e.g. "Mind in Society" vs "Mind in Society: The
+    Development of Higher Psychological Processes"). Tries all combinations
+    of full title and main title (before colon) and returns the best score.
+    """
+    a = _normalize(tex_title)
+    b = _normalize(api_title)
+    if not a or not b:
+        return 0.0
+    a_variants = [a]
+    if ":" in a:
+        a_variants.append(a.split(":")[0].strip())
+    b_variants = [b]
+    if ":" in b:
+        b_variants.append(b.split(":")[0].strip())
+    return max(_fuzzy_ratio(av, bv) for av in a_variants for bv in b_variants)
+
+
+def author_similarity(tex_authors: list[str], api_authors: list[str]) -> float:
+    """Pairwise fuzzy author similarity with name variant generation.
+
+    Returns 0.0 to 1.0. Handles name order, initials, transliteration,
+    middle name dropping, and East Asian name conventions.
+
+    Delegates to ``_author_overlap`` in api.py (single implementation
+    shared with the candidate selection engine).
+    """
+    from sciwrite_lint.api import _author_overlap
+
+    return _author_overlap(tex_authors, api_authors)
+
+
+def _normalize_venue(venue: str) -> str:
+    """Normalize venue name: lowercase, strip boilerplate prefixes/suffixes."""
+    v = _normalize(venue)
+    # Strip common prefixes
+    for prefix in ("proceedings of the ", "proceedings of ", "proc. ", "in "):
+        if v.startswith(prefix):
+            v = v[len(prefix) :]
+    # Strip trailing year patterns like ", 2020" or " 2020"
+    v = re.sub(r"[,\s]+\d{4}$", "", v)
+    return v
+
+
+try:
+    from rapidfuzz import fuzz as _rf_fuzz_venue
+
+    def _fuzzy_partial_ratio(a: str, b: str) -> float:
+        return _rf_fuzz_venue.partial_ratio(a, b) / 100.0
+
+except ImportError:
+
+    def _fuzzy_partial_ratio(a: str, b: str) -> float:
+        """Fallback: use standard ratio when rapidfuzz not installed."""
+        return _fuzzy_ratio(a, b)
+
+
+def venue_similarity(tex_venue: str, api_venue: str) -> float:
+    """Fuzzy venue similarity score (0.0 to 1.0).
+
+    Uses partial_ratio for better handling of abbreviations and contained
+    strings (e.g. "NIPS" inside "NeurIPS", "Softw." vs "Software").
+    No hardcoded alias table — relies on fuzzy matching only.
+    Returns 1.0 if either venue is empty (can't compare).
+    """
+    a = _normalize_venue(tex_venue)
+    b = _normalize_venue(api_venue)
+    if not a or not b:
+        return 1.0  # can't compare, not a mismatch
+    if a == b:
+        return 1.0
+    return _fuzzy_partial_ratio(a, b)
+
+
+VENUE_THRESHOLD = (
+    0.65  # venue partial_ratio (slightly above 0.60 to avoid false positives)
+)
+
+
+_VENUE_CONFIRM_SCHEMA = vllm_schema(VenueMatch)
+
+
+async def venue_match_llm(
+    tex_venue: str,
+    api_venue: str,
+    config: Any | None = None,
+) -> bool | None:
+    """Ask vLLM whether two venue names refer to the same publication venue.
+
+    Returns True (same), False (different), or None (vLLM unavailable).
+    Only called when fuzzy matching is ambiguous (score below threshold).
+    """
+    from sciwrite_lint.llm_utils import llm_query
+
+    result = await llm_query(
+        system=(
+            "You compare academic venue names. Reply with JSON only. "
+            "same_venue=true if both names refer to the same journal or conference, "
+            "even if one is an abbreviation or acronym of the other."
+        ),
+        user=f'Venue A: "{tex_venue}"\nVenue B: "{api_venue}"\nAre these the same venue?',
+        schema=_VENUE_CONFIRM_SCHEMA,
+        schema_name="venue_match",
+        config=config,
+        max_tokens=32,
+        thinking="off",
+    )
+    if result is None:
+        return None
+    return result.get("same_venue")
+
+
+def year_match(tex_year: str, api_year: Any) -> bool:
+    """Check if years match within tolerance."""
+    if not tex_year or not api_year:
+        return True  # can't compare, not a mismatch
+    try:
+        ty = int(tex_year)
+        ay = int(api_year)
+        return abs(ty - ay) <= YEAR_TOLERANCE
+    except (ValueError, TypeError):
+        return True
+
+
+def compare_citation_detailed(
+    citation: Citation,
+    api_data: dict[str, Any],
+) -> list[str]:
+    """Compare parsed citation against API data with fuzzy matching.
+
+    Returns list of issue strings.
+    """
+    if api_data.get("error"):
+        return [f"API error: {api_data['error']}"]
+
+    issues = []
+
+    # Year check
+    api_year = api_data.get("year")
+    if citation.year and api_year and not year_match(citation.year, api_year):
+        issues.append(f"Year mismatch: tex={citation.year}, API={api_year}")
+
+    # Title check
+    api_title = api_data.get("title", "")
+    if api_title and citation.title:
+        sim = title_similarity(citation.title, api_title)
+        if sim < TITLE_THRESHOLD:
+            issues.append(
+                f"Title mismatch (similarity={sim:.2f}): "
+                f"tex='{citation.title[:60]}', API='{api_title[:60]}'"
+            )
+
+    # Author check
+    api_authors = api_data.get("authors", [])
+    if api_authors and citation.authors:
+        sim = author_similarity(citation.authors, api_authors)
+        if sim < AUTHOR_THRESHOLD:
+            tex_str = ", ".join(citation.authors[:3])
+            api_str = ", ".join(str(a) for a in api_authors[:3])
+            issues.append(
+                f"Author mismatch (similarity={sim:.2f}): "
+                f"tex='{tex_str}', API='{api_str}'"
+            )
+
+    # Venue check
+    api_venue = api_data.get("venue", "")
+    if api_venue and citation.venue:
+        sim = venue_similarity(citation.venue, api_venue)
+        if sim < VENUE_THRESHOLD:
+            issues.append(
+                f"Venue mismatch (similarity={sim:.2f}): "
+                f"tex='{citation.venue[:60]}', API='{api_venue[:60]}'"
+            )
+
+    # Wrong identifier in bib entry (ID lookup returned different paper)
+    id_mismatch = api_data.get("id_mismatch")
+    if id_mismatch:
+        wrong_doi = id_mismatch.get("bib_doi", "")
+        wrong_arxiv = id_mismatch.get("bib_arxiv_id", "")
+        wrong_pmid = id_mismatch.get("bib_pmid", "")
+        wrong_id = wrong_doi or wrong_arxiv or wrong_pmid
+        issues.append(
+            f"Wrong identifier in bib entry: '{wrong_id}' resolves to "
+            f"'{id_mismatch.get('wrong_title', '')[:60]}', "
+            f"not the paper found by title search"
+        )
+
+    # No identifier in bib entry
+    has_any_id = (
+        citation.doi
+        or citation.isbn
+        or citation.pmid
+        or citation.arxiv_id
+        or citation.pmc_id
+        or citation.lccn
+    )
+    if not has_any_id:
+        if api_data.get("doi"):
+            issues.append(
+                f"No identifier in bib entry; DOI available: {api_data['doi']}"
+            )
+        else:
+            issues.append(
+                "No identifier in bib entry (no DOI, ISBN, PMID, or arXiv ID)"
+            )
+
+    # Open access PDF available but no local file
+    if api_data.get("pdf_url") and citation.local_status == "none":
+        issues.append(f"Open access PDF available: {api_data['pdf_url']}")
+
+    # Retraction check
+    if api_data.get("retracted"):
+        issues.append("RETRACTED: This paper has been retracted")
+
+    return issues

sciwrite_lint/references/metadata.py

@@ -0,0 +1,273 @@
+"""Persistent per-citation metadata store.
+
+Stores CitationMetadata in workspace.db (citation_metadata table).
+All functions accept ``references_dir`` — the per-paper workspace root
+(e.g. ``references/paper_a/``).
+"""
+
+from __future__ import annotations
+
+from datetime import date
+from pathlib import Path
+from typing import Any
+
+from sciwrite_lint.models import Citation, CitationMetadata
+
+
+def load_metadata(key: str, references_dir: Path) -> CitationMetadata | None:
+    """Load metadata for a citation key from workspace.db."""
+    from sciwrite_lint.references.workspace_db import (
+        load_citation_metadata,
+        open_db,
+    )
+
+    conn = open_db(references_dir)
+    try:
+        return load_citation_metadata(conn, key)
+    finally:
+        conn.close()
+
+
+def load_all_metadata(references_dir: Path) -> dict[str, CitationMetadata]:
+    """Load all metadata from workspace.db."""
+    from sciwrite_lint.references.workspace_db import (
+        load_all_citation_metadata,
+        open_db,
+    )
+
+    conn = open_db(references_dir)
+    try:
+        return load_all_citation_metadata(conn)
+    finally:
+        conn.close()
+
+
+def save_metadata(meta: CitationMetadata, references_dir: Path) -> None:
+    """Save metadata to workspace.db."""
+    from sciwrite_lint.references.workspace_db import (
+        open_db,
+        save_citation_metadata,
+    )
+
+    conn = open_db(references_dir)
+    try:
+        save_citation_metadata(conn, meta)
+    finally:
+        conn.close()
+
+
+def compute_tier(meta: CitationMetadata) -> str:
+    """Compute access tier from metadata.
+
+    Academic citations:
+      T1: verified + full text locally (can check claims)
+      T2: verified (confirmed real, no full text to check claims against)
+      T3: not found in APIs (cannot confirm existence)
+
+    Web resources:
+      T1: URL alive + content downloaded locally
+      T2: URL alive but no local content
+      T3: Dead URL
+    """
+    # Manual override takes precedence
+    if meta.manual_override and meta.manual_override.get("tier"):
+        return meta.manual_override["tier"]
+
+    has_local = bool(meta.access.get("local_file"))
+
+    # Web resource path
+    if meta.api_match == "web_dead":
+        return "T3"
+    if meta.api_match == "web_verified":
+        return "T1" if has_local else "T2"
+
+    # Academic path
+    is_verified = meta.api_match in ("verified", "mismatch")
+
+    if not is_verified:
+        return "T3"
+    if has_local:
+        return "T1"
+    return "T2"
+
+
+def build_metadata_from_citation(
+    citation: Citation,
+    api_data: dict[str, Any] | None = None,
+    references_dir: Path | None = None,
+) -> CitationMetadata:
+    """Build a CitationMetadata from a Citation and optional API data."""
+    meta = CitationMetadata(key=citation.key)
+
+    # Bibitem data (what our .tex says)
+    meta.bibitem = {
+        "title": citation.title,
+        "authors": citation.authors,
+        "year": citation.year,
+        "venue": citation.venue,
+        "doi": citation.doi,
+        "url": citation.url,
+        "arxiv_id": citation.arxiv_id,
+        "pmid": citation.pmid,
+        "pmc_id": citation.pmc_id,
+        "isbn": citation.isbn,
+        "lccn": citation.lccn,
+        "entry_type": citation.entry_type,
+        "source_papers": [citation.source_paper],
+    }
+
+    # API data (what the world says)
+    if api_data and api_data.get("source") == "web":
+        meta.api_source = "web"
+        meta.verified_date = str(date.today())
+        meta.canonical = {
+            "title": api_data.get("title") or citation.title,
+            "url": api_data.get("url"),
+            "status_code": api_data.get("status_code"),
+            "content_type": api_data.get("content_type"),
+            "resource_type": "web",
+        }
+        meta.api_match = citation.api_match or "web_verified"
+    elif api_data and not api_data.get("error"):
+        meta.api_source = api_data.get("source", "")
+        meta.verified_date = str(date.today())
+        meta.canonical = {
+            "title": api_data.get("title", ""),
+            "authors": api_data.get("authors", []),
+            "year": api_data.get("year"),
+            "venue": api_data.get("venue", ""),
+            "doi": api_data.get("doi", ""),
+            "url": api_data.get("url"),
+            "abstract": api_data.get("abstract"),
+            "arxiv_id": api_data.get("arxiv_id"),
+            "pmid": api_data.get("pmid"),
+            "pmcid": api_data.get("pmcid"),
+            "isbn": api_data.get("isbn"),
+            "lccn": api_data.get("lccn"),
+            "s2_pdf_url": api_data.get("s2_pdf_url"),
+            "citation_count": api_data.get("citation_count", 0),
+            "retracted": api_data.get("retracted", False),
+        }
+        meta.api_match = citation.api_match or "verified"
+    else:
+        meta.api_match = citation.api_match or "not_found"
+
+    # Access status
+    local_file = None
+    if citation.local_path and references_dir:
+        try:
+            local_file = str(Path(citation.local_path).relative_to(references_dir))
+        except ValueError:
+            local_file = Path(citation.local_path).name
+    elif citation.local_path:
+        local_file = Path(citation.local_path).name
+
+    meta.access = {
+        "tier": "",
+        "local_file": local_file,
+        "local_type": _classify_local_file(local_file, references_dir),
+        "oa_url": api_data.get("pdf_url") if api_data else None,
+        "oa_source": None,
+    }
+
+    meta.mismatches = [i for i in citation.issues if "mismatch" in i.lower()]
+    meta.issues = citation.issues
+
+    # Compute tier
+    meta.access["tier"] = compute_tier(meta)
+
+    return meta
+
+
+def _classify_local_file(
+    local_file: str | None,
+    references_dir: Path | None = None,
+) -> str:
+    """Classify a local reference file by type.
+
+    Returns: "pdf", "summary", "web_page", or "none"
+    """
+    if not local_file:
+        return "none"
+
+    path = Path(local_file)
+    if not path.is_absolute() and references_dir:
+        path = references_dir / local_file
+
+    if path.suffix == ".pdf":
+        return "pdf"
+
+    if path.suffix == ".md":
+        if "_web" in path.stem:
+            return "web_page"
+        if path.exists():
+            try:
+                head = path.read_text(encoding="utf-8")[:500]
+                if "Source: http" in head or "source: http" in head:
+                    return "web_page"
+            except OSError:
+                pass
+        return "summary"
+
+    return "none"
+
+
+def enrich_retraction_status(
+    meta: CitationMetadata,
+    rw_db: dict[str, Any],
+) -> bool:
+    """Enrich metadata with Retraction Watch database lookup.
+
+    Looks up the reference DOI in the RW database. If found, sets
+    canonical["retracted"] and canonical["retraction_status"] with
+    nature, reason, date, and source.
+
+    Args:
+        meta: Citation metadata to enrich.
+        rw_db: Retraction Watch database (DOI-keyed dict of RWEntry).
+
+    Returns:
+        True if metadata was changed (caller should re-save).
+    """
+    from sciwrite_lint.references.retraction_watch import RWEntry, lookup_doi
+
+    doi = meta.canonical.get("doi", "")
+    if not doi:
+        return False
+
+    entry: RWEntry | None = lookup_doi(rw_db, doi)
+    if entry is None:
+        return False
+
+    # Reinstatement clears retraction
+    if entry.nature == "Reinstatement":
+        changed = meta.canonical.get("retracted", False)
+        meta.canonical["retracted"] = False
+        meta.canonical.pop("retraction_status", None)
+        return changed
+
+    # Retraction or Expression of Concern → set status
+    new_status = {
+        "nature": entry.nature,
+        "reason": entry.reason,
+        "date": entry.date,
+        "source": "retraction_watch",
+    }
+
+    old_status = meta.canonical.get("retraction_status")
+    if old_status == new_status:
+        return False
+
+    meta.canonical["retraction_status"] = new_status
+    if entry.nature in ("Retraction", "Expression of Concern"):
+        meta.canonical["retracted"] = True
+
+    return True
+
+
+def merge_source_paper(meta: CitationMetadata, source_paper: str) -> None:
+    """Add a source paper to an existing metadata record."""
+    papers = meta.bibitem.get("source_papers", [])
+    if source_paper not in papers:
+        papers.append(source_paper)
+        meta.bibitem["source_papers"] = papers