reference-audit 0.1.0__py3-none-any.whl

reference_audit/inputs.py

@@ -0,0 +1,96 @@
+"""What the positional CLI arguments mean. Pure and side-effect-free, so it is directly testable.
+
+Two input shapes are supported, distinguished by file extension:
+
+    reference-audit audit main.tex references.bib     # authored sources
+    reference-audit audit paper.pdf                   # GROBID extracts both from the PDF
+
+Every rejected combination gets a message naming the fix, because "usage: ..." tells a user what the
+grammar is but not which of their two paths was the wrong one.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+CACHE_DIRNAME = ".reference_audit"
+_TEX_SUFFIXES = (".tex", ".ltx")
+
+
+class InvalidInputError(ValueError):
+    """The given paths are not a recognized input combination."""
+
+
+@dataclass(frozen=True)
+class TexBibInput:
+    """A `.bib` to audit, optionally with the manuscript that cites it."""
+
+    bib_path: Path
+    tex_path: Path | None = None
+
+    @property
+    def anchor(self) -> Path:
+        return self.bib_path
+
+
+@dataclass(frozen=True)
+class PdfInput:
+    """A PDF that is the whole input: its references and citations are both extracted from it."""
+
+    pdf_path: Path
+
+    @property
+    def anchor(self) -> Path:
+        return self.pdf_path
+
+
+AuditInput = TexBibInput | PdfInput
+
+
+def resolve_input(first: Path, second: Path | None) -> AuditInput:
+    """Classify the positional arguments, or explain why they cannot be."""
+    a = first.suffix.lower()
+    b = second.suffix.lower() if second is not None else None
+
+    if a == ".pdf":
+        if second is not None:
+            raise InvalidInputError(
+                f"a PDF is the whole input: GROBID extracts both the reference list and the in-text "
+                f"citations from {first.name}, so there is nothing for {second.name} to add. "
+                f"Run 'reference-audit audit {first.name}' on its own, or pass a .tex + .bib pair "
+                f"instead."
+            )
+        return PdfInput(first)
+
+    if b == ".pdf":
+        raise InvalidInputError(
+            f"a .pdf cannot be combined with another input ({first.name} + {second.name}). "
+            f"Run 'reference-audit audit {second.name}' on its own."
+        )
+
+    if second is None:
+        raise InvalidInputError(
+            f"expected 'audit <paper.pdf>' or 'audit <main.tex> <references.bib>', but got a single "
+            f"{a or 'extension-less'} file ({first.name}). A .bib needs its manuscript; a PDF stands "
+            f"alone."
+        )
+
+    if a == ".bib" and b in _TEX_SUFFIXES:
+        raise InvalidInputError(
+            f"the arguments look swapped — the manuscript comes first: "
+            f"'reference-audit audit {second.name} {first.name}'."
+        )
+
+    if b != ".bib":
+        raise InvalidInputError(
+            f"the second argument must be the .bib bibliography to audit, but {second.name} has "
+            f"suffix '{b or '(none)'}'."
+        )
+
+    return TexBibInput(bib_path=second, tex_path=first)
+
+
+def default_cache_path(source: AuditInput) -> Path:
+    """`<input_dir>/.reference_audit/cache.db`, beside whichever file the run is anchored on."""
+    return source.anchor.parent / CACHE_DIRNAME / "cache.db"

reference_audit/llm/__init__.py

@@ -0,0 +1,6 @@
+"""LLM adjudication: structured-output filtering of the ambiguous candidate zone.
+
+The LLM is authoritative only in the danger/ambiguous zones (the `adjudicate` bucket and, in M5,
+the SAME_WORK tie-break). Both prompts require the model to *affirmatively assert* sameness, so a
+near-miss neighbour is never force-accepted (the README's hallucination-screening guarantee).
+"""

reference_audit/llm/client.py

@@ -0,0 +1,85 @@
+"""Async OpenAI wrapper for structured-output adjudication.
+
+Thin layer over `AsyncOpenAI` returning validated pydantic objects. Retries transient failures;
+caps in-flight requests with a semaphore for batch calls. The model defaults to `gpt-6-luna`
+(configurable). `temperature` is sent only when set; otherwise the model default applies. The cache (layer 2) is consulted by the pipeline *before* calling here, so this
+layer is pure I/O.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import TypeVar
+
+from pydantic import BaseModel, ValidationError
+from tenacity import retry, stop_after_attempt, wait_exponential
+
+from reference_audit.llm.schemas import response_format
+
+T = TypeVar("T", bound=BaseModel)
+
+
+class LLMError(Exception):
+    """A structured-output call failed after retries (caller leaves the entry unresolved)."""
+
+
+class LLMClient:
+    def __init__(
+        self,
+        *,
+        model: str,
+        api_key: str | None,
+        base_url: str | None = None,
+        concurrency: int = 8,
+        temperature: float | None = None,
+        client=None,
+    ):
+        self.model = model
+        self.temperature = temperature
+        self._semaphore = asyncio.Semaphore(concurrency)
+        if client is not None:
+            self._client = client
+        else:
+            from openai import AsyncOpenAI
+
+            self._client = AsyncOpenAI(api_key=api_key, base_url=base_url)
+
+    @property
+    def available(self) -> bool:
+        return self._client is not None
+
+    async def structured(
+        self, system: str, user: str, schema_model: type[T], schema_name: str
+    ) -> T:
+        """One structured call → a validated `schema_model` instance. Raises LLMError on failure."""
+
+        @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=0.5, min=0.5, max=8),
+               reraise=True)
+        async def _call() -> str:
+            extra = {} if self.temperature is None else {"temperature": self.temperature}
+            resp = await self._client.chat.completions.create(
+                model=self.model,
+                messages=[
+                    {"role": "system", "content": system},
+                    {"role": "user", "content": user},
+                ],
+                response_format=response_format(schema_model, schema_name),
+                **extra,
+            )
+            return resp.choices[0].message.content or ""
+
+        async with self._semaphore:
+            try:
+                content = await _call()
+                return schema_model.model_validate_json(content)
+            except (ValidationError, Exception) as exc:  # noqa: BLE001 — normalize to LLMError
+                if isinstance(exc, LLMError):
+                    raise
+                raise LLMError(f"{schema_name} call failed: {exc}") from exc
+
+    async def aclose(self) -> None:
+        close = getattr(self._client, "aclose", None) or getattr(self._client, "close", None)
+        if close is not None:
+            result = close()
+            if asyncio.iscoroutine(result):
+                await result

reference_audit/llm/prompts.py

@@ -0,0 +1,206 @@
+"""Prompt construction for the LLM adjudication calls.
+
+Both prompts use AFFIRMATIVE polarity: the model must positively conclude sameness; absence of
+evidence ⇒ "no". This is what makes a garbled entry's nearest fuzzy neighbour fail to match (T4).
+"""
+
+from __future__ import annotations
+
+from reference_audit.models import BibEntry, FeatureVector, SourceRecord
+
+CAN_CORRESPOND_SYSTEM = (
+    "You verify whether a returned database record is the SAME PUBLISHED ARTIFACT that a "
+    "bibliography entry refers to. Titles and author names may be spelled differently, "
+    "transliterated, abbreviated, or have a missing field (e.g. a dropped co-author or wrong "
+    "month) — do NOT reject for those. Set can_correspond=true ONLY if you can affirmatively "
+    "conclude they are the same document. Set can_correspond=false when there is positive evidence "
+    "they are DIFFERENT works: a different scope or topic in the title tail, disjoint page ranges "
+    "in the same venue, an author set that differs by more than spelling, or a clearly different "
+    "subject. A merely missing field is not evidence of difference. Respond in strict JSON."
+)
+
+SAME_WORK_SYSTEM = (
+    "Decide whether two database records describe the SAME WORK. Choose exactly one relation: "
+    "'same_artifact' (the identical item), 'versions_of_same_work' (a preprint and its published "
+    "version, or different editions/printings of one work), 'distinct_works' (different papers, "
+    "even if by the same authors in the same venue/year), or 'uncertain'. Two papers with "
+    "different titles and non-overlapping page ranges are distinct_works. Conclude 'same' or "
+    "'versions' ONLY with positive evidence. Respond in strict JSON."
+)
+
+WEB_MATCH_SYSTEM = (
+    "A bibliography entry cites a WEB PAGE by URL — a blog post, a software/project page, "
+    "documentation, a dataset, or another non-journal resource. You are given the .bib entry and the "
+    "title, metadata, and text of the page actually fetched from that URL. Decide whether the fetched "
+    "page IS the resource the entry cites. Set corresponds=true ONLY if you can affirmatively conclude "
+    "the page is that resource — its title/topic/authors match the citation, allowing for spelling, "
+    "abbreviation, transliteration, or an appended site name. Set corresponds=false ONLY with positive "
+    "evidence the page is something else: a different article, a site homepage or index/listing, a "
+    "login/paywall wall, a 'page not found'/error notice, or an unrelated topic. When the page has too "
+    "little content to tell, or it is ambiguous, use low confidence rather than guessing. Respond in "
+    "strict JSON."
+)
+
+FIELD_CHECK_SYSTEM = (
+    "A bibliography entry has already been matched, by identifier, to a specific real publication, "
+    "so the two refer to the SAME work. You judge ONE field of that entry against the authoritative "
+    "database value and decide whether the entry's value is merely formatted differently or is a "
+    "genuine mistake.\n"
+    "Classify 'formatting_variant' (NOT a mistake) for: capitalization; punctuation; accents and "
+    "transliteration (Müller/Mueller); LaTeX braces; standard abbreviations vs full forms of a "
+    "journal or conference (e.g. 'J. Theor. Biol.' = 'Journal of Theoretical Biology', 'NeurIPS' = "
+    "'Advances in Neural Information Processing Systems'); an added or dropped 'Proceedings of'/year/"
+    "edition qualifier on a conference series; en-dash vs hyphen; 'and' vs '&'; word-order in a "
+    "series name.\n"
+    "Classify 'error' for a substantively different value: a different number; a wrong, misspelled, "
+    "or split word (e.g. 'Un iversity'); a missing word that changes the name (e.g. a journal "
+    "written 'Annual Review Condensed Matter Physics', dropping 'of'); a truncated or different "
+    "title or venue; a value that belongs to a different field.\n"
+    "Databases are themselves sometimes wrong or incomplete, and the bibliography value is usually "
+    "correct: \n"
+    "- If the database VENUE is a preprint server, institutional repository, or aggregator "
+    "('arXiv', 'bioRxiv', 'medRxiv', 'SSRN', 'ResearchGate', 'Radboud Repository', 'HAL', a "
+    "university or 'Technical Reports Server') while the entry names a journal or conference, the "
+    "database simply indexed a different copy — classify 'formatting_variant', not 'error'.\n"
+    "- If the database value looks like a TRUNCATION or substring of the entry's value (e.g. "
+    "database 'Complex' vs entry 'Complexity'), the entry is the fuller, correct form — classify "
+    "'formatting_variant' or 'uncertain', never 'error'.\n"
+    "- A TITLE that differs from the database title by a substituted, inserted or deleted content "
+    "word ('Robust' vs 'Resilient', 'towards' vs 'for', 'Self-supervised' vs 'Unsupervised') is an "
+    "'error': the identifier already fixed the work, so its title is known.\n"
+    "- A VENUE that names a different journal or conference series than the database venue ('ACL' vs "
+    "'ICLR', 'UAI' vs 'ICML') is an 'error': the database records where the identified work "
+    "appeared (this does not apply to a preprint server or repository, above).\n"
+    "If the entry's value is plausibly correct and the database merely differs, prefer 'uncertain' "
+    "over 'error'. Use 'uncertain' whenever you cannot affirmatively decide. Respond in strict JSON."
+)
+
+
+CITATION_ALIGNMENT_SYSTEM = (
+    "A bibliography entry has been matched to a specific real publication (the CITED WORK). You are "
+    "given (1) the sentence(s) from the citing paper that invoke the cited work — the CLAIM the "
+    "author attaches to the citation — and (2) the ABSTRACT of the cited work. Decide whether the "
+    "cited work's abstract is consistent with the way it is cited.\n"
+    "Classify 'supported' when the abstract affirmatively corroborates the citing claim.\n"
+    "Classify 'contradicted' ONLY with positive evidence that the abstract asserts the OPPOSITE of "
+    "the citing claim (the claim attributes to the work a finding the abstract explicitly refutes, "
+    "reverses, or excludes).\n"
+    "Classify 'not_in_abstract' when the abstract neither supports nor contradicts the claim — it is "
+    "silent on that point. An abstract is only a summary, so its silence is NOT evidence of misuse: "
+    "prefer 'not_in_abstract' over 'contradicted' whenever the abstract simply does not address the "
+    "claim. Do NOT use outside knowledge; judge only against the abstract text provided.\n"
+    "For a 'supported' or 'contradicted' verdict, put the short span of the abstract that justifies "
+    "it in evidence_quote (leave it empty for 'not_in_abstract'). Respond in strict JSON."
+)
+
+# Cap the abstract text fed to the model: enough to judge the claim, bounded for cost.
+ABSTRACT_TEXT_LIMIT = 4000
+
+
+def citation_alignment_user(entry: BibEntry, context_text: str, abstract: str) -> str:
+    return (
+        "CITED WORK:\n"
+        f"  title:   {entry.title or '(none)'}\n"
+        f"  authors: {'; '.join(entry.authors) or '(none)'}\n"
+        f"  year:    {entry.year or '(none)'}\n\n"
+        "HOW IT IS CITED (the claim the citing paper attaches to this reference):\n"
+        f"  {context_text}\n\n"
+        "ABSTRACT OF THE CITED WORK:\n"
+        f"{abstract[:ABSTRACT_TEXT_LIMIT] or '(none)'}\n\n"
+        "Is the citing claim supported by, contradicted by, or simply not addressed in the abstract?"
+    )
+
+
+def field_check_user(
+    field: str, bib_value: str, canonical_value: str, sources: list[str], work: SourceRecord
+) -> str:
+    """`work` is the matched work as the database records it, never the entry under review."""
+    src = ", ".join(sources) if sources else "database"
+    authors = "; ".join(work.authors[:12]) + ("; …" if len(work.authors) > 12 else "")
+    return (
+        f"FIELD UNDER REVIEW: {field}\n"
+        f"  citation (.bib) value:  {bib_value or '(empty)'}\n"
+        f"  authoritative value:    {canonical_value or '(empty)'}   [source: {src}]\n\n"
+        "CONTEXT — the matched work as the database records it (for grounding only):\n"
+        f"  title:   {work.title or '(none)'}\n"
+        f"  authors: {authors or '(none)'}\n"
+        f"  year:    {work.year or '(none)'}\n"
+        f"  venue:   {work.venue or '(none)'}\n\n"
+        f"Is the .bib {field} value the same as the authoritative value apart from formatting, "
+        "or is it a genuine mistake?"
+    )
+
+
+def _fmt_entry(entry: BibEntry) -> str:
+    ids = entry.ids
+    return (
+        f"  title:   {entry.title}\n"
+        f"  authors: {'; '.join(entry.authors) or '(none)'}\n"
+        f"  year:    {entry.year or '(none)'}\n"
+        f"  venue:   {entry.venue or '(none)'}\n"
+        f"  pages:   {entry.pages or '(none)'}\n"
+        f"  ids:     doi={ids.doi} arxiv={ids.arxiv_id} isbn={ids.isbn13}"
+    )
+
+
+def _fmt_record(rec: SourceRecord) -> str:
+    ids = rec.ids
+    return (
+        f"  source:  {rec.source}\n"
+        f"  title:   {rec.title}\n"
+        f"  authors: {'; '.join(rec.authors) or '(none)'}\n"
+        f"  year:    {rec.year or '(none)'}\n"
+        f"  venue:   {rec.venue or '(none)'}\n"
+        f"  pages:   {rec.pages or '(none)'}\n"
+        f"  ids:     doi={ids.doi} arxiv={ids.arxiv_id} isbn={ids.isbn13}"
+    )
+
+
+def _fmt_flags(f: FeatureVector) -> str:
+    return (
+        f"  title_prefix_trap={f.title_prefix_trap} pages_conflict={f.pages_conflict} "
+        f"author_set_jaccard={f.author_set_jaccard:.2f} id_agreement={f.id_agreement} "
+        f"title_ratio={f.title_ratio:.2f}"
+    )
+
+
+def can_correspond_user(entry: BibEntry, rec: SourceRecord, features: FeatureVector) -> str:
+    return (
+        "BIB ENTRY:\n" + _fmt_entry(entry) + "\n\n"
+        "DATABASE RECORD:\n" + _fmt_record(rec) + "\n\n"
+        "ATTENTION FLAGS (formal signals; weigh them):\n" + _fmt_flags(features) + "\n\n"
+        "Does the database record correspond to the same published artifact as the bib entry?"
+    )
+
+
+def same_work_user(a: SourceRecord, b: SourceRecord) -> str:
+    return (
+        "RECORD A:\n" + _fmt_record(a) + "\n\n"
+        "RECORD B:\n" + _fmt_record(b) + "\n\n"
+        "What is the relation between record A and record B?"
+    )
+
+
+# Cap the fetched page text fed to the model: enough to judge identity, bounded for cost.
+WEB_PAGE_TEXT_LIMIT = 4000
+
+
+def web_match_user(entry: BibEntry, page: SourceRecord) -> str:
+    raw = page.raw or {}
+    text = (raw.get("text") or "").strip()[:WEB_PAGE_TEXT_LIMIT]
+    return (
+        "CITED REFERENCE (.bib):\n"
+        f"  title:     {entry.title or '(none)'}\n"
+        f"  authors:   {'; '.join(entry.authors) or '(none)'}\n"
+        f"  year:      {entry.year or '(none)'}\n"
+        f"  cited url: {entry.ids.url or '(none)'}\n\n"
+        "FETCHED PAGE (from the cited URL):\n"
+        f"  final url:    {page.ids.url or '(none)'}\n"
+        f"  meta title:   {page.title or '(none)'}\n"
+        f"  meta authors: {'; '.join(page.authors) or '(none)'}\n"
+        f"  site name:    {raw.get('site_name') or '(none)'}\n"
+        f"  description:  {raw.get('description') or '(none)'}\n\n"
+        "PAGE TEXT (truncated):\n"
+        f"{text or '(no extractable text)'}\n\n"
+        "Is the fetched page the same resource the .bib entry cites?"
+    )

reference_audit/llm/schemas.py

@@ -0,0 +1,38 @@
+"""pydantic → strict JSON schema for OpenAI structured outputs.
+
+OpenAI `response_format=json_schema` with `strict: true` requires every object to set
+`additionalProperties: false` and list ALL properties in `required`. pydantic's default schema
+omits fields that have defaults from `required`, so we tighten it here.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel
+
+
+def _strictify(node: Any) -> Any:
+    if isinstance(node, dict):
+        node.pop("default", None)
+        if node.get("type") == "object" and "properties" in node:
+            node["additionalProperties"] = False
+            node["required"] = list(node["properties"].keys())
+        for value in node.values():
+            _strictify(value)
+    elif isinstance(node, list):
+        for item in node:
+            _strictify(item)
+    return node
+
+
+def strict_schema(model: type[BaseModel]) -> dict:
+    """Return an OpenAI strict-mode JSON schema for a pydantic model."""
+    return _strictify(model.model_json_schema())
+
+
+def response_format(model: type[BaseModel], name: str) -> dict:
+    return {
+        "type": "json_schema",
+        "json_schema": {"name": name, "schema": strict_schema(model), "strict": True},
+    }

reference_audit/matching/__init__.py

@@ -0,0 +1,6 @@
+"""Matching / disambiguation — the heart.
+
+Formal code produces interpretable *features/evidence* (`features.py`); buckets and the
+SAME-OBJECT rule turn evidence into verdicts. The LLM (M4+) adjudicates only the danger/ambiguous
+zones. `_best_match`'s single scalar is demoted to one advisory `composite` feature.
+"""

reference_audit/matching/adjudicate.py

@@ -0,0 +1,63 @@
+"""LLM adjudication of the `adjudicate` bucket (the danger zone).
+
+Runs one independent `CAN_CORRESPOND` call per surviving candidate (capped, top-k by composite),
+caching each decision. A candidate the model affirmatively confirms is promoted to `auto_accept`;
+everything else stays as-is. Returns the LLM errors that occurred (empty when none did), so the
+caller can keep an entry UNRESOLVED rather than emit a false `none` when the deciding call failed,
+and say which call failed.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+from reference_audit.cache.store import AuditCache, prompt_hash
+from reference_audit.config import AuditConfig
+from reference_audit.llm.client import LLMClient, LLMError
+from reference_audit.llm.prompts import CAN_CORRESPOND_SYSTEM, can_correspond_user
+from reference_audit.models import CanCorrespondResult, EntryAudit
+
+
+async def adjudicate_entry(
+    audit: EntryAudit, llm: LLMClient, config: AuditConfig, cache: AuditCache | None
+) -> list[str]:
+    """Adjudicate an entry's `adjudicate` candidates with the LLM. Returns the LLM errors."""
+    candidates = [c for c in audit.candidates if c.bucket == "adjudicate"]
+    candidates.sort(key=lambda c: c.features.composite, reverse=True)
+    candidates = candidates[: config.llm_max_candidates]
+    if not candidates:
+        return []
+
+    async def decide(candidate) -> tuple[object, CanCorrespondResult | None, str | None]:
+        system = CAN_CORRESPOND_SYSTEM
+        user = can_correspond_user(audit.entry, candidate.record, candidate.features)
+        p_hash = prompt_hash(system + "\n" + user)
+        if cache is not None:
+            cached = cache.get_llm_decision(p_hash, "can_correspond")
+            if cached is not None:
+                return candidate, CanCorrespondResult.model_validate_json(cached), None
+        try:
+            result = await llm.structured(
+                system, user, CanCorrespondResult, "can_correspond"
+            )
+        except LLMError as exc:
+            where = f"{candidate.record.source} '{candidate.record.title[:80]}'"
+            return candidate, None, f"candidate {where}: {exc}"
+        if cache is not None:
+            cache.put_llm_decision(p_hash, "can_correspond", result.model_dump_json())
+        return candidate, result, None
+
+    outcomes = await asyncio.gather(*(decide(c) for c in candidates))
+    errors: list[str] = []
+    for candidate, result, err in outcomes:
+        if err:
+            errors.append(err)
+        if result is None:
+            continue
+        candidate.llm = result
+        if result.confidence in ("high", "medium"):
+            # Confident LLM ruling is decisive: accept → match; reject → confirmed non-match, so the
+            # entry can conclude `none` when every candidate is rejected. Low confidence stays
+            # `adjudicate` (the entry remains unresolved rather than guessed).
+            candidate.bucket = "auto_accept" if result.can_correspond else "auto_reject"
+    return errors