reference-audit 0.1.0__py3-none-any.whl

reference_audit/matching/features.py

@@ -0,0 +1,185 @@
+"""FeatureVector: interpretable per-(entry, candidate) signals.
+
+Decomposes `_best_match`'s scalar into named features. M2 wires title/author/year/id_agreement +
+the advisory `composite`; the distinct-work detectors (prefix_trap, author_set_jaccard,
+pages_conflict) are implemented here too and become load-bearing in M5's SAME-OBJECT rule.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Literal
+
+from anyascii import anyascii
+from rapidfuzz import fuzz
+
+from reference_audit.models import BibEntry, FeatureVector, Identifiers, SourceRecord
+from reference_audit.parsing.entities import decode_html_entities
+from reference_audit.matching.names import author_overlap, author_set_jaccard, author_subset
+
+
+def _norm_title(t: str) -> str:
+    text = anyascii(decode_html_entities(t or "")).lower()  # "&" is no word "amp"
+    return re.sub(r"\s+", " ", re.sub(r"[^a-z0-9 ]", " ", text)).strip()
+
+
+def title_ratio(a: str, b: str) -> float:
+    """Length-guarded title similarity.
+
+    `token_set_ratio` returns 100 when the shorter title's tokens are a *subset* of the longer's
+    ("Fitness Landscapes" ⊂ "Fitness Landscapes and the Origin of Species"), which would falsely
+    match a different, shorter-titled work. So token_set_ratio is only trusted when the two titles
+    have comparable token counts; otherwise fall back to order-aware ratios that penalize the
+    length gap.
+    """
+    na, nb = _norm_title(a), _norm_title(b)
+    if not na or not nb:
+        return 0.0
+    base = max(fuzz.ratio(na, nb), fuzz.token_sort_ratio(na, nb))
+    ta, tb = len(na.split()), len(nb.split())
+    length_ratio = min(ta, tb) / max(ta, tb) if max(ta, tb) else 0.0
+    if length_ratio >= 0.7:  # comparable lengths → token_set reordering bonus is safe
+        base = max(base, fuzz.token_set_ratio(na, nb))
+    return base / 100.0
+
+
+def _tokens(t: str) -> list[str]:
+    return [w for w in _norm_title(t).split() if w]
+
+
+def title_prefix_trap(a: str, b: str, *, tail_threshold: float) -> bool:
+    """High shared leading-token overlap but divergent tails (bagrov vs kravchenko, T2a)."""
+    ta, tb = _tokens(a), _tokens(b)
+    if len(ta) < 4 or len(tb) < 4:
+        return False
+    shared_prefix = 0
+    for x, y in zip(ta, tb):
+        if x == y:
+            shared_prefix += 1
+        else:
+            break
+    if shared_prefix < 3:
+        return False
+    tail_a, tail_b = set(ta[shared_prefix:]), set(tb[shared_prefix:])
+    if not tail_a or not tail_b:
+        return False
+    tail_jaccard = len(tail_a & tail_b) / len(tail_a | tail_b)
+    return tail_jaccard < tail_threshold
+
+
+def year_factor(qy: int | None, iy: int | None) -> float:
+    if qy is None or iy is None:
+        return 1.0
+    delta = abs(qy - iy)
+    return 1.0 / (1.0 + (delta / 3.0) ** 2)  # ~1.0 at ±1, never a hard gate (preprint↔published)
+
+
+def _published_doi(doi: str | None) -> str | None:
+    """A DOI that identifies a published artifact (an arXiv DataCite DOI is the preprint id, not a
+    competing published DOI, so it must not register as a DOI conflict)."""
+    if not doi or doi.startswith("10.48550/arxiv"):
+        return None
+    return doi
+
+
+def id_agreement(a: Identifiers, b: Identifiers) -> Literal["match", "conflict", "absent"]:
+    """Opaque-token compare across shared id kinds: match wins; conflict if a shared kind disagrees.
+
+    arXiv DataCite DOIs are compared as arXiv ids, not as published DOIs — a published DOI vs an
+    arXiv preprint DOI is a version relationship, not a conflict.
+    """
+    saw_shared = False
+    # ISBN is set-valued: one book registers several ISBN-13s (print + electronic, per edition), so a
+    # shared ISBN anywhere in the two sets is a match, and it's a conflict only when the sets are
+    # wholly disjoint — never because the cite and the source picked different printings of one book.
+    a_isbns, b_isbns = a.all_isbn13(), b.all_isbn13()
+    if a_isbns and b_isbns:
+        saw_shared = True
+        if a_isbns & b_isbns:
+            return "match"
+    pairs = (
+        (_published_doi(a.doi), _published_doi(b.doi)),
+        (a.arxiv_id, b.arxiv_id),
+        (a.pmid, b.pmid),
+        (a.openalex, b.openalex),
+    )
+    for x, y in pairs:
+        if x and y:
+            saw_shared = True
+            if x == y:
+                return "match"
+    return "conflict" if saw_shared else "absent"
+
+
+_PAGE_RANGE_RE = re.compile(r"(\d+)\s*[-–—]+\s*(\d+)")
+
+
+def _page_range(pages: str) -> tuple[int, int] | None:
+    m = _PAGE_RANGE_RE.search(pages or "")
+    if not m:
+        return None
+    lo, hi = int(m.group(1)), int(m.group(2))
+    return (lo, hi) if lo <= hi else (hi, lo)
+
+
+def _pages_conflict(pages_a: str, venue_a: str, pages_b: str, venue_b: str) -> bool:
+    ra, rb = _page_range(pages_a), _page_range(pages_b)
+    if not ra or not rb:
+        return False
+    same_venue = title_ratio(venue_a, venue_b) > 0.8 if venue_a and venue_b else True
+    if not same_venue:
+        return False
+    return rb[1] < ra[0] or ra[1] < rb[0]  # disjoint
+
+
+def pages_conflict(entry: BibEntry, record: SourceRecord) -> bool:
+    """Disjoint page ranges in the same venue (laughlin consecutive pages, T2c)."""
+    return _pages_conflict(entry.pages, entry.venue, record.pages, record.venue)
+
+
+def pages_conflict_records(a: SourceRecord, b: SourceRecord) -> bool:
+    """Record-vs-record page conflict (used by SAME-OBJECT clustering)."""
+    return _pages_conflict(a.pages, a.venue, b.pages, b.venue)
+
+
+def venue_compatible(entry_venue: str, record_venue: str) -> float:
+    if not entry_venue or not record_venue:
+        return 1.0  # missing venue never penalizes
+    return 0.9 + 0.1 * (fuzz.partial_ratio(_norm_title(entry_venue), _norm_title(record_venue)) / 100.0)
+
+
+# A "venue" that is really a preprint server, an institutional repository or an aggregator. A record
+# with such a venue is a preprint/repository copy: it cannot confirm (or refute) the journal or
+# conference an entry cites, and pooling must not let it stand in for the published venue. DBLP files
+# arXiv preprints under the venue "CoRR".
+_REPOSITORY_VENUE_RE = re.compile(
+    r"arxiv|\bcorr\b|bio\s*rxiv|med\s*rxiv|chem\s*rxiv|preprint|repositor|researchgate|\bssrn\b|"
+    r"zenodo|figshare|\bosf\b|hal[-\s]|scholarworks|dspace|eprints|infoscience|research\s+square|"
+    r"technical reports server|\bscholar \(|\(.*\buniversit",
+    re.IGNORECASE,
+)
+
+
+def is_repository_venue(venue: str | None) -> bool:
+    """Whether `venue` names a preprint server or repository rather than a journal or conference."""
+    return bool(venue and _REPOSITORY_VENUE_RE.search(venue))
+
+
+def compute_features(entry: BibEntry, record: SourceRecord, *, tail_threshold: float) -> FeatureVector:
+    t_ratio = title_ratio(entry.title, record.title)
+    a_overlap = author_overlap(entry.authors, record.authors)
+    yf = year_factor(entry.year, record.year)
+    vc = venue_compatible(entry.venue, record.venue)
+    composite = t_ratio * (0.5 + 0.5 * a_overlap) * yf * vc
+    return FeatureVector(
+        title_ratio=t_ratio,
+        title_prefix_trap=title_prefix_trap(entry.title, record.title, tail_threshold=tail_threshold),
+        author_overlap=a_overlap,
+        author_set_jaccard=author_set_jaccard(entry.authors, record.authors),
+        author_subset=author_subset(entry.authors, record.authors),
+        year_factor=yf,
+        venue_compatible=vc,
+        id_agreement=id_agreement(entry.ids, record.ids),
+        pages_conflict=pages_conflict(entry, record),
+        composite=composite,
+    )

reference_audit/matching/names.py

@@ -0,0 +1,238 @@
+"""Author-name matching (variant- and transliteration-aware).
+
+Distilled from `sciwrite-lint/api.py:_name_variants/_author_overlap`. Names may be "Last, First",
+"First Last", initials, or transliterated (anyascii) — we compare on normalized last names with a
+fuzzy fallback, so "Vanchurin, Vitaly" ≈ "Vitaly Vanchurin" and "Müller" ≈ "Mueller".
+"""
+
+from __future__ import annotations
+
+import re
+
+from anyascii import anyascii
+from rapidfuzz import fuzz
+
+from reference_audit.parsing.entities import decode_html_entities
+
+
+def _norm(text: str) -> str:
+    # Decode HTML references first: "d'Amore" must compare as "d'Amore", not "dapos amore".
+    text = anyascii(decode_html_entities(text or ""))
+    return re.sub(r"[^a-z\s,.-]", "", text.lower()).strip()
+
+
+# The BibTeX `and others` convention (and a written-out "et al.") is a *truncation marker*, not a
+# real author. Left in, the phantom surname "others" both drags `author_overlap` down and breaks the
+# subset check (an intentionally abbreviated list is no longer ⊆ the full author list), which then
+# trips the distinct-author-set veto and forces needless adjudication. Drop it everywhere.
+_ETAL_MARKERS = {"others", "et al", "and others"}
+
+
+def _is_etal(name: str) -> bool:
+    return _norm(name).replace(".", "").strip() in _ETAL_MARKERS
+
+
+def _named(authors: list[str]) -> list[str]:
+    """Author list with any et-al./`others` truncation marker removed."""
+    return [a for a in authors if not _is_etal(a)]
+
+
+def last_name(name: str) -> str:
+    """Best-effort surname extraction handling 'Last, First' and 'First Last'."""
+    n = _norm(name)
+    if not n:
+        return ""
+    if "," in n:
+        return n.split(",", 1)[0].strip()
+    tokens = [t for t in n.replace(".", " ").split() if t]
+    return tokens[-1] if tokens else ""
+
+
+def author_set(authors: list[str]) -> set[str]:
+    """Set of normalized surnames (for set-level Jaccard / subset checks)."""
+    return {ln for a in _named(authors) if (ln := last_name(a))}
+
+
+def author_overlap(query_authors: list[str], item_authors: list[str]) -> float:
+    """Average best per-query-author surname match in [0, 1]."""
+    query_authors, item_authors = _named(query_authors), _named(item_authors)
+    if not query_authors or not item_authors:
+        return 0.0
+    item_last = [last_name(a) for a in item_authors]
+    item_last = [ln for ln in item_last if ln]
+    if not item_last:
+        return 0.0
+    scores: list[float] = []
+    for qa in query_authors:
+        ql = last_name(qa)
+        if not ql:
+            continue
+        best = max((fuzz.ratio(ql, il) / 100.0 for il in item_last), default=0.0)
+        scores.append(best)
+    return sum(scores) / len(scores) if scores else 0.0
+
+
+def author_set_jaccard(query_authors: list[str], item_authors: list[str]) -> float:
+    a, b = author_set(query_authors), author_set(item_authors)
+    if not a or not b:
+        return 0.0
+    return len(a & b) / len(a | b)
+
+
+def author_subset(query_authors: list[str], item_authors: list[str]) -> bool:
+    """True if one author set is contained in the other (beyond spelling) — laughlin asymmetry."""
+    a, b = author_set(query_authors), author_set(item_authors)
+    if not a or not b:
+        return False
+    return a <= b or b <= a
+
+
+# ── person-level matching (the author check) ─────────────────────────────────
+#
+# The surname-only fuzzy match above suits scoring, where an average over the list absorbs one bad
+# name. The author check names individual people, so it compares whole names, order-free: databases
+# disagree on name order ('Li Tian' / 'Tian Li', OpenAlex's 'Moriano Pablo') and on how much of a
+# compound surname they keep ('Lierni Sestorain' / 'Sestorain Saralegui, Lierni', 'Carlos Riquelme
+# Ruiz' / 'Carlos Riquelme'). A surname-only fuzzy match also lets a fabricated author through on a
+# near-namesake ('Carreira' ≈ 'Barreira' at 0.875).
+
+_NAME_SPLIT_RE = re.compile(r"[\s,.\-‐]+")
+_UMLAUT_RE = re.compile(r"([aou])e")
+
+# Forms of one given name: a diminutive ('Tim Mann' / 'Timothy A. Mann'), and a few spellings and
+# transliterations ('Aleksandar' / 'Alexander'). A prefix or fuzzy rule cannot stand in for this
+# list: it would also equate surnames such as Chen / Cheng or Li / Liang.
+_GIVEN_NAME_FORMS: tuple[frozenset[str], ...] = tuple(frozenset(g) for g in (
+    ("timothy", "tim"), ("thomas", "tom"), ("daniel", "dan", "danny"), ("david", "dave"),
+    ("michael", "mike"), ("william", "will", "bill", "liam"), ("robert", "rob", "bob", "bobby"),
+    ("alexander", "alex", "aleksandar", "aleksander", "alexandre"), ("christopher", "chris"),
+    ("benjamin", "ben"), ("samuel", "sam"), ("nicholas", "nick", "nicolas", "nikolas"),
+    ("matthew", "matt"), ("james", "jim", "jamie"), ("joseph", "joe"),
+    ("stephen", "steve", "steven"), ("edward", "ed", "eddie"), ("andrew", "andy"),
+    ("anthony", "tony"), ("elizabeth", "liz", "beth"), ("katherine", "kate", "katie", "catherine"),
+    ("jennifer", "jen", "jenny"), ("gregory", "greg"), ("jeffrey", "jeff"), ("geoffrey", "geoff"),
+    ("peter", "pete"), ("philip", "phil", "phillip"), ("nathan", "nate", "nathaniel"),
+    ("zachary", "zach"), ("joshua", "josh"), ("jonathan", "jon"), ("kenneth", "ken"),
+    ("ronald", "ron"), ("donald", "don"), ("frederick", "fred"), ("richard", "rich", "rick"),
+    ("charles", "charlie"), ("lawrence", "larry"), ("patrick", "pat"), ("susan", "sue"),
+    ("margaret", "maggie", "meg"), ("rebecca", "becky"),
+))
+
+
+def _name_tokens(name: str, *, join_hyphens: bool = False) -> tuple[list[str], list[str]]:
+    """(full tokens, initials) of a name, lower-cased and transliterated; order is not kept.
+
+    A hyphen splits by default ('Gontijo-Lopes' / 'Gontijo Lopes'); `join_hyphens` closes it instead,
+    for given names a source writes without one ('Guo-Sen' / 'Guosen').
+    """
+    text = _norm(name)
+    if join_hyphens:
+        text = re.sub(r"(?<=[a-z])[-‐](?=[a-z])", "", text)
+    tokens = [t for t in _NAME_SPLIT_RE.split(text) if t]
+    return [t for t in tokens if len(t) > 1], [t for t in tokens if len(t) == 1]
+
+
+def _token_match(a: str, b: str) -> bool:
+    """Same name token, allowing umlaut transliteration ('mueller' / 'muller'), a listed form of a
+    given name ('tim' / 'timothy') and a one-letter slip in a long token, but not a near-namesake
+    ('carreira' / 'barreira')."""
+    if a == b or _UMLAUT_RE.sub(r"\1", a) == _UMLAUT_RE.sub(r"\1", b):
+        return True
+    if any(a in forms and b in forms for forms in _GIVEN_NAME_FORMS):
+        return True
+    return min(len(a), len(b)) >= 5 and fuzz.ratio(a, b) >= 90
+
+
+def same_person(a: str, b: str) -> bool:
+    """Whether two author strings can name the same person.
+
+    Every full token of the shorter name must appear in the other (in any order); the longer name may
+    carry extra tokens (a second surname, a middle name). Each initial of the shorter name must start
+    some token of the other ('D.P. Woodruff' / 'David P. Woodruff', but not 'J. Smith' / 'Adam Smith').
+    """
+    return any(
+        _tokens_agree(_name_tokens(a, join_hyphens=j), _name_tokens(b, join_hyphens=j))
+        for j in (False, True)
+    )
+
+
+def _tokens_agree(a: tuple[list[str], list[str]], b: tuple[list[str], list[str]]) -> bool:
+    if not (a[0] or a[1]) or not (b[0] or b[1]):
+        return False
+    # An initials-only name ('A. B.') sorts first and agrees when its initials start the other's.
+    (short_full, short_init), (long_full, long_init) = sorted((a, b), key=lambda t: len(t[0]))
+    long_tokens = long_full + long_init
+    matched: set[str] = set()
+    for tok in short_full:
+        hit = next((other for other in long_full if _token_match(tok, other)), None)
+        if hit is not None:
+            matched.add(hit)
+        # a given name may be an initial on the other side ('David' / 'D.')
+        elif not (tok[0] in long_init and len(short_full) > 1):
+            return False
+    # An initial must fit the other name ('J.' does not fit 'Adam Smith'), unless every name of the
+    # other is already accounted for: then it is a dropped middle initial ('Timothy A. Mann' /
+    # 'Tim Mann').
+    unaccounted = [t for t in long_full if t not in matched]
+    return all(
+        any(t.startswith(i) for t in long_tokens) or not unaccounted for i in short_init
+    )
+
+
+def check_cited_authors(
+    bib_authors: list[str], record_author_lists: list[list[str]]
+) -> tuple[list[str], list[str]]:
+    """(missing, unchecked): cited authors who appear in none of the matched work's records, and
+    cited authors no record reaches far enough to check.
+
+    Every source record of the work is consulted, so one source's defect (S2's 'Wenhan Wang' for
+    Wenhan Yang, OpenAlex's surname-less 'Ed H.') does not make a real author look fabricated.
+
+    A record may be truncated (OpenAlex stops at 100 authors). When every record is shorter than the
+    cited list and the longest one is, in order, its leading part, the cited authors past its end are
+    `unchecked`: the record may have been cut there. When the record and the cited list disagree, the
+    list was not truncated but replaced, and every cited author is checked.
+    """
+    bib = _named(bib_authors)
+    lists = [named for lst in record_author_lists if (named := _named(lst))]
+    if not bib or not lists:
+        return [], []
+    pool = [name for lst in lists for name in lst]
+    missing = [i for i, a in enumerate(bib) if not any(same_person(a, other) for other in pool)]
+    longest = max(lists, key=len)
+    k = len(longest)
+    if missing and k < len(bib) and all(same_person(x, y) for x, y in zip(longest, bib)):
+        return [bib[i] for i in missing if i < k], [bib[i] for i in missing if i >= k]
+    return [bib[i] for i in missing], []
+
+
+def authors_missing(bib_authors: list[str], record_author_lists: list[list[str]]) -> list[str]:
+    """Cited authors who appear in none of the matched work's records (see `check_cited_authors`)."""
+    return check_cited_authors(bib_authors, record_author_lists)[0]
+
+
+def omitted_authors(bib_authors: list[str], record_authors: list[str]) -> list[str]:
+    """Authors of the record that the cited list leaves out, when it does not say it is shortened.
+
+    An `and others` / `et al.` in the cited list is an explicit truncation, so nothing is omitted.
+    A cited list as long as the record's omits nobody: a record name it does not match is spelled
+    differently ('Brandon RichardWebster', 'Misha Belkin', a record's typo), not left out. So a list
+    is partial only when it names fewer authors than the record.
+    """
+    if any(_is_etal(a) for a in bib_authors):
+        return []
+    bib, record = _named(bib_authors), _named(record_authors)
+    if not bib or len(bib) >= len(record):
+        return []
+    return [r for r in record if not any(same_person(r, b) for b in bib)]
+
+
+def authors_compatible(a: list[str], b: list[str], threshold: float = 0.8) -> bool:
+    """Whether two records' author lists can belong to one work: at least `threshold` of the
+    shorter list are people on the longer one. Unknown (an empty list) counts as compatible."""
+    a, b = _named(a), _named(b)
+    if not a or not b:
+        return True
+    short, long_ = sorted((a, b), key=len)
+    found = sum(1 for x in short if any(same_person(x, y) for y in long_))
+    return found / len(short) >= threshold