citegraph 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
citegraph/authors.py ADDED
@@ -0,0 +1,1997 @@
1
+ """Author parsing and corpus-level normalization.
2
+
3
+ The pipeline extracts author *strings* per reference, but a citation graph
4
+ is much more useful when those strings are clustered into canonical
5
+ authors so the user can answer "who is the most cited person across the
6
+ corpus?" or "show me every reference that cites Cárdenas".
7
+
8
+ Two layers live here:
9
+
10
+ 1. :func:`parse_author` turns one raw string ("Cárdenas, J.-C.",
11
+ "Adele Diamond", "García Márquez, Gabriel") into a structured
12
+ :class:`ParsedAuthor` — surname / given-names / initials / suffix —
13
+ with a diacritic-stripped, lowercased ``surname_norm`` for blocking.
14
+
15
+ 2. :func:`normalize_authors` clusters every parsed author across the
16
+ corpus into canonical author records. Parsing is evidence-driven: a
17
+ first pass over every raw string builds a lexicon of attested
18
+ multi-word surnames (comma forms, hyphenated and particle compounds,
19
+ enrichment family names), and a second pass re-parses comma-less
20
+ names against it, so "Jose Alberto Guerra Forero" adopts the
21
+ "Guerra Forero" boundary when — and only when — the corpus attests
22
+ it. The default clustering mode is precision-first: records are
23
+ bucketed by their full given-name *signature* (words + initials in
24
+ order) and a bucket joins a cluster only when its whole signature is
25
+ compatible with exactly one candidate (or a co-author overlap breaks
26
+ the tie); otherwise it is held aside as ambiguous. Word-vs-word
27
+ comparison tolerates near-identical OCR typos ("Camillo"/"Camilo")
28
+ but never prefix pairs like Gabriel/Gabriela. ``merge_mode="loose"``
29
+ collapses by ``(surname, first_initial)`` regardless of full-name
30
+ evidence — useful when you want recall over precision.
31
+
32
+ OpenAlex / ORCID identifiers (collected by the optional enrichment
33
+ stage; see ``enrich.py``) are treated as ground truth when present:
34
+ records that share an OpenAlex author id are merged regardless of
35
+ string evidence, and records with different OpenAlex ids are kept
36
+ separate even when their names look identical.
37
+
38
+ Hand-curated overrides (``aliases``) are the final escape hatch —
39
+ two-column CSV mapping arbitrary cluster ids to a canonical id, applied
40
+ after clustering so the user can fix anything the algorithm gets wrong.
41
+ """
42
+
43
+ from __future__ import annotations
44
+
45
+ import ast
46
+ import csv
47
+ import logging
48
+ import re
49
+ import unicodedata
50
+ from collections import defaultdict
51
+ from collections.abc import Iterable
52
+ from dataclasses import dataclass, field
53
+ from pathlib import Path
54
+
55
+ import pandas as pd
56
+ from rapidfuzz import fuzz
57
+ from slugify import slugify
58
+
59
+ from citegraph.io import (
60
+ AUTHOR_CITATION_COLUMNS,
61
+ AUTHOR_COLUMNS,
62
+ parse_authors_list,
63
+ require_columns,
64
+ )
65
+
66
+ logger = logging.getLogger(__name__)
67
+
68
+
69
+ # Surname particles that are conventionally lowercased and form part of
70
+ # the surname rather than the given names ("García Márquez", "de la Cruz",
71
+ # "von Neumann"). When a name is given comma-first ("Márquez, Gabriel
72
+ # García") the parser already trusts the comma; this list only matters
73
+ # when there is no comma and we have to guess where the surname ends.
74
+ _PARTICLES = frozenset(
75
+ {
76
+ "de", "del", "la", "las", "los", "da", "das", "do", "dos",
77
+ "von", "van", "der", "den", "ten", "ter", "le", "lo",
78
+ "di", "du", "el", "al", "bin", "ibn", "abu",
79
+ }
80
+ )
81
+
82
+ # Honorific suffixes that should be stripped off the surname before
83
+ # comparing — "Jr.", "III", etc. Kept on the parsed record for display
84
+ # but not part of surname_norm.
85
+ _SUFFIX_PATTERN = re.compile(
86
+ r"\b(Jr\.?|Sr\.?|II|III|IV|PhD\.?|M\.?D\.?|Esq\.?)\b",
87
+ re.IGNORECASE,
88
+ )
89
+
90
+ # A run of letters within a token (used to decide whether a token is
91
+ # "essentially an initial": every contiguous letter run is one letter).
92
+ _LETTER_RUN = re.compile(r"[A-Za-zÀ-ÖØ-öø-ÿ]+")
93
+
94
+
95
+ def _is_initial_only_token(token: str) -> bool:
96
+ """True when every alphabetic run inside ``token`` is a single letter.
97
+
98
+ Catches all the citation-style spellings of initials we see in the
99
+ wild: ``"J."``, ``"J.C."``, ``"J. C."`` (each token), ``"J.-C."``.
100
+ Multi-letter words like ``"Juan"`` correctly come back ``False``.
101
+ """
102
+ runs = _LETTER_RUN.findall(token)
103
+ return bool(runs) and all(len(r) == 1 for r in runs)
104
+
105
+ # Stuff the LLM occasionally leaves in author strings — footnote markers,
106
+ # affiliation glyphs, stray asterisks, leading "and"/"&", trailing
107
+ # punctuation. Stripped before parsing.
108
+ _NOISE = re.compile(r"[†‡§¶*★]+|^\s*(and|&)\s+", re.IGNORECASE)
109
+
110
+ # 'et al.' glued onto a real name ("B. Puranen et al.") — the bare forms
111
+ # are caught earlier by the exact-string check in parse_author.
112
+ _ET_AL_TRAILING = re.compile(r"[,\s]+et\s+al\.?\s*$", re.IGNORECASE)
113
+
114
+ # OCR-mangled accents: PDF extraction renders 'Bénabou' as "B ' enabou"
115
+ # and 'Ibáñez' as "Ib ' a ˜ nez". An apostrophe-like or floating accent
116
+ # mark with whitespace on *both* sides is never part of a real name
117
+ # (genuine spaced apostrophes — Dutch "van 't Hoff", quoted nicknames —
118
+ # touch a letter on one side), so the mark and its spaces are removed to
119
+ # rejoin the letter runs.
120
+ _OCR_SPACED_MARK = re.compile(r"\s+['’´ʼ`˜~̃]\s+")
121
+
122
+ # Bare uppercase initials run as one token — the Vancouver / NLM citation
123
+ # style writes "Cardenas JC" (surname first, initials last, no comma).
124
+ # Accented uppercase included so "Ibáñez ÁC" is recognized too.
125
+ _BARE_CAPS_INITIALS = re.compile(r"[A-ZÀ-ÖØ-Þ]{1,3}")
126
+
127
+ # Words that mark an author string as an institution rather than a person.
128
+ # Deliberately conservative: every entry is a word that essentially never
129
+ # appears in a personal name. Matched on diacritic-stripped lowercase
130
+ # tokens, and only for comma-less multi-token strings (comma forms are
131
+ # person-shaped), so "Banks, J." never trips it.
132
+ _CORPORATE_WORDS = frozenset(
133
+ {
134
+ "university", "universidad", "universidade", "universite",
135
+ "institute", "instituto", "institution", "ministry", "ministerio",
136
+ "department", "departamento", "association", "asociacion",
137
+ "bank", "banco", "council", "commission", "comision", "committee",
138
+ "organization", "organisation", "organizacion", "agency", "agencia",
139
+ "foundation", "fundacion", "fondo", "center", "centre", "centro",
140
+ "office", "oficina", "bureau", "society", "sociedad", "nations",
141
+ "congreso", "congress", "senado", "senate", "alcaldia",
142
+ "gobernacion", "secretaria", "authority", "administration",
143
+ "administracion", "programme", "observatory", "observatorio",
144
+ "consortium", "assessment", "corporation", "grupo", "world",
145
+ }
146
+ )
147
+
148
+ # "(NRC)", "(USAID)" — a parenthesised all-caps acronym is an
149
+ # organization signature regardless of the surrounding shape.
150
+ _ACRONYM_PARENS = re.compile(r"\([A-Z]{2,}\)")
151
+
152
+
153
+ def _strip_diacritics(text: str) -> str:
154
+ """NFD-decompose and drop combining marks: 'Cárdenas' -> 'Cardenas'."""
155
+ return "".join(
156
+ ch for ch in unicodedata.normalize("NFD", text)
157
+ if unicodedata.category(ch) != "Mn"
158
+ )
159
+
160
+
161
+ # Providers spell hyphenated names with typographic dashes: OpenAlex returns
162
+ # "Villegas‐Palacio" (U+2010 HYPHEN), not an ASCII "-". They are the same
163
+ # character to a reader and must be the same to the normaliser, or one person
164
+ # gets two blocking keys and no provider id ever attaches. The soft hyphen is an
165
+ # invisible line-break hint, so it is deleted rather than turned into a hyphen.
166
+ _UNICODE_DASHES = str.maketrans({
167
+ "\u2010": "-", "\u2011": "-", "\u2012": "-", "\u2013": "-",
168
+ "\u2014": "-", "\u2015": "-", "\u2212": "-", "\u00ad": "",
169
+ })
170
+
171
+
172
+ def _fold_dashes(text: str) -> str:
173
+ """Map typographic dashes onto the ASCII hyphen the parser understands."""
174
+ return text.translate(_UNICODE_DASHES)
175
+
176
+
177
+ def _norm_surname(surname: str) -> str:
178
+ """Lowercase + diacritic-strip + drop non-alphabetic characters.
179
+
180
+ Spaces are kept so multi-word surnames ('garcia marquez') can be
181
+ compared by full normalised form. The blocking step uses this as the
182
+ bucket key, so it MUST be deterministic across runs.
183
+ """
184
+ s = _strip_diacritics(_fold_dashes(surname)).lower().strip()
185
+ # Hyphens are spelling variance, not structure: 'Casas-Casas' and
186
+ # 'Casas Casas' must produce the same blocking key.
187
+ s = s.replace("-", " ")
188
+ # Keep letters and internal spaces; drop apostrophes, periods, etc.
189
+ s = re.sub(r"[^a-z\s]", "", s)
190
+ s = re.sub(r"\s+", " ", s).strip()
191
+ return s
192
+
193
+
194
+ def _initials_from_token(token: str) -> str:
195
+ """Extract initial letters from a given-name token.
196
+
197
+ Handles every common spelling of multi-initial citations:
198
+ ``'J.'`` -> ``'J'`` ; ``'J.C.'`` -> ``'JC'`` ; ``'J.-C.'`` -> ``'JC'`` ;
199
+ ``'Juan-Camilo'`` -> ``'JC'`` ; ``'Juan'`` -> ``'J'`` ; ``''`` -> ``''``.
200
+
201
+ The implementation walks every contiguous letter run and takes its
202
+ first letter — so it doesn't matter whether the segments are joined
203
+ by periods, hyphens, or whitespace.
204
+ """
205
+ if not token:
206
+ return ""
207
+ return "".join(run[0].upper() for run in _LETTER_RUN.findall(token))
208
+
209
+
210
+ @dataclass(frozen=True)
211
+ class ParsedAuthor:
212
+ """Structured representation of a single author string.
213
+
214
+ Attributes
215
+ ----------
216
+ raw:
217
+ The original string as it appeared in the citation, untouched.
218
+ surname:
219
+ The surname with original casing and diacritics, used for display.
220
+ surname_norm:
221
+ Diacritic-stripped, lowercased, alphabetic-only surname. The
222
+ clustering step blocks on this.
223
+ given_names:
224
+ List of given-name tokens in citation order ("Juan", "Camilo")
225
+ or initials ("J.", "C."). Empty when only a surname was given.
226
+ initials:
227
+ Concatenated first letters of every given-name token, uppercased.
228
+ "Juan-Camilo" -> "JC"; "J. C." -> "JC".
229
+ has_full_first:
230
+ ``True`` if the first given-name token is a real word, not just
231
+ an initial. The clustering step uses this to anchor clusters.
232
+ suffix:
233
+ "Jr.", "III", etc. Preserved for display; ignored by matching.
234
+ is_corporate:
235
+ ``True`` when the string names an institution ("The World Bank")
236
+ rather than a person. The whole string is kept as the surname and
237
+ such records only ever cluster with identical normalized strings.
238
+ """
239
+
240
+ raw: str
241
+ surname: str
242
+ surname_norm: str
243
+ given_names: tuple[str, ...]
244
+ initials: str
245
+ has_full_first: bool
246
+ suffix: str | None = None
247
+ is_corporate: bool = False
248
+
249
+ @property
250
+ def first_initial(self) -> str:
251
+ return self.initials[:1]
252
+
253
+ @property
254
+ def first_given_norm(self) -> str:
255
+ """Normalised form of the first given-name token, '' if only initial."""
256
+ if not self.has_full_first or not self.given_names:
257
+ return ""
258
+ return _strip_diacritics(self.given_names[0]).lower().strip(".-")
259
+
260
+ def display(self) -> str:
261
+ """Best human-readable form of this name (surname-first)."""
262
+ if self.given_names:
263
+ given = " ".join(self.given_names)
264
+ return f"{self.surname}, {given}"
265
+ return self.surname
266
+
267
+
268
+ def parse_author(
269
+ raw: str,
270
+ *,
271
+ known_surnames: Iterable[str] | None = None,
272
+ ) -> ParsedAuthor | None:
273
+ """Parse a single raw author string into a :class:`ParsedAuthor`.
274
+
275
+ Returns ``None`` for strings that contain no usable surname (empty
276
+ string, pure punctuation, "et al.", "Anonymous", etc.). Callers
277
+ should treat ``None`` as "skip this author".
278
+
279
+ ``known_surnames`` is an optional set of *normalized* multi-word
280
+ surnames already attested elsewhere (comma forms, hyphenated forms,
281
+ enrichment family names). When given, comma-less names whose token
282
+ suffix matches an attested compound surname are split at that
283
+ boundary instead of the conservative last-token guess — this is how
284
+ "Jose Alberto Guerra Forero" learns its surname is "Guerra Forero"
285
+ from a "Guerra Forero, J.A." citation elsewhere in the corpus.
286
+ """
287
+ if not isinstance(raw, str):
288
+ return None
289
+ # Fold dashes before anything inspects the string: the compound-surname
290
+ # checks test for a literal "-", so a typographic one would read as an
291
+ # unhyphenated name. ``raw`` keeps the original spelling for display.
292
+ s = _fold_dashes(raw).strip()
293
+ if not s:
294
+ return None
295
+
296
+ # Drop common noise tokens that signal "not an author".
297
+ bare = _strip_diacritics(s).lower()
298
+ if bare in {"et al", "et al.", "anonymous", "anon", "n.a.", "na", "unknown"}:
299
+ return None
300
+
301
+ # Strip footnote/affiliation glyphs and a leading 'and'/'&'.
302
+ s = _NOISE.sub(" ", s).strip()
303
+ # Rejoin OCR-mangled accents ("B ' enabou" -> "Benabou") before any
304
+ # tokenization; a spaced mark would otherwise split the surname.
305
+ s = _OCR_SPACED_MARK.sub("", s)
306
+ # 'et al.' glued onto a real name keeps the name, drops the tail.
307
+ s = _ET_AL_TRAILING.sub("", s)
308
+ s = re.sub(r"\s+", " ", s).strip(" ,")
309
+ if not s:
310
+ return None
311
+
312
+ # Institutions are not person-parsed: the whole string is the name.
313
+ if _is_corporate_author(s):
314
+ return ParsedAuthor(
315
+ raw=raw,
316
+ surname=s,
317
+ surname_norm=_norm_surname(s),
318
+ given_names=(),
319
+ initials="",
320
+ has_full_first=False,
321
+ suffix=None,
322
+ is_corporate=True,
323
+ )
324
+
325
+ # For persons, a parenthetical is an affiliation ("Smith, J. (MIT)"),
326
+ # never part of the name.
327
+ s = re.sub(r"\s*\([^)]*\)", " ", s)
328
+ s = re.sub(r"\s+", " ", s).strip(" ,")
329
+ if not s:
330
+ return None
331
+
332
+ # Pull off honorific suffix if present.
333
+ suffix: str | None = None
334
+ m = _SUFFIX_PATTERN.search(s)
335
+ if m:
336
+ suffix = m.group(1)
337
+ s = (s[: m.start()] + s[m.end():]).strip(" ,")
338
+
339
+ # Two parsing branches: comma-style (Surname, Given) vs space-style.
340
+ if "," in s:
341
+ surname_part, given_part = s.split(",", 1)
342
+ surname = surname_part.strip()
343
+ given_str = given_part.strip()
344
+ else:
345
+ reversed_split = _split_reversed_citation(s.split())
346
+ if reversed_split is not None:
347
+ # Vancouver / NLM style: "Guerra JA" is surname-then-initials.
348
+ surname, given_str = reversed_split
349
+ else:
350
+ surname, given_str = _split_no_comma(s, known_surnames)
351
+
352
+ surname = surname.strip(" .-")
353
+ if not surname or not re.search(r"[A-Za-zÀ-ÖØ-öø-ÿ]", surname):
354
+ return None
355
+ # A "surname" whose every letter run is a single letter is almost
356
+ # certainly a misparsed initial ("J.", "X. Y."). Real surnames have
357
+ # at least one multi-letter run. Rejecting here prevents the mega-
358
+ # clusters of one-letter surnames that arise when an upstream stage
359
+ # has comma-split "Smith, J., García, A." into separate fragments.
360
+ if _is_initial_only_token(surname):
361
+ return None
362
+
363
+ given_tokens = _tokenize_given(given_str)
364
+ initials = "".join(_initials_from_token(t) for t in given_tokens)
365
+ has_full_first = bool(given_tokens) and not _is_initial_only_token(given_tokens[0])
366
+
367
+ return ParsedAuthor(
368
+ raw=raw,
369
+ surname=surname,
370
+ surname_norm=_norm_surname(surname),
371
+ given_names=tuple(given_tokens),
372
+ initials=initials,
373
+ has_full_first=has_full_first,
374
+ suffix=suffix,
375
+ )
376
+
377
+
378
+ def _split_no_comma(
379
+ s: str,
380
+ known_surnames: Iterable[str] | None = None,
381
+ ) -> tuple[str, str]:
382
+ """Split a name with no comma into (surname, given-string).
383
+
384
+ Heuristic: the last token is the surname, *unless* the second-to-last
385
+ token is a particle ('de', 'van', 'García' before 'Márquez'…), in
386
+ which case the surname extends backwards to include it. Compound
387
+ surnames without particles ('García Márquez' alone, no comma) are
388
+ ambiguous — we can't tell from the string alone, and we conservatively
389
+ take only the last token as the surname *unless* ``known_surnames``
390
+ attests the compound (see :func:`parse_author`).
391
+ """
392
+ tokens = s.split()
393
+ if not tokens:
394
+ return "", ""
395
+ if len(tokens) == 1:
396
+ return tokens[0], ""
397
+
398
+ # Corpus-evidence pass: adopt a multi-token surname the corpus has
399
+ # already attested. Longest suffix wins; a whole-string match means
400
+ # the string is a bare compound surname with no given names.
401
+ if known_surnames:
402
+ for k in range(min(4, len(tokens)), 1, -1):
403
+ cand = _norm_surname(" ".join(tokens[-k:]))
404
+ if cand in known_surnames:
405
+ if k == len(tokens):
406
+ return s, ""
407
+ return " ".join(tokens[-k:]), " ".join(tokens[:-k])
408
+
409
+ surname_idx = len(tokens) - 1
410
+ # Walk backwards while the preceding token looks like a particle.
411
+ while surname_idx > 0:
412
+ prev_raw = tokens[surname_idx - 1]
413
+ prev = prev_raw.strip(".-").lower()
414
+ if prev in _PARTICLES:
415
+ surname_idx -= 1
416
+ elif (
417
+ prev_raw == "y"
418
+ and surname_idx >= 2
419
+ and not _is_initial_only_token(tokens[surname_idx - 2])
420
+ ):
421
+ # Spanish surname connector: 'Ortega y Gasset'. Only the bare
422
+ # lowercase word counts — a 'Y.' initial must not trigger it.
423
+ surname_idx -= 2
424
+ else:
425
+ break
426
+
427
+ surname = " ".join(tokens[surname_idx:])
428
+ given = " ".join(tokens[:surname_idx])
429
+ return surname, given
430
+
431
+
432
+ def _split_reversed_citation(tokens: list[str]) -> tuple[str, str] | None:
433
+ """Detect Vancouver-style 'Surname(s) AB' ordering; ``None`` if absent.
434
+
435
+ The trailing run of initials-like tokens ("JA", "J.A.", "J") becomes
436
+ the given part and everything before it the surname — but only when
437
+ no leading token is a single-letter initial, so "A. Diamond"
438
+ (given-first) is left alone. All-caps leading tokens are fine ("LEE
439
+ KY", "CAMERON L"). Bare caps runs are expanded ("JA" -> "J. A.") so
440
+ the initials extraction sees one letter per run.
441
+ """
442
+
443
+ def is_initials_like(t: str) -> bool:
444
+ return _is_initial_only_token(t) or bool(_BARE_CAPS_INITIALS.fullmatch(t))
445
+
446
+ # Consume the trailing initials run, but never the first token — in
447
+ # "LI X" the all-caps "LI" is the surname, not more initials.
448
+ i = len(tokens)
449
+ while i > 1 and is_initials_like(tokens[i - 1]):
450
+ i -= 1
451
+ if i == len(tokens):
452
+ return None
453
+ leading = tokens[:i]
454
+ if any(_is_initial_only_token(t) for t in leading):
455
+ return None
456
+ trailing = tokens[i:]
457
+ trailing_letters = sum(len(run) for t in trailing for run in _LETTER_RUN.findall(t))
458
+ if len(leading) == 1 and len(leading[0]) <= 2 and trailing_letters >= 2:
459
+ # 'Bo XU' vs 'Ma JA' is genuinely ambiguous (caps surname vs
460
+ # Vancouver initials); keep the conservative last-token parse.
461
+ return None
462
+ given_tokens = [
463
+ " ".join(f"{c}." for c in t) if _BARE_CAPS_INITIALS.fullmatch(t) else t
464
+ for t in trailing
465
+ ]
466
+ return " ".join(leading), " ".join(given_tokens)
467
+
468
+
469
+ def _is_corporate_author(s: str) -> bool:
470
+ """True when the string names an institution rather than a person."""
471
+ if "," in s:
472
+ # Comma forms are person-shaped ("Banks, J.", "Smith, J. (MIT)").
473
+ return False
474
+ if _ACRONYM_PARENS.search(s):
475
+ return True
476
+ tokens = re.findall(r"[^\W\d_]+", _strip_diacritics(s).lower())
477
+ if len(tokens) < 2:
478
+ return False
479
+ if not any(t in _CORPORATE_WORDS for t in tokens):
480
+ return False
481
+ # Two-token strings only count when the corporate word leads
482
+ # ("Fundación Natura", "World Bank") — a trailing corporate word may
483
+ # be a real surname ("Steven Bank").
484
+ return len(tokens) >= 3 or tokens[0] in _CORPORATE_WORDS
485
+
486
+
487
+ # Below this length a token is too short to absorb a typo without pairing
488
+ # unrelated people ("Lee"/"Loe"), so it must match exactly.
489
+ _AGREEMENT_FUZZ_MIN_LEN = 5
490
+
491
+ # Providers spell apostrophes typographically ("D’Adda") where
492
+ # bibliographies use ASCII ("D'Adda"); both must fold to one token, the same
493
+ # way :func:`_fold_dashes` handles the unicode dashes OpenAlex emits. The
494
+ # OCR-spaced form ("Ca ' rdenas") folds here too.
495
+ _APOSTROPHES = ("'", "’", "‘", "ʼ", "´", "`")
496
+
497
+
498
+ def author_list_agreement(
499
+ extracted: list[str],
500
+ provider: list[str],
501
+ *,
502
+ fuzz_threshold: float = 85.0,
503
+ ) -> bool | None:
504
+ """Do two author lists name any of the same people?
505
+
506
+ ``True`` when at least one name pairs, ``False`` when both sides carry
507
+ usable person names and none pair, ``None`` to abstain.
508
+
509
+ Comparison is over order-free name tokens rather than
510
+ :attr:`ParsedAuthor.surname_norm`, because the surname path needs the
511
+ corpus-wide compound-surname lexicon and that does not exist yet when
512
+ enrichment runs. Without it every comma-less provider name collapses to
513
+ its last token, so "Moros, L." and "Lina Moros Canon" stop matching.
514
+ Tokens pair on equality, or on ``fuzz_threshold`` similarity when long
515
+ enough to survive it, which is what lets a bibliography's
516
+ "Hirschmann, A." reach the provider's "Albert O. Hirschman".
517
+
518
+ Abstains whenever either side names an institution, since a provider
519
+ record for a corporate-authored report legitimately lists the human
520
+ chapter authors instead ("IFPRI" -> "Fredrick O. Wanyama").
521
+ """
522
+ left = _agreement_tokens(extracted)
523
+ right = _agreement_tokens(provider)
524
+ if left is None or right is None:
525
+ return None
526
+ for a in left:
527
+ for b in right:
528
+ if a == b:
529
+ return True
530
+ if (
531
+ len(a) >= _AGREEMENT_FUZZ_MIN_LEN
532
+ and len(b) >= _AGREEMENT_FUZZ_MIN_LEN
533
+ and fuzz.ratio(a, b) >= fuzz_threshold
534
+ ):
535
+ return True
536
+ return False
537
+
538
+
539
+ def _is_acronym_author(s: str) -> bool:
540
+ """True for a bare all-caps acronym ("IFPRI", "UNFCCC", "IPCC").
541
+
542
+ :func:`_is_corporate_author` needs two tokens to fire, so acronyms slip
543
+ past it. They are institutions all the same.
544
+ """
545
+ stripped = s.strip()
546
+ return (
547
+ len(stripped) >= 3
548
+ and stripped.isupper()
549
+ and stripped.isalpha()
550
+ and " " not in stripped
551
+ )
552
+
553
+
554
+ def _agreement_tokens(names: list[str]) -> set[str] | None:
555
+ """Order-free name tokens, or ``None`` when there is nothing to compare."""
556
+ tokens: set[str] = set()
557
+ for raw in names:
558
+ if not isinstance(raw, str) or not raw.strip():
559
+ continue
560
+ if _is_corporate_author(raw) or _is_acronym_author(raw):
561
+ return None
562
+ cleaned = _strip_diacritics(_fold_dashes(raw))
563
+ for apostrophe in _APOSTROPHES:
564
+ cleaned = cleaned.replace(apostrophe, "")
565
+ for token in re.split(r"[^A-Za-z]+", cleaned):
566
+ lowered = token.lower()
567
+ if len(token) > 2 and lowered not in _PARTICLES:
568
+ tokens.add(lowered)
569
+ return tokens or None
570
+
571
+
572
+ def _tokenize_given(given_str: str) -> list[str]:
573
+ """Split a given-name string into tokens, preserving 'J.-C.' as one."""
574
+ if not given_str:
575
+ return []
576
+ # Split on whitespace; keep hyphenated initials intact.
577
+ parts = [p for p in given_str.split() if p]
578
+ return parts
579
+
580
+
581
+ # ---------------------------------------------------------------------------
582
+ # Clustering
583
+ # ---------------------------------------------------------------------------
584
+
585
+
586
+ @dataclass
587
+ class AuthorOccurrence:
588
+ """One concrete appearance of an author on a canonical work.
589
+
590
+ The clustering output is grouped by canonical author id; each
591
+ occurrence keeps the work id so callers can join back onto
592
+ ``works.csv`` (for rings) and ``citation_graph.csv`` (for citers).
593
+ """
594
+
595
+ parsed: ParsedAuthor
596
+ record_id: str # 'w-…' canonical work id
597
+ position: int # 0-based index in the author list
598
+ co_author_keys: tuple[str, ...] = () # surname_norm of the other authors on the same record
599
+ openalex_id: str | None = None
600
+ orcid: str | None = None
601
+ cannot_link: set[tuple[str, int]] = field(default_factory=set)
602
+ ambiguous_identity_links: set[tuple[str, int]] = field(default_factory=set)
603
+
604
+
605
+ @dataclass
606
+ class AuthorClusterConfig:
607
+ """Tunable knobs for :func:`normalize_authors`."""
608
+
609
+ # 'strict' (default) — precision-first. Merges 'Diamond, A.' into
610
+ # 'Diamond, Adele' only when 'Adele' is the *only* full first name
611
+ # observed for surname 'Diamond' in this corpus. Otherwise the
612
+ # initial-only record is held aside in its own cluster and flagged.
613
+ #
614
+ # 'loose' — recall-first. Collapses every (surname, first_initial)
615
+ # combination into one cluster regardless of full-name disagreement.
616
+ merge_mode: str = "strict"
617
+
618
+
619
+ @dataclass
620
+ class AuthorCluster:
621
+ """One canonical author after corpus-wide normalization."""
622
+
623
+ id: str
624
+ display_name: str
625
+ surname: str
626
+ surname_norm: str
627
+ canonical_given: str
628
+ initials: str
629
+ openalex_id: str | None
630
+ orcid: str | None
631
+ occurrences: list[AuthorOccurrence] = field(default_factory=list)
632
+ review_reason: str | None = None # populated for low-confidence clusters
633
+
634
+ @property
635
+ def n_occurrences(self) -> int:
636
+ return len(self.occurrences)
637
+
638
+
639
+ def _make_author_id(surname_norm: str, given_token: str, n: int = 0) -> str:
640
+ """Stable slug-style id for an author cluster.
641
+
642
+ ``n`` is appended only to disambiguate within a corpus when the same
643
+ (surname, given-names) tuple genuinely refers to two distinct people
644
+ that the algorithm split apart. Corporate authors have no given
645
+ token and slug to just the normalized name.
646
+ """
647
+ base = f"a-{slugify(surname_norm) or 'unknown'}"
648
+ given_slug = slugify(given_token) if given_token else ""
649
+ if given_slug:
650
+ base = f"{base}-{given_slug}"
651
+ if n:
652
+ base = f"{base}-{n}"
653
+ return base
654
+
655
+
656
+ def _given_units(parsed: ParsedAuthor) -> tuple[tuple[str, str], ...]:
657
+ """Flatten given names into comparable units.
658
+
659
+ Each contiguous letter run becomes one unit: a full word ``("w",
660
+ "juan")`` or an initial ``("i", "j")``. Hyphens and periods dissolve,
661
+ so "Juan-Camilo", "Juan Camilo", and "J.C." become comparable:
662
+ ``[w:juan, w:camilo]`` / ``[w:juan, w:camilo]`` / ``[i:j, i:c]``.
663
+ """
664
+ units: list[tuple[str, str]] = []
665
+ for tok in parsed.given_names:
666
+ for run in _LETTER_RUN.findall(tok):
667
+ kind = "i" if len(run) == 1 else "w"
668
+ units.append((kind, _strip_diacritics(run).lower()))
669
+ return tuple(units)
670
+
671
+
672
+ def _w_equivalent(a: str, b: str) -> bool:
673
+ """True when two full given-name words plausibly name the same person.
674
+
675
+ Exact match, or a near-identical spelling that reads as an OCR /
676
+ extraction typo ('camilo' vs 'camillo'): both words at least 4
677
+ letters, rapidfuzz ratio >= 90, and *neither a prefix of the other* —
678
+ the prefix guard is what keeps gendered pairs like Gabriel/Gabriela
679
+ and Daniel/Daniela (ratio > 90) firmly apart.
680
+ """
681
+ if a == b:
682
+ return True
683
+ if len(a) < 4 or len(b) < 4:
684
+ return False
685
+ if a.startswith(b) or b.startswith(a):
686
+ return False
687
+ return fuzz.ratio(a, b) >= 90
688
+
689
+
690
+ def _units_compatible(
691
+ a: tuple[tuple[str, str], ...],
692
+ b: tuple[tuple[str, str], ...],
693
+ ) -> bool:
694
+ """True when two given-name unit sequences could name the same person.
695
+
696
+ Position-by-position over the shared prefix: word vs word must be
697
+ equivalent (exact or typo-close, see :func:`_w_equivalent`), initial
698
+ vs initial must be equal, and an initial matches a word starting
699
+ with it. The *whole* overlapping sequence must agree — this is what
700
+ keeps "J.P." out of a "Juan Camilo" cluster even though the first
701
+ initials match.
702
+ """
703
+ # strict=False is the point: only the shared prefix is compared.
704
+ for (ka, va), (kb, vb) in zip(a, b, strict=False):
705
+ if ka == kb:
706
+ if ka == "w":
707
+ if not _w_equivalent(va, vb):
708
+ return False
709
+ elif va != vb:
710
+ return False
711
+ else:
712
+ word = va if ka == "w" else vb
713
+ initial = vb if ka == "w" else va
714
+ if not word.startswith(initial):
715
+ return False
716
+ return True
717
+
718
+
719
+ def _units_informativeness(units: tuple[tuple[str, str], ...]) -> tuple[int, int]:
720
+ return (sum(1 for kind, _ in units if kind == "w"), len(units))
721
+
722
+
723
+ def _cluster_best_units(
724
+ cluster: list[AuthorOccurrence],
725
+ ) -> tuple[tuple[str, str], ...]:
726
+ """The most informative given-name signature observed in a cluster."""
727
+ best: tuple[tuple[str, str], ...] = ()
728
+ best_key = (-1, -1)
729
+ for o in cluster:
730
+ units = _given_units(o.parsed)
731
+ key = _units_informativeness(units)
732
+ if key > best_key:
733
+ best, best_key = units, key
734
+ return best
735
+
736
+
737
+ def _canonical_given_for_cluster(occs: list[AuthorOccurrence]) -> tuple[str, str]:
738
+ """Pick a (display-friendly given string, slug-friendly token) pair.
739
+
740
+ Prefer the fullest given-name *sequence* actually seen in the cluster
741
+ ("Jose Alberto", not just "Jose"), so display names never drop
742
+ observed middle names. If the cluster only has initials, use the
743
+ canonical initial sequence. Corporate clusters return empty strings.
744
+ """
745
+ frequency: dict[str, int] = defaultdict(int)
746
+ for o in occs:
747
+ frequency[" ".join(o.parsed.given_names)] += 1
748
+
749
+ best_key: tuple[int, int, int, int, str] | None = None
750
+ best_units: tuple[tuple[str, str], ...] = ()
751
+ best_given = ""
752
+ for o in occs:
753
+ if o.parsed.is_corporate:
754
+ continue
755
+ units = _given_units(o.parsed)
756
+ n_words = sum(1 for kind, _ in units if kind == "w")
757
+ if n_words == 0:
758
+ continue
759
+ given_str = " ".join(o.parsed.given_names)
760
+ # Fullest sequence first; among equally full ones, the spelling
761
+ # seen most often wins so a lone typo never names the cluster.
762
+ key = (n_words, len(units), frequency[given_str], len(given_str), given_str)
763
+ if best_key is None or key > best_key:
764
+ best_key, best_units, best_given = key, units, given_str
765
+ if best_given:
766
+ token = " ".join(v for kind, v in best_units if kind == "w")
767
+ return best_given, token
768
+ if any(o.parsed.is_corporate for o in occs):
769
+ return "", ""
770
+ # Fallback to the longest initial sequence we saw.
771
+ inits = max((o.parsed.initials for o in occs), key=len, default="")
772
+ return inits, inits.lower()
773
+
774
+
775
+ def _has_coauthor_overlap(
776
+ occ: AuthorOccurrence,
777
+ cluster: list[AuthorOccurrence],
778
+ ) -> bool:
779
+ """True if ``occ`` shares a co-author surname with any cluster member."""
780
+ if not occ.co_author_keys:
781
+ return False
782
+ cluster_keys: set[str] = set()
783
+ for c in cluster:
784
+ cluster_keys.update(c.co_author_keys)
785
+ return bool(set(occ.co_author_keys) & cluster_keys)
786
+
787
+
788
+ def _has_external_id(cluster: list[AuthorOccurrence]) -> bool:
789
+ return any(o.openalex_id or o.orcid for o in cluster)
790
+
791
+
792
+ def _cluster_signatures(
793
+ cluster: list[AuthorOccurrence],
794
+ ) -> set[tuple[tuple[str, str], ...]]:
795
+ """Every distinct non-empty given-name signature observed in a cluster."""
796
+ return {
797
+ units
798
+ for o in cluster
799
+ if (units := _given_units(o.parsed))
800
+ }
801
+
802
+
803
+ def _author_conflict_reasons(left: list[AuthorOccurrence], right: list[AuthorOccurrence]) -> list[str]:
804
+ """Contradictions are independent of positive matching evidence."""
805
+ if any((r.record_id, r.position) in member.cannot_link for member in left for r in right):
806
+ return ["explicit_separate"]
807
+ if any((r.record_id, r.position) in member.ambiguous_identity_links for member in left for r in right):
808
+ return ["ambiguous_external_identity"]
809
+ lo = {o.orcid for o in left if o.orcid}
810
+ ro = {o.orcid for o in right if o.orcid}
811
+ if lo and ro and len(lo | ro) > 1:
812
+ return ["orcid_conflict"]
813
+ if any(a.parsed.is_corporate != b.parsed.is_corporate for a in left for b in right):
814
+ return ["corporate_person_conflict"]
815
+ li, ri = _cluster_external_ids(left), _cluster_external_ids(right)
816
+ if li & ri:
817
+ return [] # Consistent direct identity supports observed name variants.
818
+ reasons = []
819
+ if li and ri:
820
+ reasons.append("distinct_external_ids")
821
+ if {o.record_id for o in left} & {o.record_id for o in right}:
822
+ reasons.append("same_work_distinct_positions")
823
+ for left_occurrence in left:
824
+ for right_occurrence in right:
825
+ lu, ru = _given_units(left_occurrence.parsed), _given_units(right_occurrence.parsed)
826
+ if lu and ru and not (_units_compatible(lu, ru) or _given_subsequence(lu, ru)):
827
+ reasons.append("given_name_conflict")
828
+ return sorted(set(reasons))
829
+
830
+
831
+ def _author_clusters_conflict(left: list[AuthorOccurrence], right: list[AuthorOccurrence]) -> bool:
832
+ return bool(_author_conflict_reasons(left, right))
833
+
834
+
835
+ def _name_supported(left: list[AuthorOccurrence], right: list[AuthorOccurrence]) -> bool:
836
+ for left_occurrence in left:
837
+ for right_occurrence in right:
838
+ if left_occurrence.parsed.surname_norm != right_occurrence.parsed.surname_norm:
839
+ continue
840
+ if left_occurrence.parsed.is_corporate and right_occurrence.parsed.is_corporate:
841
+ return True
842
+ lu, ru = _given_units(left_occurrence.parsed), _given_units(right_occurrence.parsed)
843
+ if lu and ru and _units_compatible(lu, ru) and (
844
+ lu == ru or any(k == "w" for k, _ in lu + ru)
845
+ ):
846
+ return True
847
+ return False
848
+
849
+
850
+ def _merge_unidentified_into_external_clusters(
851
+ clusters: list[list[AuthorOccurrence]],
852
+ ) -> list[list[AuthorOccurrence]]:
853
+ external = [c for c in clusters if _has_external_id(c)]
854
+ unresolved = []
855
+ for cluster in clusters:
856
+ if _has_external_id(cluster):
857
+ continue
858
+ # Coauthor evidence belongs to each occurrence, never its signature bucket.
859
+ for occurrence in cluster:
860
+ candidates = [c for c in external if _name_supported([occurrence], c)
861
+ and not _author_clusters_conflict([occurrence], c)]
862
+ if len(candidates) > 1:
863
+ candidates = [c for c in candidates if _has_coauthor_overlap(occurrence, c)]
864
+ if len(candidates) == 1:
865
+ candidates[0].append(occurrence)
866
+ else:
867
+ unresolved.append([occurrence])
868
+ return external + unresolved
869
+
870
+
871
+ def _cluster_block(occs: list[AuthorOccurrence], cfg: AuthorClusterConfig,
872
+ seeds: list[list[AuthorOccurrence]] | None = None) -> list[list[AuthorOccurrence]]:
873
+ """Resolve external identities globally before occurrence-specific name joins."""
874
+ seeds = seeds if seeds is not None else [[o] for o in occs]
875
+ external = _merge_clusters_by_external_id([c for c in seeds if len(c) > 1 or _has_external_id(c)])
876
+ unidentified = [c[0] for c in seeds if len(c) == 1 and not _has_external_id(c)]
877
+ # Full evidence comes first; stable tie ordering preserves deterministic IDs.
878
+ unidentified.sort(key=lambda o: (
879
+ tuple(-x for x in _units_informativeness(_given_units(o.parsed))),
880
+ " ".join(v for _, v in _given_units(o.parsed)), o.record_id, o.position,
881
+ ))
882
+ clusters = external[:]
883
+ surname_candidates: dict[str, list[list[AuthorOccurrence]]] = defaultdict(list)
884
+ def index_cluster(cluster):
885
+ for surname in {o.parsed.surname_norm for o in cluster}:
886
+ surname_candidates[surname].append(cluster)
887
+ for cluster in clusters:
888
+ index_cluster(cluster)
889
+ held: list[list[AuthorOccurrence]] = []
890
+ for occurrence in unidentified:
891
+ one = [occurrence]
892
+ if cfg.merge_mode == "loose":
893
+ candidates = [c for c in surname_candidates[occurrence.parsed.surname_norm] if c[0].parsed.surname_norm == occurrence.parsed.surname_norm
894
+ and c[0].parsed.first_initial == occurrence.parsed.first_initial
895
+ and not _author_clusters_conflict(one, c)]
896
+ else:
897
+ candidates = [c for c in surname_candidates[occurrence.parsed.surname_norm] if _name_supported(one, c)
898
+ and not _author_clusters_conflict(one, c)]
899
+ if len(candidates) > 1:
900
+ overlap = [c for c in candidates if _has_coauthor_overlap(occurrence, c)]
901
+ if len(overlap) == 1:
902
+ candidates = overlap
903
+ if len(candidates) == 1:
904
+ candidates[0].append(occurrence)
905
+ elif candidates:
906
+ held.append(one)
907
+ else:
908
+ clusters.append(one)
909
+ index_cluster(one)
910
+ # Identical unresolved signatures can combine across works, without making
911
+ # an ambiguous initial cluster an anchor for other abbreviated signatures.
912
+ for one in held:
913
+ matches = [c for c in surname_candidates[one[0].parsed.surname_norm] if not _has_external_id(c)
914
+ and c[0].parsed.surname_norm == one[0].parsed.surname_norm
915
+ and _given_units(c[0].parsed) == _given_units(one[0].parsed)
916
+ and not _author_clusters_conflict(one, c)]
917
+ if len(matches) == 1:
918
+ matches[0].extend(one)
919
+ else:
920
+ clusters.append(one)
921
+ index_cluster(one)
922
+ return clusters
923
+
924
+
925
+ def _cluster_external_ids(cluster: list[AuthorOccurrence]) -> set[str]:
926
+ return {kind + value for o in cluster
927
+ for kind, value in (("oa:", o.openalex_id), ("orcid:", o.orcid)) if value}
928
+
929
+
930
+ def _mark_identity_ambiguities(clusters: list[list[AuthorOccurrence]]) -> None:
931
+ """Find contradictory identity components before any order-dependent unions.
932
+
933
+ A seed compatible with both sides of a contradiction cannot choose either.
934
+ Keep that evidence unresolved across every later name/identity merge pass.
935
+ Explicitly forced groups are already seeds, so reviewed choices remain valid.
936
+ """
937
+ parent = list(range(len(clusters)))
938
+ def root(i):
939
+ while parent[i] != i:
940
+ parent[i] = parent[parent[i]]
941
+ i = parent[i]
942
+ return i
943
+ owner: dict[str, int] = {}
944
+ for i, cluster in enumerate(clusters):
945
+ for key in _cluster_external_ids(cluster):
946
+ if key in owner:
947
+ parent[root(i)] = root(owner[key])
948
+ else:
949
+ owner[key] = i
950
+ components: dict[int, list[int]] = defaultdict(list)
951
+ for i in range(len(clusters)):
952
+ components[root(i)].append(i)
953
+ blocked: set[tuple[int, int]] = set()
954
+ for indices in components.values():
955
+ conflicts = [(i, j) for pos, i in enumerate(indices) for j in indices[pos + 1:]
956
+ if {"orcid_conflict", "explicit_separate"}
957
+ & set(_author_conflict_reasons(clusters[i], clusters[j]))]
958
+ for left, right in conflicts:
959
+ for i in indices:
960
+ if i in (left, right):
961
+ continue
962
+ # Membership of this component already supplies transitive ID
963
+ # evidence. Distinct direct IDs or name spellings are not hard
964
+ # contradictions to an anchor elsewhere in the component.
965
+ hard_conflicts = {"orcid_conflict", "explicit_separate", "corporate_person_conflict"}
966
+ if (not hard_conflicts.intersection(_author_conflict_reasons(clusters[i], clusters[left]))
967
+ and not hard_conflicts.intersection(_author_conflict_reasons(clusters[i], clusters[right]))):
968
+ blocked.update(((i, left), (i, right)))
969
+ for left, right in blocked:
970
+ for own, other in ((left, right), (right, left)):
971
+ coordinates = {(o.record_id, o.position) for o in clusters[other]}
972
+ for occurrence in clusters[own]:
973
+ occurrence.ambiguous_identity_links.update(coordinates)
974
+
975
+
976
+ def _merge_clusters_by_external_id(clusters: list[list[AuthorOccurrence]]) -> list[list[AuthorOccurrence]]:
977
+ """Union direct identities only while every known ORCID remains consistent."""
978
+ result: list[list[AuthorOccurrence]] = []
979
+ for cluster in clusters:
980
+ targets = [c for c in result if _cluster_external_ids(c) & _cluster_external_ids(cluster)
981
+ and not _author_clusters_conflict(c, cluster)]
982
+ # An ORCID-free bridge shared by conflicting identities cannot choose one.
983
+ if len({o.orcid for c in targets + [cluster] for o in c if o.orcid}) > 1:
984
+ targets = []
985
+ if any('explicit_separate' in _author_conflict_reasons(a, b)
986
+ for i, a in enumerate(targets) for b in targets[i + 1:]):
987
+ targets = []
988
+ if targets:
989
+ primary = targets[0]
990
+ primary.extend(cluster)
991
+ for other in targets[1:]:
992
+ primary.extend(other)
993
+ result = [c for c in result if c is not other]
994
+ else:
995
+ result.append(cluster)
996
+ return result
997
+
998
+
999
+ def _cluster_surname(occs: list[AuthorOccurrence]) -> tuple[str, str]:
1000
+ """Order-independent (surname_norm, display surname) for a cluster.
1001
+
1002
+ The most specific (most words), then most frequent, normalized
1003
+ surname wins — so a cluster merged across blocks by external ids or
1004
+ bridging is named by its compound form no matter which occurrence
1005
+ happened to land first. Display casing follows the chosen norm,
1006
+ preferring properly-cased spellings, then the most frequent one.
1007
+ """
1008
+ norm_counts: dict[str, int] = defaultdict(int)
1009
+ for o in occs:
1010
+ norm_counts[o.parsed.surname_norm] += 1
1011
+ surname_norm = max(
1012
+ norm_counts,
1013
+ key=lambda n: (len(n.split()), norm_counts[n], n),
1014
+ )
1015
+ display_counts: dict[str, int] = defaultdict(int)
1016
+ for o in occs:
1017
+ if o.parsed.surname_norm == surname_norm:
1018
+ display_counts[o.parsed.surname] += 1
1019
+ surname_display = max(
1020
+ display_counts,
1021
+ key=lambda s: (any(c.isupper() for c in s), display_counts[s], s),
1022
+ )
1023
+ return surname_norm, surname_display
1024
+
1025
+
1026
+ def _bridge_compound_surnames(
1027
+ clusters: list[list[AuthorOccurrence]],
1028
+ ) -> list[list[AuthorOccurrence]]:
1029
+ """Merge single-surname clusters into compound-surname clusters.
1030
+
1031
+ 'Reyes, Sandra' can join 'Polanía Reyes, Sandra' — but only on strong
1032
+ evidence: the compound surname starts or ends with the single one,
1033
+ the given-name signatures are fully compatible, *and* at least one
1034
+ exact full given name is shared. Clusters that both carry external
1035
+ ids never bridge (same-id cases were already merged by
1036
+ :func:`_merge_clusters_by_external_id`, so remaining id pairs
1037
+ disagree). Initials-only records never bridge; ambiguity (two
1038
+ possible compound targets) leaves the cluster where it is.
1039
+ """
1040
+
1041
+ def block_key(cluster: list[AuthorOccurrence]) -> str:
1042
+ return _cluster_surname(cluster)[0]
1043
+
1044
+ by_key: dict[str, list[list[AuthorOccurrence]]] = defaultdict(list)
1045
+ for cluster in clusters:
1046
+ by_key[block_key(cluster)].append(cluster)
1047
+ compound_keys = sorted(k for k in by_key if " " in k)
1048
+ if not compound_keys:
1049
+ return clusters
1050
+
1051
+ absorbed: set[int] = set()
1052
+ for single_key in sorted(k for k in by_key if " " not in k):
1053
+ target_keys = [
1054
+ ck for ck in compound_keys
1055
+ if ck.split()[0] == single_key or ck.split()[-1] == single_key
1056
+ ]
1057
+ if not target_keys:
1058
+ continue
1059
+ for single in by_key[single_key]:
1060
+ s_units = _cluster_best_units(single)
1061
+ s_words = {v for kind, v in s_units if kind == "w"}
1062
+ if not s_words or any(o.parsed.is_corporate for o in single):
1063
+ continue
1064
+ s_ids = _cluster_external_ids(single)
1065
+ targets: list[list[AuthorOccurrence]] = []
1066
+ for ck in target_keys:
1067
+ for compound in by_key[ck]:
1068
+ c_ids = _cluster_external_ids(compound)
1069
+ if s_ids and c_ids:
1070
+ # Both sides carry ids and (post id-merge) they
1071
+ # necessarily disagree — never bridge over that.
1072
+ continue
1073
+ c_units = _cluster_best_units(compound)
1074
+ c_words = {v for kind, v in c_units if kind == "w"}
1075
+ if (
1076
+ c_words
1077
+ and not _author_clusters_conflict(single, compound)
1078
+ and (s_words & c_words)
1079
+ and _units_compatible(s_units, c_units)
1080
+ ):
1081
+ targets.append(compound)
1082
+ if len(targets) == 1:
1083
+ targets[0].extend(single)
1084
+ absorbed.add(id(single))
1085
+ return [c for c in clusters if id(c) not in absorbed]
1086
+
1087
+
1088
+ def normalize_authors(
1089
+ *,
1090
+ works: pd.DataFrame,
1091
+ enriched_works: pd.DataFrame | None = None,
1092
+ citation_edges: pd.DataFrame | None = None,
1093
+ cfg: AuthorClusterConfig | None = None,
1094
+ aliases: dict[str, str] | None = None,
1095
+ external_ids: dict[str, dict[str, str | None]] | None = None,
1096
+ constraints: list[dict] | None = None,
1097
+ audit: list[dict] | None = None,
1098
+ identity_state: dict | None = None,
1099
+ ) -> tuple[pd.DataFrame, pd.DataFrame, list[dict]]:
1100
+ """Cluster authors across the corpus.
1101
+
1102
+ Parameters
1103
+ ----------
1104
+ works:
1105
+ Canonical works DataFrame (the output of
1106
+ :func:`citegraph.dedup.canonicalize_works`). Must be indexed by
1107
+ ``id`` and contain an ``Authors_List`` column (or ``Authors``).
1108
+ A ``ring`` column, when present, feeds the ``n_core_works``
1109
+ metric; frames without it are accepted (the metric reads 0).
1110
+ enriched_works:
1111
+ Optional output of :func:`citegraph.enrich.enrich_works`.
1112
+ If present and contains the ``OpenAlex_Authors`` column (a list
1113
+ of ``{display_name, openalex_id, orcid}`` dicts per row), those
1114
+ identifiers are attached to the matching work authors so
1115
+ OpenAlex acts as ground truth — for core works too.
1116
+ citation_edges:
1117
+ Optional citation graph with ``citing_id`` and ``cited_id`` columns.
1118
+ When provided, ``authors.csv`` counts citations received by each
1119
+ canonical author's works and the distinct works citing them.
1120
+ cfg:
1121
+ Tunable :class:`AuthorClusterConfig`.
1122
+ aliases:
1123
+ Optional ``{cluster_id: canonical_id}`` mapping. Applied after
1124
+ clustering: every cluster whose id is a key in ``aliases`` is
1125
+ merged into the cluster keyed by the value. Lets the user fix
1126
+ the algorithm's mistakes without re-running.
1127
+
1128
+ Returns
1129
+ -------
1130
+ authors_df:
1131
+ One row per canonical author, indexed by ``id``.
1132
+ citations_df:
1133
+ Edge table linking each canonical author to every work they
1134
+ appear on (``author_id``, ``record_id``, ``position``,
1135
+ ``raw_author``).
1136
+ review:
1137
+ List of dicts describing low-confidence clusters that the user
1138
+ may want to inspect.
1139
+ """
1140
+ cfg = cfg or AuthorClusterConfig()
1141
+ aliases = aliases or {}
1142
+
1143
+ attachment_audit: list[dict] = []
1144
+ occurrences = _collect_occurrences(
1145
+ works=works,
1146
+ enriched_works=enriched_works,
1147
+ audit=attachment_audit,
1148
+ )
1149
+
1150
+ # Block by surname_norm and cluster within each block.
1151
+ blocks: dict[str, list[AuthorOccurrence]] = defaultdict(list)
1152
+ for o in occurrences:
1153
+ blocks[o.parsed.surname_norm].append(o)
1154
+
1155
+ from citegraph.author_overrides import prepare_constraints
1156
+ seeds = prepare_constraints(occurrences, constraints or [])
1157
+ _mark_identity_ambiguities(seeds)
1158
+ raw_clusters = _cluster_block(occurrences, cfg, seeds)
1159
+
1160
+ # Cross-block passes: shared external ids override blocking, then
1161
+ # single-surname clusters bridge into compound-surname clusters when
1162
+ # the evidence is strong enough (see the function docstrings).
1163
+ raw_clusters = _merge_clusters_by_external_id(raw_clusters)
1164
+ raw_clusters = _bridge_compound_surnames(raw_clusters)
1165
+ block_keys = frozenset(blocks)
1166
+
1167
+ # Build AuthorCluster records with stable ids.
1168
+ used_ids: set[str] = set()
1169
+ clusters: list[AuthorCluster] = []
1170
+ for occs in raw_clusters:
1171
+ if not occs:
1172
+ continue
1173
+ canonical_given, given_token = _canonical_given_for_cluster(occs)
1174
+ surname_norm, surname_display = _cluster_surname(occs)
1175
+ base_id = _make_author_id(surname_norm, given_token)
1176
+ cid = base_id
1177
+ suffix = 1
1178
+ while cid in used_ids:
1179
+ cid = _make_author_id(surname_norm, given_token, suffix)
1180
+ suffix += 1
1181
+ used_ids.add(cid)
1182
+
1183
+ oa_id = next((o.openalex_id for o in occs if o.openalex_id), None)
1184
+ orcid = next((o.orcid for o in occs if o.orcid), None)
1185
+ initials = max((o.parsed.initials for o in occs), key=len, default="")
1186
+
1187
+ display = (
1188
+ f"{canonical_given} {surname_display}".strip()
1189
+ if canonical_given and not canonical_given.isupper()
1190
+ else f"{surname_display}, {canonical_given}".strip(", ")
1191
+ )
1192
+
1193
+ clusters.append(
1194
+ AuthorCluster(
1195
+ id=cid,
1196
+ display_name=display,
1197
+ surname=surname_display,
1198
+ surname_norm=surname_norm,
1199
+ canonical_given=canonical_given,
1200
+ initials=initials,
1201
+ openalex_id=oa_id,
1202
+ orcid=orcid,
1203
+ occurrences=occs,
1204
+ review_reason=_review_reason(occs, cfg, block_keys),
1205
+ )
1206
+ )
1207
+
1208
+ # Post-hoc ambiguity flag: a no-id cluster whose name evidence is
1209
+ # compatible with two or more sibling clusters in the same surname
1210
+ # block was held apart only because the evidence could not decide —
1211
+ # surface that in the review file instead of leaving the split invisible.
1212
+ by_block: dict[str, list[AuthorCluster]] = defaultdict(list)
1213
+ for c in clusters:
1214
+ by_block[c.surname_norm].append(c)
1215
+ for block_clusters in by_block.values():
1216
+ if len(block_clusters) < 3:
1217
+ continue # fewer than two possible rivals — nothing ambiguous
1218
+ for c in block_clusters:
1219
+ if c.review_reason or c.openalex_id or c.orcid:
1220
+ continue
1221
+ units = _cluster_best_units(c.occurrences)
1222
+ if not units:
1223
+ continue
1224
+ n_compatible = sum(
1225
+ 1
1226
+ for other in block_clusters
1227
+ if other is not c
1228
+ and (other_units := _cluster_best_units(other.occurrences))
1229
+ and _units_compatible(units, other_units)
1230
+ )
1231
+ if n_compatible >= 2:
1232
+ c.review_reason = (
1233
+ "ambiguous name evidence: compatible with multiple "
1234
+ "clusters in this surname block"
1235
+ )
1236
+
1237
+ identity_changes = []
1238
+ if identity_state is not None:
1239
+ from citegraph.identity import occurrence_key, reconcile, resolve_redirect
1240
+
1241
+ def memberships():
1242
+ return {c.id: {occurrence_key(o.record_id, o.position, o.parsed.raw)
1243
+ for o in c.occurrences} for c in clusters}
1244
+
1245
+ mapping, algorithm, changes = reconcile(memberships(), identity_state.get('algorithm'))
1246
+ for c in clusters:
1247
+ c.id = mapping[c.id]
1248
+ identity_changes.extend(changes)
1249
+ for source in aliases:
1250
+ resolve_redirect(source, aliases) # reject cycles before resolving history
1251
+ # A curated alias asserts that two spellings are one person. That
1252
+ # outranks both how the algorithm clustered them and how an earlier run
1253
+ # published them, so the merge is settled here — against stored
1254
+ # occurrences, before any published ID is assigned — and identity
1255
+ # assignment simply follows the membership the user chose.
1256
+ previous_entries = (identity_state.get('published') or {}).get('entries', {})
1257
+ current = memberships()
1258
+ parent = {cid: cid for cid in current}
1259
+
1260
+ def _find(cid: str) -> str:
1261
+ while parent[cid] != cid:
1262
+ parent[cid] = parent[parent[cid]]
1263
+ cid = parent[cid]
1264
+ return cid
1265
+
1266
+ def _union(left: str, right: str) -> None:
1267
+ left, right = _find(left), _find(right)
1268
+ if left != right:
1269
+ parent[right] = left
1270
+
1271
+ def _holders(public_id: str) -> list[str]:
1272
+ """Clusters now holding a published ID's occurrences.
1273
+
1274
+ An ID may have been retired by a split, continued by one cluster, or
1275
+ spread over several; following its stored occurrences covers all
1276
+ three without the caller needing to know which happened. An ID with
1277
+ no stored occurrences is only usable if it names a cluster outright.
1278
+ """
1279
+ members = set(previous_entries.get(public_id, ()))
1280
+ holders = sorted(cid for cid, held in current.items() if members & held)
1281
+ if holders:
1282
+ return holders
1283
+ return [public_id] if public_id in current else []
1284
+
1285
+ named: dict[str, set[str]] = defaultdict(set)
1286
+ stale: set[str] = set()
1287
+ bindings = {}
1288
+ for source, target in aliases.items():
1289
+ sources, targets = _holders(source), _holders(target)
1290
+ if not sources or not targets:
1291
+ stale.add(source if not sources else target)
1292
+ continue
1293
+ for cid in sources[1:] + targets:
1294
+ _union(sources[0], cid)
1295
+ for cid in targets:
1296
+ named[cid].add(target)
1297
+ bindings[source] = {'target': target, 'source_ids': sources, 'target_ids': targets}
1298
+ if stale:
1299
+ raise ValueError(
1300
+ f'Stale author aliases: {sorted(stale)}; review against current authors')
1301
+
1302
+ components: dict[str, list[str]] = defaultdict(list)
1303
+ for cid in current:
1304
+ components[_find(cid)].append(cid)
1305
+ migrated_aliases, pins = {}, {}
1306
+ for component in components.values():
1307
+ claims = {t for cid in component for t in named.get(cid, ())}
1308
+ if not claims:
1309
+ continue
1310
+ # Absorb into a cluster the user actually named, the largest one
1311
+ # first, so the survivor carries that person's display name instead
1312
+ # of an initials-only fragment's.
1313
+ canonical = max((cid for cid in component if named.get(cid)),
1314
+ key=lambda cid: (len(current[cid]), cid))
1315
+ for cid in component:
1316
+ if cid != canonical:
1317
+ migrated_aliases[cid] = canonical
1318
+ pins[canonical] = max(claims, key=lambda t: (len(previous_entries.get(t, ())), t))
1319
+ aliases = migrated_aliases
1320
+
1321
+ # Apply user-curated aliases: merge cluster B into cluster A.
1322
+ if aliases:
1323
+ clusters = _apply_aliases(clusters, aliases)
1324
+
1325
+ if identity_state is not None:
1326
+ # Membership is already what the user asked for; assigning IDs is all
1327
+ # that is left. The pins carry their merges through a split that would
1328
+ # otherwise retire the very IDs external crosswalks join on.
1329
+ mapping, published, changes = reconcile(
1330
+ memberships(), identity_state.get('published'), pinned=pins)
1331
+ for c in clusters:
1332
+ c.id = mapping[c.id]
1333
+ identity_changes.extend(changes)
1334
+ identity_state.update(schema_version=1, algorithm=algorithm, published=published,
1335
+ alias_bindings=bindings)
1336
+
1337
+ if external_ids:
1338
+ _apply_external_ids(clusters, external_ids)
1339
+
1340
+ authors_df = _clusters_to_authors_df(
1341
+ clusters, works=works, citation_edges=citation_edges
1342
+ )
1343
+ citations_df = _clusters_to_citations_df(clusters)
1344
+ review = [
1345
+ {
1346
+ "author_id": c.id,
1347
+ "display_name": c.display_name,
1348
+ "surname": c.surname_norm,
1349
+ "n_occurrences": c.n_occurrences,
1350
+ "raw_strings": sorted({o.parsed.raw for o in c.occurrences}),
1351
+ "reason": c.review_reason,
1352
+ }
1353
+ for c in clusters
1354
+ if c.review_reason
1355
+ ]
1356
+ review.extend({'author_id': entry['old_id'], 'reason': 'identity_split', **entry}
1357
+ for entry in identity_changes if entry['decision'] == 'identity_split')
1358
+
1359
+ author_by_occurrence = {(row.record_id, row.position): row.author_id
1360
+ for row in citations_df.itertuples(index=False)}
1361
+ for entry in attachment_audit:
1362
+ entry["author_id"] = author_by_occurrence.get((entry["record_id"], entry["position"]))
1363
+ if entry["decision"] in {"ambiguous", "conflict"}:
1364
+ # A provider author nobody claimed is the mirror image of a
1365
+ # conflict, not a cluster to review: it has no position and no
1366
+ # author_id, so filing it as one inflates the backlog with rows
1367
+ # that cannot be acted on.
1368
+ review.append({**entry, "reason": "enrichment_provider_author_unused"
1369
+ if entry.get("detail") == "unassigned_provider_author"
1370
+ else "enrichment_" + entry["decision"]})
1371
+ resolution_audit: list[dict] = []
1372
+ for cluster in clusters:
1373
+ identifiers = {"openalex_ids": sorted({o.openalex_id for o in cluster.occurrences if o.openalex_id}),
1374
+ "orcids": sorted({o.orcid for o in cluster.occurrences if o.orcid})}
1375
+ resolution_audit.append({"decision": "cluster_identity", "author_id": cluster.id,
1376
+ "occurrences": [{"record_id": o.record_id, "position": o.position}
1377
+ for o in cluster.occurrences], **identifiers})
1378
+ reasons = []
1379
+ if len(identifiers["openalex_ids"]) > 1:
1380
+ reasons.append("multiple_openalex_ids")
1381
+ if len({o.record_id for o in cluster.occurrences}) < len(cluster.occurrences):
1382
+ reasons.append("duplicate_authorship")
1383
+ for reason in reasons:
1384
+ review.append({"author_id": cluster.id, "reason": reason, **identifiers})
1385
+ evidence_candidates: dict[str, set[int]] = defaultdict(set)
1386
+ for i, cluster in enumerate(clusters):
1387
+ for key in ({'surname:' + o.parsed.surname_norm for o in cluster.occurrences}
1388
+ | _cluster_external_ids(cluster.occurrences)):
1389
+ evidence_candidates[key].add(i)
1390
+ for i, left in enumerate(clusters):
1391
+ candidate_indices: set[int] = set()
1392
+ for key in ({'surname:' + o.parsed.surname_norm for o in left.occurrences}
1393
+ | _cluster_external_ids(left.occurrences)):
1394
+ candidate_indices.update(evidence_candidates[key])
1395
+ for j in sorted(index for index in candidate_indices if index > i):
1396
+ right = clusters[j]
1397
+ if not (_name_supported(left.occurrences, right.occurrences)
1398
+ or _cluster_external_ids(left.occurrences) & _cluster_external_ids(right.occurrences)):
1399
+ continue
1400
+ reasons = _author_conflict_reasons(left.occurrences, right.occurrences)
1401
+ if reasons:
1402
+ finding = {"author_id": left.id, "other_author_id": right.id,
1403
+ "reason": ",".join(reasons), "decision": "prevented_merge"}
1404
+ review.append(finding)
1405
+ resolution_audit.append(finding)
1406
+ if audit is not None:
1407
+ audit.extend(identity_changes)
1408
+ audit.extend(attachment_audit)
1409
+ audit.extend(resolution_audit)
1410
+ audit.extend({"decision": "manual_override", **row} for row in constraints or [])
1411
+ audit.extend({"decision": "legacy_alias", "cluster_id": source, "canonical_id": target}
1412
+ for source, target in aliases.items())
1413
+
1414
+ logger.info(
1415
+ "Author normalization: %d occurrences → %d clusters (%d flagged for review)",
1416
+ len(occurrences),
1417
+ len(clusters),
1418
+ len(review),
1419
+ )
1420
+ return authors_df, citations_df, review
1421
+
1422
+
1423
+ def _collect_occurrences(
1424
+ *,
1425
+ works: pd.DataFrame,
1426
+ enriched_works: pd.DataFrame | None,
1427
+ audit: list[dict] | None = None,
1428
+ ) -> list[AuthorOccurrence]:
1429
+ """Walk the works table and emit one AuthorOccurrence per author.
1430
+
1431
+ Runs in two passes: the first parse of every author string builds a
1432
+ corpus surname lexicon from trusted forms (comma forms, hyphenated
1433
+ and particle compounds, enrichment family names); the second parse
1434
+ uses that lexicon so comma-less names adopt attested compound
1435
+ surname boundaries. Co-author surname-norm keys are computed per
1436
+ record so the clustering step can use them as a tiebreak signal.
1437
+ """
1438
+ occurrences: list[AuthorOccurrence] = []
1439
+
1440
+ # Build a per-work enrichment map: work_id -> [(display, oa_id, orcid), ...]
1441
+ enrich_map: dict[str, list[dict]] = {}
1442
+ if enriched_works is not None and not enriched_works.empty:
1443
+ if "OpenAlex_Authors" in enriched_works.columns:
1444
+ for work_id, row in enriched_works.iterrows():
1445
+ authors = row.get("OpenAlex_Authors")
1446
+ if isinstance(authors, list):
1447
+ enrich_map[str(work_id)] = authors
1448
+
1449
+ work_rows: list[tuple[str, list[str]]] = []
1450
+ if not works.empty:
1451
+ for work_id, row in works.iterrows():
1452
+ work_rows.append((str(work_id), _row_authors(row)))
1453
+
1454
+ lexicon = _build_surname_lexicon(
1455
+ (a for _wid, authors_list in work_rows for a in authors_list),
1456
+ enrich_map,
1457
+ )
1458
+
1459
+ for work_id, authors_list in work_rows:
1460
+ parsed = [parse_author(a, known_surnames=lexicon) for a in authors_list]
1461
+ co_keys = tuple(p.surname_norm if p is not None else "" for p in parsed)
1462
+ enrich_authors = enrich_map.get(work_id, [])
1463
+ work_audit: list[dict] = []
1464
+ assignments = _match_enrichment_authors(parsed, enrich_authors, lexicon, work_audit)
1465
+ if audit is not None:
1466
+ audit.extend({"record_id": work_id, **entry} for entry in work_audit)
1467
+ for pos, p in enumerate(parsed):
1468
+ if p is None:
1469
+ continue
1470
+ oa_id, orcid = assignments[pos]
1471
+ occurrences.append(
1472
+ AuthorOccurrence(
1473
+ parsed=p,
1474
+ record_id=work_id,
1475
+ position=pos,
1476
+ co_author_keys=tuple(k for i, k in enumerate(co_keys) if i != pos and k),
1477
+ openalex_id=oa_id,
1478
+ orcid=orcid,
1479
+ )
1480
+ )
1481
+
1482
+ return occurrences
1483
+
1484
+
1485
+ def _build_surname_lexicon(
1486
+ raw_strings: Iterable[str],
1487
+ enrich_map: dict[str, list[dict]],
1488
+ ) -> frozenset[str]:
1489
+ """Collect normalized multi-word surnames attested by trusted forms.
1490
+
1491
+ Trusted sources: comma forms ("Guerra Forero, J.A." states its own
1492
+ boundary), hyphenated surnames ("Polania-Reyes" attests "polania
1493
+ reyes"), particle compounds ("van der Berg"), the Spanish 'y'
1494
+ connector, and enrichment ``family`` fields. Single-word surnames are
1495
+ never collected — a bare "Knowles" elsewhere must not re-split
1496
+ "Caitlin Knowles Myers".
1497
+ """
1498
+ lexicon: set[str] = set()
1499
+ for raw in raw_strings:
1500
+ p = parse_author(raw)
1501
+ if p is None or p.is_corporate:
1502
+ continue
1503
+ norm = p.surname_norm
1504
+ if " " not in norm:
1505
+ continue
1506
+ trusted = (
1507
+ "," in p.raw
1508
+ or "-" in p.surname
1509
+ or norm.split()[0] in _PARTICLES
1510
+ or " y " in f" {norm} "
1511
+ )
1512
+ if trusted:
1513
+ lexicon.add(norm)
1514
+ # A multi-word `family` redefines a surname boundary for the whole corpus,
1515
+ # so a provider record that *contradicts* another record for the same name
1516
+ # must not carry it: CrossRef returned family="Claudia Lopez"/given="Maria"
1517
+ # for María Claudia López on one work while others said "Lopez", and
1518
+ # adopting the longer boundary shattered her into three clusters (one of
1519
+ # them holding her OpenAlex id). When the corpus attests a shorter boundary
1520
+ # for the same display name, keep the conservative last-token parse — which
1521
+ # `author_review.json` already flags as a possible compound surname —
1522
+ # rather than trusting the longer one. An uncontradicted `family` is still
1523
+ # trusted on its own, so a lone "Guerra Forero" record still teaches it.
1524
+ families_by_display: dict[str, set[str]] = defaultdict(set)
1525
+ candidates: list[tuple[str, str]] = []
1526
+ for items in enrich_map.values():
1527
+ for item in items:
1528
+ family = item.get("family")
1529
+ if not isinstance(family, str):
1530
+ continue
1531
+ family_norm = _norm_surname(family)
1532
+ display = item.get("display_name")
1533
+ display_norm = _norm_surname(display) if isinstance(display, str) else ""
1534
+ families_by_display[display_norm].add(family_norm)
1535
+ if " " not in family_norm:
1536
+ continue
1537
+ # CrossRef sometimes stuffs the entire name into `family`
1538
+ # (with an empty `given`); a family that swallows the whole
1539
+ # display name attests nothing about the surname boundary.
1540
+ if display_norm and display_norm == family_norm:
1541
+ continue
1542
+ candidates.append((display_norm, family_norm))
1543
+ for display_norm, family_norm in candidates:
1544
+ shorter = {
1545
+ other
1546
+ for other in families_by_display[display_norm] - {family_norm}
1547
+ if other and family_norm.endswith(f" {other}")
1548
+ }
1549
+ if display_norm and shorter:
1550
+ continue
1551
+ lexicon.add(family_norm)
1552
+ return frozenset(lexicon)
1553
+
1554
+
1555
+ def _row_authors(row: pd.Series) -> list[str]:
1556
+ """Pull a list of author strings from a DataFrame row.
1557
+
1558
+ ``Authors_List`` is preferred (the canonical schema field) and round-
1559
+ trips through CSV as a Python repr that we parse back with
1560
+ :func:`ast.literal_eval`. When only the comma-joined ``Authors``
1561
+ string is available — e.g. a ``references.csv`` written before
1562
+ Authors_List was preserved through dedup — we fall back to
1563
+ :func:`_split_joined_authors`, which is safe against the
1564
+ ``"Smith, J., García, A."`` pattern.
1565
+ """
1566
+ val = row.get("Authors_List")
1567
+ if isinstance(val, list):
1568
+ return [str(a) if a is not None else "" for a in val]
1569
+ if isinstance(val, str):
1570
+ try:
1571
+ slots = ast.literal_eval(val)
1572
+ except (ValueError, SyntaxError):
1573
+ slots = None
1574
+ if isinstance(slots, list):
1575
+ return [str(a) if a is not None else "" for a in slots]
1576
+ parsed = parse_authors_list(val)
1577
+ if len(parsed) > 1:
1578
+ return parsed
1579
+ if parsed:
1580
+ return _split_joined_authors(parsed[0])
1581
+ fallback = row.get("Authors")
1582
+ if isinstance(fallback, str) and fallback.strip():
1583
+ return _split_joined_authors(fallback)
1584
+ return []
1585
+
1586
+
1587
+ def _split_joined_authors(s: str) -> list[str]:
1588
+ """Split a delimiter-joined author string into individual names.
1589
+
1590
+ Prefers ``;`` (unambiguous) when present. Otherwise splits on ``,``
1591
+ and re-glues obvious ``Surname, Given`` pairs so that
1592
+ ``"Smith, J., García, A."`` and ``"Smith, John, García, Ana"`` become
1593
+ two authors rather than four fragments. Strings without that pattern
1594
+ (e.g. ``"John Smith, Mary Jones"``) pass through unchanged.
1595
+ """
1596
+ s = s.strip()
1597
+ if not s:
1598
+ return []
1599
+ if ";" in s:
1600
+ return [p.strip() for p in s.split(";") if p.strip()]
1601
+
1602
+ parts = [p.strip() for p in s.split(",") if p.strip()]
1603
+ out: list[str] = []
1604
+ i = 0
1605
+ while i < len(parts):
1606
+ cur = parts[i]
1607
+ nxt = parts[i + 1] if i + 1 < len(parts) else None
1608
+ if (
1609
+ nxt is not None
1610
+ and not _is_initial_only_token(cur)
1611
+ and (
1612
+ _is_initial_only_token(nxt)
1613
+ or _looks_like_surname_given_pair(cur, nxt)
1614
+ )
1615
+ ):
1616
+ out.append(f"{cur}, {nxt}")
1617
+ i += 2
1618
+ else:
1619
+ out.append(cur)
1620
+ i += 1
1621
+ return out
1622
+
1623
+
1624
+ def _looks_like_surname_given_pair(surname_part: str, given_part: str) -> bool:
1625
+ """Heuristic for comma-joined fallback strings.
1626
+
1627
+ This intentionally handles only high-confidence cases: one-word
1628
+ surnames (``Smith, John``) and particle compounds
1629
+ (``de la Cruz, Juan``). Multi-word chunks such as ``Talbot Page`` are
1630
+ likely already in first-last order, so pairing them with the next
1631
+ comma part would create a false author.
1632
+ """
1633
+ surname_words = surname_part.split()
1634
+ given_words = given_part.split()
1635
+ if not surname_words or not given_words or len(given_words) > 2:
1636
+ return False
1637
+ if len(surname_words) == 1:
1638
+ return True
1639
+ return any(w.strip(".-").lower() in _PARTICLES for w in surname_words[:-1])
1640
+
1641
+
1642
+ def _given_subsequence(a: tuple, b: tuple) -> bool:
1643
+ """Recognize omitted given names without admitting conflicting full words."""
1644
+ short, long = sorted((a, b), key=len)
1645
+ if not short or len(short) == len(long):
1646
+ return False
1647
+ remaining = iter(long)
1648
+ return all(any(_units_compatible((unit,), (other,)) for other in remaining)
1649
+ for unit in short)
1650
+
1651
+
1652
+ def _match_enrichment_authors(
1653
+ parsed: list[ParsedAuthor | None],
1654
+ enrichment: list[dict],
1655
+ known_surnames: frozenset[str],
1656
+ audit: list[dict],
1657
+ ) -> list[tuple[str | None, str | None]]:
1658
+ """Attach only mutually unique names, consuming each provider slot once."""
1659
+ providers = [parse_author(item.get("display_name") or "", known_surnames=known_surnames)
1660
+ for item in enrichment]
1661
+ pairs: dict[tuple[int, int], str] = {}
1662
+ evidence: list[list[dict]] = [[] for _ in parsed]
1663
+ for i, author in enumerate(parsed):
1664
+ for j, candidate in enumerate(providers):
1665
+ reason = "unusable_name"
1666
+ tier = None
1667
+ if author is not None and candidate is not None:
1668
+ au, cu = _given_units(author), _given_units(candidate)
1669
+ aw = {v for k, v in au if k == "w"}
1670
+ cw = {v for k, v in cu if k == "w"}
1671
+ single, compound = sorted((author.surname_norm, candidate.surname_norm),
1672
+ key=lambda name: len(name.split()))
1673
+ surname_ok = author.surname_norm == candidate.surname_norm or (
1674
+ len(single.split()) == 1 and single not in _PARTICLES
1675
+ and len(compound.split()) > 1
1676
+ and single in (compound.split()[0], compound.split()[-1])
1677
+ and bool(aw & cw)
1678
+ )
1679
+ if author.is_corporate != candidate.is_corporate:
1680
+ reason = "corporate_person_conflict"
1681
+ elif not surname_ok:
1682
+ reason = "surname_conflict"
1683
+ elif author.is_corporate:
1684
+ tier, reason = "full", "exact_corporate_name"
1685
+ elif not au or not cu:
1686
+ reason = "missing_given_name"
1687
+ elif not (_units_compatible(au, cu) or _given_subsequence(au, cu)):
1688
+ reason = "given_name_conflict"
1689
+ else:
1690
+ tier = "full" if aw and cw and aw == cw else "initial"
1691
+ reason = "compatible_" + tier
1692
+ if tier:
1693
+ pairs[i, j] = tier
1694
+ evidence[i].append({"provider_position": j, "reason": reason, "tier": tier})
1695
+ assigned: dict[int, int] = {}
1696
+ used: set[int] = set()
1697
+ for tier in ("full", "initial"):
1698
+ while True:
1699
+ candidates = {(i, j) for (i, j), t in pairs.items()
1700
+ if i not in assigned and j not in used
1701
+ and (tier == "initial" or t == "full")}
1702
+ round_pairs = [(i, j) for i, j in sorted(candidates)
1703
+ if sum(a == i for a, _ in candidates) == 1
1704
+ and sum(b == j for _, b in candidates) == 1]
1705
+ if not round_pairs:
1706
+ break
1707
+ assigned.update(round_pairs)
1708
+ used.update(j for _, j in round_pairs)
1709
+ result = []
1710
+ for i, author in enumerate(parsed):
1711
+ j = assigned.get(i)
1712
+ item = enrichment[j] if j is not None else {}
1713
+ ids = tuple(
1714
+ value.strip().rstrip("/").rsplit("/", 1)[-1] if isinstance(value, str) and value.strip() else None
1715
+ for value in (item.get("openalex_id"), item.get("orcid"))
1716
+ )
1717
+ result.append(ids)
1718
+ decision = ("assigned" if j is not None else "ambiguous"
1719
+ if any(a == i for a, _ in pairs) else "conflict" if enrichment else "unmatched")
1720
+ audit.append({"position": i, "raw_author": author.raw if author else None,
1721
+ "normalized_name": {"surname": author.surname_norm,
1722
+ "given_units": list(_given_units(author))} if author else None,
1723
+ "candidates": evidence[i], "provider_position": j,
1724
+ "openalex_id": ids[0], "orcid": ids[1], "decision": decision})
1725
+ for j, candidate in enumerate(providers):
1726
+ if j not in used:
1727
+ audit.append({"position": None, "raw_author": None,
1728
+ "provider_position": j,
1729
+ "provider_name": candidate.raw if candidate else None,
1730
+ "decision": "conflict", "detail": "unassigned_provider_author"})
1731
+ return result
1732
+
1733
+
1734
+ def _review_reason(
1735
+ occs: list[AuthorOccurrence],
1736
+ cfg: AuthorClusterConfig,
1737
+ block_keys: frozenset[str] = frozenset(),
1738
+ ) -> str | None:
1739
+ """Return a short reason string when the cluster looks low-confidence."""
1740
+ if any(o.parsed.is_corporate for o in occs):
1741
+ return "detected as corporate author (not a person)"
1742
+ has_external = any(o.openalex_id or o.orcid for o in occs)
1743
+ if has_external:
1744
+ return None
1745
+ for o in occs:
1746
+ p = o.parsed
1747
+ if "," in p.raw:
1748
+ continue
1749
+ words = [v for kind, v in _given_units(p) if kind == "w"]
1750
+ if len(words) >= 3:
1751
+ # 'Jose Alberto Guerra Forero' parsed conservatively — the
1752
+ # trailing given names may really be a compound surname the
1753
+ # corpus could not corroborate.
1754
+ return "possible compound surname: multiple full given names in no-comma form"
1755
+ if len(words) == 2 and words[-1] in block_keys and words[-1] != p.surname_norm:
1756
+ return "possible compound surname: a given name matches another surname block"
1757
+ has_full = any(o.parsed.has_full_first for o in occs)
1758
+ if not has_full and len(occs) >= 3:
1759
+ # Initial-only cluster with several citations — most worth checking.
1760
+ return "initial-only cluster with no full-name evidence"
1761
+ return None
1762
+
1763
+
1764
+ def _apply_aliases(
1765
+ clusters: list[AuthorCluster],
1766
+ aliases: dict[str, str],
1767
+ ) -> list[AuthorCluster]:
1768
+ """Merge clusters according to ``aliases``: {cluster_id: canonical_id}.
1769
+
1770
+ Aliases are followed transitively (A→B, B→C ⇒ A absorbed into C).
1771
+ Reject stale identifiers and cycles so corrections cannot silently disappear.
1772
+ """
1773
+ by_id = {c.id: c for c in clusters}
1774
+ stale = (set(aliases) | set(aliases.values())) - set(by_id)
1775
+ if stale:
1776
+ raise ValueError(f'Stale author aliases: {sorted(stale)}; review against current authors')
1777
+
1778
+ def resolve(cid: str, seen: set[str]) -> str:
1779
+ if cid in seen:
1780
+ raise ValueError(f'Author alias cycle involving {cid}')
1781
+ if cid not in aliases:
1782
+ return cid
1783
+ seen.add(cid)
1784
+ return resolve(aliases[cid], seen)
1785
+
1786
+ groups: dict[str, list[AuthorOccurrence]] = defaultdict(list)
1787
+ for c in clusters:
1788
+ groups[resolve(c.id, set())].extend(c.occurrences)
1789
+ for target, group in groups.items():
1790
+ if len({o.orcid for o in group if o.orcid}) > 1:
1791
+ raise ValueError(f'Author alias {target} has conflicting ORCIDs')
1792
+ if any((r.record_id, r.position) in member.cannot_link for member in group for r in group):
1793
+ raise ValueError(f'Author alias {target} contradicts explicit separate constraint')
1794
+ merged: dict[str, AuthorCluster] = {}
1795
+ for c in clusters:
1796
+ target_id = resolve(c.id, set())
1797
+ if target_id == c.id:
1798
+ merged.setdefault(c.id, c)
1799
+ continue
1800
+ target = by_id.get(target_id)
1801
+ if target is None:
1802
+ merged.setdefault(c.id, c)
1803
+ continue
1804
+ absorber = merged.setdefault(target.id, target)
1805
+ absorber.occurrences.extend(c.occurrences)
1806
+ # Pull external ids forward if the absorber didn't have them.
1807
+ absorber.openalex_id = absorber.openalex_id or c.openalex_id
1808
+ absorber.orcid = absorber.orcid or c.orcid
1809
+
1810
+ return list(merged.values())
1811
+
1812
+
1813
+ def _clusters_to_authors_df(
1814
+ clusters: list[AuthorCluster],
1815
+ *,
1816
+ works: pd.DataFrame,
1817
+ citation_edges: pd.DataFrame | None = None,
1818
+ ) -> pd.DataFrame:
1819
+ """Render clusters as the on-disk ``authors.csv`` shape.
1820
+
1821
+ Metrics per canonical author:
1822
+
1823
+ - ``n_works`` — distinct canonical works the author appears on,
1824
+ - ``n_core_works`` — of those, how many are ring 0 (the user's corpus),
1825
+ - ``n_citations_received`` — citation edges into any of their works,
1826
+ - ``n_distinct_citing_works`` — distinct works doing that citing.
1827
+ """
1828
+ ring_by_work = works["ring"].to_dict() if "ring" in works.columns else {}
1829
+ in_edges: dict[str, set[str]] = defaultdict(set)
1830
+ n_in_edges: dict[str, int] = defaultdict(int)
1831
+ if citation_edges is not None and not citation_edges.empty:
1832
+ require_columns(
1833
+ citation_edges,
1834
+ ["citing_id", "cited_id"],
1835
+ artifact="citation graph",
1836
+ )
1837
+ for _, row in citation_edges.iterrows():
1838
+ cited_id = row.get("cited_id")
1839
+ citing_id = row.get("citing_id")
1840
+ if cited_id and citing_id:
1841
+ in_edges[str(cited_id)].add(str(citing_id))
1842
+ n_in_edges[str(cited_id)] += 1
1843
+
1844
+ rows = []
1845
+ for c in clusters:
1846
+ work_ids = {o.record_id for o in c.occurrences}
1847
+ citing: set[str] = set()
1848
+ for w in work_ids:
1849
+ citing |= in_edges.get(w, set())
1850
+ rows.append(
1851
+ {
1852
+ "id": c.id,
1853
+ "display_name": c.display_name,
1854
+ "surname": c.surname,
1855
+ "surname_norm": c.surname_norm,
1856
+ "canonical_given": c.canonical_given,
1857
+ "initials": c.initials,
1858
+ "openalex_id": c.openalex_id,
1859
+ "orcid": c.orcid,
1860
+ "n_works": len(work_ids),
1861
+ "n_core_works": sum(1 for w in work_ids if ring_by_work.get(w) == 0),
1862
+ "n_citations_received": sum(n_in_edges.get(w, 0) for w in work_ids),
1863
+ "n_distinct_citing_works": len(citing),
1864
+ }
1865
+ )
1866
+ df = pd.DataFrame(rows, columns=AUTHOR_COLUMNS)
1867
+ if df.empty:
1868
+ return df.set_index("id")
1869
+ # Stable two-key sort: without edges, authorship volume still yields a
1870
+ # deterministic, meaningful order.
1871
+ return df.set_index("id").sort_values(
1872
+ ["n_citations_received", "n_works"],
1873
+ ascending=[False, False],
1874
+ kind="mergesort",
1875
+ )
1876
+
1877
+
1878
+ def _clusters_to_citations_df(clusters: list[AuthorCluster]) -> pd.DataFrame:
1879
+ """Render the per-occurrence edges as ``author_citations.csv``."""
1880
+ rows = []
1881
+ for c in clusters:
1882
+ for o in c.occurrences:
1883
+ rows.append(
1884
+ {
1885
+ "author_id": c.id,
1886
+ "record_id": o.record_id,
1887
+ "position": o.position,
1888
+ "raw_author": o.parsed.raw,
1889
+ }
1890
+ )
1891
+ return pd.DataFrame(rows, columns=AUTHOR_CITATION_COLUMNS)
1892
+
1893
+
1894
+ def _apply_external_ids(
1895
+ clusters: list[AuthorCluster],
1896
+ external_ids: dict[str, dict[str, str | None]],
1897
+ ) -> None:
1898
+ """Stamp hand-curated identifiers onto canonical authors, in place.
1899
+
1900
+ Applied *after* identity reconciliation, because the file keys on the
1901
+ published ``a-`` id the user actually sees. A curated value wins over
1902
+ whatever the provider supplied: someone who opened the provider record
1903
+ and checked is better evidence than a name match that failed.
1904
+
1905
+ This deliberately does not merge anything. ``author_aliases.csv`` decides
1906
+ who is the same person; letting an identifier also force merges would put
1907
+ two curated files in charge of the same question.
1908
+ """
1909
+ by_id = {c.id: c for c in clusters}
1910
+ stale = sorted(set(external_ids) - set(by_id))
1911
+ if stale:
1912
+ raise ValueError(
1913
+ f'Stale author external ids: {stale}; review against current authors.csv'
1914
+ )
1915
+ claimed: dict[tuple[str, str], str] = {}
1916
+ for author_id, ids in external_ids.items():
1917
+ for field_name in ("openalex_id", "orcid"):
1918
+ value = ids.get(field_name)
1919
+ if not value:
1920
+ continue
1921
+ owner = claimed.get((field_name, value))
1922
+ if owner is not None and owner != author_id:
1923
+ raise ValueError(
1924
+ f'External {field_name} {value} claimed by both {owner} and '
1925
+ f'{author_id}; one identifier cannot name two authors'
1926
+ )
1927
+ claimed[field_name, value] = author_id
1928
+ setattr(by_id[author_id], field_name, value)
1929
+
1930
+
1931
+ def load_external_ids(path: Path | str | None) -> dict[str, dict[str, str | None]]:
1932
+ """Load ``author_id,openalex_id,orcid`` overrides keyed by author id.
1933
+
1934
+ Missing path or file returns ``{}``. Blank identifier cells mean "leave
1935
+ this one alone", so a row can set only an ORCID; a row that sets neither
1936
+ is a mistake and raises rather than being silently ignored. Any further
1937
+ columns (a ``note`` explaining the override) are read and discarded.
1938
+ """
1939
+ if path is None:
1940
+ return {}
1941
+ p = Path(path)
1942
+ if not p.exists():
1943
+ return {}
1944
+ out: dict[str, dict[str, str | None]] = {}
1945
+ with p.open("r", encoding="utf-8", newline="") as handle:
1946
+ rows = list(csv.DictReader(_strip_comments(handle)))
1947
+ for row in rows:
1948
+ author_id = (row.get("author_id") or "").strip()
1949
+ if not author_id:
1950
+ continue
1951
+ openalex_id = (row.get("openalex_id") or "").strip() or None
1952
+ orcid = (row.get("orcid") or "").strip() or None
1953
+ if not openalex_id and not orcid:
1954
+ raise ValueError(
1955
+ f'Author external id row for {author_id} sets no identifier; '
1956
+ 'give an openalex_id or an orcid, or delete the row'
1957
+ )
1958
+ if author_id in out:
1959
+ raise ValueError(f'Duplicate author external id row for {author_id}')
1960
+ out[author_id] = {"openalex_id": openalex_id, "orcid": orcid}
1961
+ return out
1962
+
1963
+
1964
+ def _strip_comments(lines):
1965
+ for line in lines:
1966
+ if not line.lstrip().startswith("#"):
1967
+ yield line
1968
+
1969
+
1970
+ def load_aliases(path: Path | str | None) -> dict[str, str]:
1971
+ """Load a ``cluster_id,canonical_id`` CSV into a dict.
1972
+
1973
+ Empty / missing path returns ``{}``. Header row optional. Comments
1974
+ (lines starting with ``#``) are ignored.
1975
+ """
1976
+ if path is None:
1977
+ return {}
1978
+ p = Path(path)
1979
+ if not p.exists():
1980
+ return {}
1981
+ out: dict[str, str] = {}
1982
+ with p.open("r", encoding="utf-8") as f:
1983
+ for line in f:
1984
+ line = line.strip()
1985
+ if not line or line.startswith("#"):
1986
+ continue
1987
+ parts = [x.strip() for x in line.split(",")]
1988
+ if len(parts) < 2:
1989
+ continue
1990
+ src, dst = parts[0], parts[1]
1991
+ if src.lower() == "cluster_id" and dst.lower() == "canonical_id":
1992
+ continue # header
1993
+ if src and dst:
1994
+ if src in out and out[src] != dst:
1995
+ raise ValueError(f'Author alias conflict for {src}: {out[src]} versus {dst}')
1996
+ out[src] = dst
1997
+ return out