citegraph 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- citegraph/__init__.py +44 -0
- citegraph/_progress.py +74 -0
- citegraph/annotations.py +183 -0
- citegraph/author_export.py +294 -0
- citegraph/author_overrides.py +143 -0
- citegraph/authors.py +1997 -0
- citegraph/cli.py +926 -0
- citegraph/cocitation_stats.py +370 -0
- citegraph/config.py +73 -0
- citegraph/cost_estimation.py +259 -0
- citegraph/dedup.py +498 -0
- citegraph/enrich.py +972 -0
- citegraph/extract_metadata.py +81 -0
- citegraph/extract_references.py +376 -0
- citegraph/graph.py +660 -0
- citegraph/html_report.py +1787 -0
- citegraph/identity.py +199 -0
- citegraph/ids.py +106 -0
- citegraph/io.py +566 -0
- citegraph/jel.py +167 -0
- citegraph/journals.py +182 -0
- citegraph/llm.py +186 -0
- citegraph/pdf_to_markdown.py +394 -0
- citegraph/pipeline.py +997 -0
- citegraph/plotting.py +1214 -0
- citegraph/reports.py +214 -0
- citegraph/schemas.py +89 -0
- citegraph/webui.py +624 -0
- citegraph-0.2.0.dist-info/METADATA +870 -0
- citegraph-0.2.0.dist-info/RECORD +33 -0
- citegraph-0.2.0.dist-info/WHEEL +4 -0
- citegraph-0.2.0.dist-info/entry_points.txt +2 -0
- citegraph-0.2.0.dist-info/licenses/LICENSE +21 -0
citegraph/authors.py
ADDED
|
@@ -0,0 +1,1997 @@
|
|
|
1
|
+
"""Author parsing and corpus-level normalization.
|
|
2
|
+
|
|
3
|
+
The pipeline extracts author *strings* per reference, but a citation graph
|
|
4
|
+
is much more useful when those strings are clustered into canonical
|
|
5
|
+
authors so the user can answer "who is the most cited person across the
|
|
6
|
+
corpus?" or "show me every reference that cites Cárdenas".
|
|
7
|
+
|
|
8
|
+
Two layers live here:
|
|
9
|
+
|
|
10
|
+
1. :func:`parse_author` turns one raw string ("Cárdenas, J.-C.",
|
|
11
|
+
"Adele Diamond", "García Márquez, Gabriel") into a structured
|
|
12
|
+
:class:`ParsedAuthor` — surname / given-names / initials / suffix —
|
|
13
|
+
with a diacritic-stripped, lowercased ``surname_norm`` for blocking.
|
|
14
|
+
|
|
15
|
+
2. :func:`normalize_authors` clusters every parsed author across the
|
|
16
|
+
corpus into canonical author records. Parsing is evidence-driven: a
|
|
17
|
+
first pass over every raw string builds a lexicon of attested
|
|
18
|
+
multi-word surnames (comma forms, hyphenated and particle compounds,
|
|
19
|
+
enrichment family names), and a second pass re-parses comma-less
|
|
20
|
+
names against it, so "Jose Alberto Guerra Forero" adopts the
|
|
21
|
+
"Guerra Forero" boundary when — and only when — the corpus attests
|
|
22
|
+
it. The default clustering mode is precision-first: records are
|
|
23
|
+
bucketed by their full given-name *signature* (words + initials in
|
|
24
|
+
order) and a bucket joins a cluster only when its whole signature is
|
|
25
|
+
compatible with exactly one candidate (or a co-author overlap breaks
|
|
26
|
+
the tie); otherwise it is held aside as ambiguous. Word-vs-word
|
|
27
|
+
comparison tolerates near-identical OCR typos ("Camillo"/"Camilo")
|
|
28
|
+
but never prefix pairs like Gabriel/Gabriela. ``merge_mode="loose"``
|
|
29
|
+
collapses by ``(surname, first_initial)`` regardless of full-name
|
|
30
|
+
evidence — useful when you want recall over precision.
|
|
31
|
+
|
|
32
|
+
OpenAlex / ORCID identifiers (collected by the optional enrichment
|
|
33
|
+
stage; see ``enrich.py``) are treated as ground truth when present:
|
|
34
|
+
records that share an OpenAlex author id are merged regardless of
|
|
35
|
+
string evidence, and records with different OpenAlex ids are kept
|
|
36
|
+
separate even when their names look identical.
|
|
37
|
+
|
|
38
|
+
Hand-curated overrides (``aliases``) are the final escape hatch —
|
|
39
|
+
two-column CSV mapping arbitrary cluster ids to a canonical id, applied
|
|
40
|
+
after clustering so the user can fix anything the algorithm gets wrong.
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
from __future__ import annotations
|
|
44
|
+
|
|
45
|
+
import ast
|
|
46
|
+
import csv
|
|
47
|
+
import logging
|
|
48
|
+
import re
|
|
49
|
+
import unicodedata
|
|
50
|
+
from collections import defaultdict
|
|
51
|
+
from collections.abc import Iterable
|
|
52
|
+
from dataclasses import dataclass, field
|
|
53
|
+
from pathlib import Path
|
|
54
|
+
|
|
55
|
+
import pandas as pd
|
|
56
|
+
from rapidfuzz import fuzz
|
|
57
|
+
from slugify import slugify
|
|
58
|
+
|
|
59
|
+
from citegraph.io import (
|
|
60
|
+
AUTHOR_CITATION_COLUMNS,
|
|
61
|
+
AUTHOR_COLUMNS,
|
|
62
|
+
parse_authors_list,
|
|
63
|
+
require_columns,
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
logger = logging.getLogger(__name__)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
# Surname particles that are conventionally lowercased and form part of
|
|
70
|
+
# the surname rather than the given names ("García Márquez", "de la Cruz",
|
|
71
|
+
# "von Neumann"). When a name is given comma-first ("Márquez, Gabriel
|
|
72
|
+
# García") the parser already trusts the comma; this list only matters
|
|
73
|
+
# when there is no comma and we have to guess where the surname ends.
|
|
74
|
+
_PARTICLES = frozenset(
|
|
75
|
+
{
|
|
76
|
+
"de", "del", "la", "las", "los", "da", "das", "do", "dos",
|
|
77
|
+
"von", "van", "der", "den", "ten", "ter", "le", "lo",
|
|
78
|
+
"di", "du", "el", "al", "bin", "ibn", "abu",
|
|
79
|
+
}
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
# Honorific suffixes that should be stripped off the surname before
|
|
83
|
+
# comparing — "Jr.", "III", etc. Kept on the parsed record for display
|
|
84
|
+
# but not part of surname_norm.
|
|
85
|
+
_SUFFIX_PATTERN = re.compile(
|
|
86
|
+
r"\b(Jr\.?|Sr\.?|II|III|IV|PhD\.?|M\.?D\.?|Esq\.?)\b",
|
|
87
|
+
re.IGNORECASE,
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
# A run of letters within a token (used to decide whether a token is
|
|
91
|
+
# "essentially an initial": every contiguous letter run is one letter).
|
|
92
|
+
_LETTER_RUN = re.compile(r"[A-Za-zÀ-ÖØ-öø-ÿ]+")
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _is_initial_only_token(token: str) -> bool:
|
|
96
|
+
"""True when every alphabetic run inside ``token`` is a single letter.
|
|
97
|
+
|
|
98
|
+
Catches all the citation-style spellings of initials we see in the
|
|
99
|
+
wild: ``"J."``, ``"J.C."``, ``"J. C."`` (each token), ``"J.-C."``.
|
|
100
|
+
Multi-letter words like ``"Juan"`` correctly come back ``False``.
|
|
101
|
+
"""
|
|
102
|
+
runs = _LETTER_RUN.findall(token)
|
|
103
|
+
return bool(runs) and all(len(r) == 1 for r in runs)
|
|
104
|
+
|
|
105
|
+
# Stuff the LLM occasionally leaves in author strings — footnote markers,
|
|
106
|
+
# affiliation glyphs, stray asterisks, leading "and"/"&", trailing
|
|
107
|
+
# punctuation. Stripped before parsing.
|
|
108
|
+
_NOISE = re.compile(r"[†‡§¶*★]+|^\s*(and|&)\s+", re.IGNORECASE)
|
|
109
|
+
|
|
110
|
+
# 'et al.' glued onto a real name ("B. Puranen et al.") — the bare forms
|
|
111
|
+
# are caught earlier by the exact-string check in parse_author.
|
|
112
|
+
_ET_AL_TRAILING = re.compile(r"[,\s]+et\s+al\.?\s*$", re.IGNORECASE)
|
|
113
|
+
|
|
114
|
+
# OCR-mangled accents: PDF extraction renders 'Bénabou' as "B ' enabou"
|
|
115
|
+
# and 'Ibáñez' as "Ib ' a ˜ nez". An apostrophe-like or floating accent
|
|
116
|
+
# mark with whitespace on *both* sides is never part of a real name
|
|
117
|
+
# (genuine spaced apostrophes — Dutch "van 't Hoff", quoted nicknames —
|
|
118
|
+
# touch a letter on one side), so the mark and its spaces are removed to
|
|
119
|
+
# rejoin the letter runs.
|
|
120
|
+
_OCR_SPACED_MARK = re.compile(r"\s+['’´ʼ`˜~̃]\s+")
|
|
121
|
+
|
|
122
|
+
# Bare uppercase initials run as one token — the Vancouver / NLM citation
|
|
123
|
+
# style writes "Cardenas JC" (surname first, initials last, no comma).
|
|
124
|
+
# Accented uppercase included so "Ibáñez ÁC" is recognized too.
|
|
125
|
+
_BARE_CAPS_INITIALS = re.compile(r"[A-ZÀ-ÖØ-Þ]{1,3}")
|
|
126
|
+
|
|
127
|
+
# Words that mark an author string as an institution rather than a person.
|
|
128
|
+
# Deliberately conservative: every entry is a word that essentially never
|
|
129
|
+
# appears in a personal name. Matched on diacritic-stripped lowercase
|
|
130
|
+
# tokens, and only for comma-less multi-token strings (comma forms are
|
|
131
|
+
# person-shaped), so "Banks, J." never trips it.
|
|
132
|
+
_CORPORATE_WORDS = frozenset(
|
|
133
|
+
{
|
|
134
|
+
"university", "universidad", "universidade", "universite",
|
|
135
|
+
"institute", "instituto", "institution", "ministry", "ministerio",
|
|
136
|
+
"department", "departamento", "association", "asociacion",
|
|
137
|
+
"bank", "banco", "council", "commission", "comision", "committee",
|
|
138
|
+
"organization", "organisation", "organizacion", "agency", "agencia",
|
|
139
|
+
"foundation", "fundacion", "fondo", "center", "centre", "centro",
|
|
140
|
+
"office", "oficina", "bureau", "society", "sociedad", "nations",
|
|
141
|
+
"congreso", "congress", "senado", "senate", "alcaldia",
|
|
142
|
+
"gobernacion", "secretaria", "authority", "administration",
|
|
143
|
+
"administracion", "programme", "observatory", "observatorio",
|
|
144
|
+
"consortium", "assessment", "corporation", "grupo", "world",
|
|
145
|
+
}
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
# "(NRC)", "(USAID)" — a parenthesised all-caps acronym is an
|
|
149
|
+
# organization signature regardless of the surrounding shape.
|
|
150
|
+
_ACRONYM_PARENS = re.compile(r"\([A-Z]{2,}\)")
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _strip_diacritics(text: str) -> str:
|
|
154
|
+
"""NFD-decompose and drop combining marks: 'Cárdenas' -> 'Cardenas'."""
|
|
155
|
+
return "".join(
|
|
156
|
+
ch for ch in unicodedata.normalize("NFD", text)
|
|
157
|
+
if unicodedata.category(ch) != "Mn"
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
# Providers spell hyphenated names with typographic dashes: OpenAlex returns
|
|
162
|
+
# "Villegas‐Palacio" (U+2010 HYPHEN), not an ASCII "-". They are the same
|
|
163
|
+
# character to a reader and must be the same to the normaliser, or one person
|
|
164
|
+
# gets two blocking keys and no provider id ever attaches. The soft hyphen is an
|
|
165
|
+
# invisible line-break hint, so it is deleted rather than turned into a hyphen.
|
|
166
|
+
_UNICODE_DASHES = str.maketrans({
|
|
167
|
+
"\u2010": "-", "\u2011": "-", "\u2012": "-", "\u2013": "-",
|
|
168
|
+
"\u2014": "-", "\u2015": "-", "\u2212": "-", "\u00ad": "",
|
|
169
|
+
})
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _fold_dashes(text: str) -> str:
|
|
173
|
+
"""Map typographic dashes onto the ASCII hyphen the parser understands."""
|
|
174
|
+
return text.translate(_UNICODE_DASHES)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _norm_surname(surname: str) -> str:
|
|
178
|
+
"""Lowercase + diacritic-strip + drop non-alphabetic characters.
|
|
179
|
+
|
|
180
|
+
Spaces are kept so multi-word surnames ('garcia marquez') can be
|
|
181
|
+
compared by full normalised form. The blocking step uses this as the
|
|
182
|
+
bucket key, so it MUST be deterministic across runs.
|
|
183
|
+
"""
|
|
184
|
+
s = _strip_diacritics(_fold_dashes(surname)).lower().strip()
|
|
185
|
+
# Hyphens are spelling variance, not structure: 'Casas-Casas' and
|
|
186
|
+
# 'Casas Casas' must produce the same blocking key.
|
|
187
|
+
s = s.replace("-", " ")
|
|
188
|
+
# Keep letters and internal spaces; drop apostrophes, periods, etc.
|
|
189
|
+
s = re.sub(r"[^a-z\s]", "", s)
|
|
190
|
+
s = re.sub(r"\s+", " ", s).strip()
|
|
191
|
+
return s
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _initials_from_token(token: str) -> str:
|
|
195
|
+
"""Extract initial letters from a given-name token.
|
|
196
|
+
|
|
197
|
+
Handles every common spelling of multi-initial citations:
|
|
198
|
+
``'J.'`` -> ``'J'`` ; ``'J.C.'`` -> ``'JC'`` ; ``'J.-C.'`` -> ``'JC'`` ;
|
|
199
|
+
``'Juan-Camilo'`` -> ``'JC'`` ; ``'Juan'`` -> ``'J'`` ; ``''`` -> ``''``.
|
|
200
|
+
|
|
201
|
+
The implementation walks every contiguous letter run and takes its
|
|
202
|
+
first letter — so it doesn't matter whether the segments are joined
|
|
203
|
+
by periods, hyphens, or whitespace.
|
|
204
|
+
"""
|
|
205
|
+
if not token:
|
|
206
|
+
return ""
|
|
207
|
+
return "".join(run[0].upper() for run in _LETTER_RUN.findall(token))
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
@dataclass(frozen=True)
|
|
211
|
+
class ParsedAuthor:
|
|
212
|
+
"""Structured representation of a single author string.
|
|
213
|
+
|
|
214
|
+
Attributes
|
|
215
|
+
----------
|
|
216
|
+
raw:
|
|
217
|
+
The original string as it appeared in the citation, untouched.
|
|
218
|
+
surname:
|
|
219
|
+
The surname with original casing and diacritics, used for display.
|
|
220
|
+
surname_norm:
|
|
221
|
+
Diacritic-stripped, lowercased, alphabetic-only surname. The
|
|
222
|
+
clustering step blocks on this.
|
|
223
|
+
given_names:
|
|
224
|
+
List of given-name tokens in citation order ("Juan", "Camilo")
|
|
225
|
+
or initials ("J.", "C."). Empty when only a surname was given.
|
|
226
|
+
initials:
|
|
227
|
+
Concatenated first letters of every given-name token, uppercased.
|
|
228
|
+
"Juan-Camilo" -> "JC"; "J. C." -> "JC".
|
|
229
|
+
has_full_first:
|
|
230
|
+
``True`` if the first given-name token is a real word, not just
|
|
231
|
+
an initial. The clustering step uses this to anchor clusters.
|
|
232
|
+
suffix:
|
|
233
|
+
"Jr.", "III", etc. Preserved for display; ignored by matching.
|
|
234
|
+
is_corporate:
|
|
235
|
+
``True`` when the string names an institution ("The World Bank")
|
|
236
|
+
rather than a person. The whole string is kept as the surname and
|
|
237
|
+
such records only ever cluster with identical normalized strings.
|
|
238
|
+
"""
|
|
239
|
+
|
|
240
|
+
raw: str
|
|
241
|
+
surname: str
|
|
242
|
+
surname_norm: str
|
|
243
|
+
given_names: tuple[str, ...]
|
|
244
|
+
initials: str
|
|
245
|
+
has_full_first: bool
|
|
246
|
+
suffix: str | None = None
|
|
247
|
+
is_corporate: bool = False
|
|
248
|
+
|
|
249
|
+
@property
|
|
250
|
+
def first_initial(self) -> str:
|
|
251
|
+
return self.initials[:1]
|
|
252
|
+
|
|
253
|
+
@property
|
|
254
|
+
def first_given_norm(self) -> str:
|
|
255
|
+
"""Normalised form of the first given-name token, '' if only initial."""
|
|
256
|
+
if not self.has_full_first or not self.given_names:
|
|
257
|
+
return ""
|
|
258
|
+
return _strip_diacritics(self.given_names[0]).lower().strip(".-")
|
|
259
|
+
|
|
260
|
+
def display(self) -> str:
|
|
261
|
+
"""Best human-readable form of this name (surname-first)."""
|
|
262
|
+
if self.given_names:
|
|
263
|
+
given = " ".join(self.given_names)
|
|
264
|
+
return f"{self.surname}, {given}"
|
|
265
|
+
return self.surname
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def parse_author(
|
|
269
|
+
raw: str,
|
|
270
|
+
*,
|
|
271
|
+
known_surnames: Iterable[str] | None = None,
|
|
272
|
+
) -> ParsedAuthor | None:
|
|
273
|
+
"""Parse a single raw author string into a :class:`ParsedAuthor`.
|
|
274
|
+
|
|
275
|
+
Returns ``None`` for strings that contain no usable surname (empty
|
|
276
|
+
string, pure punctuation, "et al.", "Anonymous", etc.). Callers
|
|
277
|
+
should treat ``None`` as "skip this author".
|
|
278
|
+
|
|
279
|
+
``known_surnames`` is an optional set of *normalized* multi-word
|
|
280
|
+
surnames already attested elsewhere (comma forms, hyphenated forms,
|
|
281
|
+
enrichment family names). When given, comma-less names whose token
|
|
282
|
+
suffix matches an attested compound surname are split at that
|
|
283
|
+
boundary instead of the conservative last-token guess — this is how
|
|
284
|
+
"Jose Alberto Guerra Forero" learns its surname is "Guerra Forero"
|
|
285
|
+
from a "Guerra Forero, J.A." citation elsewhere in the corpus.
|
|
286
|
+
"""
|
|
287
|
+
if not isinstance(raw, str):
|
|
288
|
+
return None
|
|
289
|
+
# Fold dashes before anything inspects the string: the compound-surname
|
|
290
|
+
# checks test for a literal "-", so a typographic one would read as an
|
|
291
|
+
# unhyphenated name. ``raw`` keeps the original spelling for display.
|
|
292
|
+
s = _fold_dashes(raw).strip()
|
|
293
|
+
if not s:
|
|
294
|
+
return None
|
|
295
|
+
|
|
296
|
+
# Drop common noise tokens that signal "not an author".
|
|
297
|
+
bare = _strip_diacritics(s).lower()
|
|
298
|
+
if bare in {"et al", "et al.", "anonymous", "anon", "n.a.", "na", "unknown"}:
|
|
299
|
+
return None
|
|
300
|
+
|
|
301
|
+
# Strip footnote/affiliation glyphs and a leading 'and'/'&'.
|
|
302
|
+
s = _NOISE.sub(" ", s).strip()
|
|
303
|
+
# Rejoin OCR-mangled accents ("B ' enabou" -> "Benabou") before any
|
|
304
|
+
# tokenization; a spaced mark would otherwise split the surname.
|
|
305
|
+
s = _OCR_SPACED_MARK.sub("", s)
|
|
306
|
+
# 'et al.' glued onto a real name keeps the name, drops the tail.
|
|
307
|
+
s = _ET_AL_TRAILING.sub("", s)
|
|
308
|
+
s = re.sub(r"\s+", " ", s).strip(" ,")
|
|
309
|
+
if not s:
|
|
310
|
+
return None
|
|
311
|
+
|
|
312
|
+
# Institutions are not person-parsed: the whole string is the name.
|
|
313
|
+
if _is_corporate_author(s):
|
|
314
|
+
return ParsedAuthor(
|
|
315
|
+
raw=raw,
|
|
316
|
+
surname=s,
|
|
317
|
+
surname_norm=_norm_surname(s),
|
|
318
|
+
given_names=(),
|
|
319
|
+
initials="",
|
|
320
|
+
has_full_first=False,
|
|
321
|
+
suffix=None,
|
|
322
|
+
is_corporate=True,
|
|
323
|
+
)
|
|
324
|
+
|
|
325
|
+
# For persons, a parenthetical is an affiliation ("Smith, J. (MIT)"),
|
|
326
|
+
# never part of the name.
|
|
327
|
+
s = re.sub(r"\s*\([^)]*\)", " ", s)
|
|
328
|
+
s = re.sub(r"\s+", " ", s).strip(" ,")
|
|
329
|
+
if not s:
|
|
330
|
+
return None
|
|
331
|
+
|
|
332
|
+
# Pull off honorific suffix if present.
|
|
333
|
+
suffix: str | None = None
|
|
334
|
+
m = _SUFFIX_PATTERN.search(s)
|
|
335
|
+
if m:
|
|
336
|
+
suffix = m.group(1)
|
|
337
|
+
s = (s[: m.start()] + s[m.end():]).strip(" ,")
|
|
338
|
+
|
|
339
|
+
# Two parsing branches: comma-style (Surname, Given) vs space-style.
|
|
340
|
+
if "," in s:
|
|
341
|
+
surname_part, given_part = s.split(",", 1)
|
|
342
|
+
surname = surname_part.strip()
|
|
343
|
+
given_str = given_part.strip()
|
|
344
|
+
else:
|
|
345
|
+
reversed_split = _split_reversed_citation(s.split())
|
|
346
|
+
if reversed_split is not None:
|
|
347
|
+
# Vancouver / NLM style: "Guerra JA" is surname-then-initials.
|
|
348
|
+
surname, given_str = reversed_split
|
|
349
|
+
else:
|
|
350
|
+
surname, given_str = _split_no_comma(s, known_surnames)
|
|
351
|
+
|
|
352
|
+
surname = surname.strip(" .-")
|
|
353
|
+
if not surname or not re.search(r"[A-Za-zÀ-ÖØ-öø-ÿ]", surname):
|
|
354
|
+
return None
|
|
355
|
+
# A "surname" whose every letter run is a single letter is almost
|
|
356
|
+
# certainly a misparsed initial ("J.", "X. Y."). Real surnames have
|
|
357
|
+
# at least one multi-letter run. Rejecting here prevents the mega-
|
|
358
|
+
# clusters of one-letter surnames that arise when an upstream stage
|
|
359
|
+
# has comma-split "Smith, J., García, A." into separate fragments.
|
|
360
|
+
if _is_initial_only_token(surname):
|
|
361
|
+
return None
|
|
362
|
+
|
|
363
|
+
given_tokens = _tokenize_given(given_str)
|
|
364
|
+
initials = "".join(_initials_from_token(t) for t in given_tokens)
|
|
365
|
+
has_full_first = bool(given_tokens) and not _is_initial_only_token(given_tokens[0])
|
|
366
|
+
|
|
367
|
+
return ParsedAuthor(
|
|
368
|
+
raw=raw,
|
|
369
|
+
surname=surname,
|
|
370
|
+
surname_norm=_norm_surname(surname),
|
|
371
|
+
given_names=tuple(given_tokens),
|
|
372
|
+
initials=initials,
|
|
373
|
+
has_full_first=has_full_first,
|
|
374
|
+
suffix=suffix,
|
|
375
|
+
)
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def _split_no_comma(
|
|
379
|
+
s: str,
|
|
380
|
+
known_surnames: Iterable[str] | None = None,
|
|
381
|
+
) -> tuple[str, str]:
|
|
382
|
+
"""Split a name with no comma into (surname, given-string).
|
|
383
|
+
|
|
384
|
+
Heuristic: the last token is the surname, *unless* the second-to-last
|
|
385
|
+
token is a particle ('de', 'van', 'García' before 'Márquez'…), in
|
|
386
|
+
which case the surname extends backwards to include it. Compound
|
|
387
|
+
surnames without particles ('García Márquez' alone, no comma) are
|
|
388
|
+
ambiguous — we can't tell from the string alone, and we conservatively
|
|
389
|
+
take only the last token as the surname *unless* ``known_surnames``
|
|
390
|
+
attests the compound (see :func:`parse_author`).
|
|
391
|
+
"""
|
|
392
|
+
tokens = s.split()
|
|
393
|
+
if not tokens:
|
|
394
|
+
return "", ""
|
|
395
|
+
if len(tokens) == 1:
|
|
396
|
+
return tokens[0], ""
|
|
397
|
+
|
|
398
|
+
# Corpus-evidence pass: adopt a multi-token surname the corpus has
|
|
399
|
+
# already attested. Longest suffix wins; a whole-string match means
|
|
400
|
+
# the string is a bare compound surname with no given names.
|
|
401
|
+
if known_surnames:
|
|
402
|
+
for k in range(min(4, len(tokens)), 1, -1):
|
|
403
|
+
cand = _norm_surname(" ".join(tokens[-k:]))
|
|
404
|
+
if cand in known_surnames:
|
|
405
|
+
if k == len(tokens):
|
|
406
|
+
return s, ""
|
|
407
|
+
return " ".join(tokens[-k:]), " ".join(tokens[:-k])
|
|
408
|
+
|
|
409
|
+
surname_idx = len(tokens) - 1
|
|
410
|
+
# Walk backwards while the preceding token looks like a particle.
|
|
411
|
+
while surname_idx > 0:
|
|
412
|
+
prev_raw = tokens[surname_idx - 1]
|
|
413
|
+
prev = prev_raw.strip(".-").lower()
|
|
414
|
+
if prev in _PARTICLES:
|
|
415
|
+
surname_idx -= 1
|
|
416
|
+
elif (
|
|
417
|
+
prev_raw == "y"
|
|
418
|
+
and surname_idx >= 2
|
|
419
|
+
and not _is_initial_only_token(tokens[surname_idx - 2])
|
|
420
|
+
):
|
|
421
|
+
# Spanish surname connector: 'Ortega y Gasset'. Only the bare
|
|
422
|
+
# lowercase word counts — a 'Y.' initial must not trigger it.
|
|
423
|
+
surname_idx -= 2
|
|
424
|
+
else:
|
|
425
|
+
break
|
|
426
|
+
|
|
427
|
+
surname = " ".join(tokens[surname_idx:])
|
|
428
|
+
given = " ".join(tokens[:surname_idx])
|
|
429
|
+
return surname, given
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def _split_reversed_citation(tokens: list[str]) -> tuple[str, str] | None:
|
|
433
|
+
"""Detect Vancouver-style 'Surname(s) AB' ordering; ``None`` if absent.
|
|
434
|
+
|
|
435
|
+
The trailing run of initials-like tokens ("JA", "J.A.", "J") becomes
|
|
436
|
+
the given part and everything before it the surname — but only when
|
|
437
|
+
no leading token is a single-letter initial, so "A. Diamond"
|
|
438
|
+
(given-first) is left alone. All-caps leading tokens are fine ("LEE
|
|
439
|
+
KY", "CAMERON L"). Bare caps runs are expanded ("JA" -> "J. A.") so
|
|
440
|
+
the initials extraction sees one letter per run.
|
|
441
|
+
"""
|
|
442
|
+
|
|
443
|
+
def is_initials_like(t: str) -> bool:
|
|
444
|
+
return _is_initial_only_token(t) or bool(_BARE_CAPS_INITIALS.fullmatch(t))
|
|
445
|
+
|
|
446
|
+
# Consume the trailing initials run, but never the first token — in
|
|
447
|
+
# "LI X" the all-caps "LI" is the surname, not more initials.
|
|
448
|
+
i = len(tokens)
|
|
449
|
+
while i > 1 and is_initials_like(tokens[i - 1]):
|
|
450
|
+
i -= 1
|
|
451
|
+
if i == len(tokens):
|
|
452
|
+
return None
|
|
453
|
+
leading = tokens[:i]
|
|
454
|
+
if any(_is_initial_only_token(t) for t in leading):
|
|
455
|
+
return None
|
|
456
|
+
trailing = tokens[i:]
|
|
457
|
+
trailing_letters = sum(len(run) for t in trailing for run in _LETTER_RUN.findall(t))
|
|
458
|
+
if len(leading) == 1 and len(leading[0]) <= 2 and trailing_letters >= 2:
|
|
459
|
+
# 'Bo XU' vs 'Ma JA' is genuinely ambiguous (caps surname vs
|
|
460
|
+
# Vancouver initials); keep the conservative last-token parse.
|
|
461
|
+
return None
|
|
462
|
+
given_tokens = [
|
|
463
|
+
" ".join(f"{c}." for c in t) if _BARE_CAPS_INITIALS.fullmatch(t) else t
|
|
464
|
+
for t in trailing
|
|
465
|
+
]
|
|
466
|
+
return " ".join(leading), " ".join(given_tokens)
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def _is_corporate_author(s: str) -> bool:
|
|
470
|
+
"""True when the string names an institution rather than a person."""
|
|
471
|
+
if "," in s:
|
|
472
|
+
# Comma forms are person-shaped ("Banks, J.", "Smith, J. (MIT)").
|
|
473
|
+
return False
|
|
474
|
+
if _ACRONYM_PARENS.search(s):
|
|
475
|
+
return True
|
|
476
|
+
tokens = re.findall(r"[^\W\d_]+", _strip_diacritics(s).lower())
|
|
477
|
+
if len(tokens) < 2:
|
|
478
|
+
return False
|
|
479
|
+
if not any(t in _CORPORATE_WORDS for t in tokens):
|
|
480
|
+
return False
|
|
481
|
+
# Two-token strings only count when the corporate word leads
|
|
482
|
+
# ("Fundación Natura", "World Bank") — a trailing corporate word may
|
|
483
|
+
# be a real surname ("Steven Bank").
|
|
484
|
+
return len(tokens) >= 3 or tokens[0] in _CORPORATE_WORDS
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
# Below this length a token is too short to absorb a typo without pairing
|
|
488
|
+
# unrelated people ("Lee"/"Loe"), so it must match exactly.
|
|
489
|
+
_AGREEMENT_FUZZ_MIN_LEN = 5
|
|
490
|
+
|
|
491
|
+
# Providers spell apostrophes typographically ("D’Adda") where
|
|
492
|
+
# bibliographies use ASCII ("D'Adda"); both must fold to one token, the same
|
|
493
|
+
# way :func:`_fold_dashes` handles the unicode dashes OpenAlex emits. The
|
|
494
|
+
# OCR-spaced form ("Ca ' rdenas") folds here too.
|
|
495
|
+
_APOSTROPHES = ("'", "’", "‘", "ʼ", "´", "`")
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
def author_list_agreement(
|
|
499
|
+
extracted: list[str],
|
|
500
|
+
provider: list[str],
|
|
501
|
+
*,
|
|
502
|
+
fuzz_threshold: float = 85.0,
|
|
503
|
+
) -> bool | None:
|
|
504
|
+
"""Do two author lists name any of the same people?
|
|
505
|
+
|
|
506
|
+
``True`` when at least one name pairs, ``False`` when both sides carry
|
|
507
|
+
usable person names and none pair, ``None`` to abstain.
|
|
508
|
+
|
|
509
|
+
Comparison is over order-free name tokens rather than
|
|
510
|
+
:attr:`ParsedAuthor.surname_norm`, because the surname path needs the
|
|
511
|
+
corpus-wide compound-surname lexicon and that does not exist yet when
|
|
512
|
+
enrichment runs. Without it every comma-less provider name collapses to
|
|
513
|
+
its last token, so "Moros, L." and "Lina Moros Canon" stop matching.
|
|
514
|
+
Tokens pair on equality, or on ``fuzz_threshold`` similarity when long
|
|
515
|
+
enough to survive it, which is what lets a bibliography's
|
|
516
|
+
"Hirschmann, A." reach the provider's "Albert O. Hirschman".
|
|
517
|
+
|
|
518
|
+
Abstains whenever either side names an institution, since a provider
|
|
519
|
+
record for a corporate-authored report legitimately lists the human
|
|
520
|
+
chapter authors instead ("IFPRI" -> "Fredrick O. Wanyama").
|
|
521
|
+
"""
|
|
522
|
+
left = _agreement_tokens(extracted)
|
|
523
|
+
right = _agreement_tokens(provider)
|
|
524
|
+
if left is None or right is None:
|
|
525
|
+
return None
|
|
526
|
+
for a in left:
|
|
527
|
+
for b in right:
|
|
528
|
+
if a == b:
|
|
529
|
+
return True
|
|
530
|
+
if (
|
|
531
|
+
len(a) >= _AGREEMENT_FUZZ_MIN_LEN
|
|
532
|
+
and len(b) >= _AGREEMENT_FUZZ_MIN_LEN
|
|
533
|
+
and fuzz.ratio(a, b) >= fuzz_threshold
|
|
534
|
+
):
|
|
535
|
+
return True
|
|
536
|
+
return False
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
def _is_acronym_author(s: str) -> bool:
|
|
540
|
+
"""True for a bare all-caps acronym ("IFPRI", "UNFCCC", "IPCC").
|
|
541
|
+
|
|
542
|
+
:func:`_is_corporate_author` needs two tokens to fire, so acronyms slip
|
|
543
|
+
past it. They are institutions all the same.
|
|
544
|
+
"""
|
|
545
|
+
stripped = s.strip()
|
|
546
|
+
return (
|
|
547
|
+
len(stripped) >= 3
|
|
548
|
+
and stripped.isupper()
|
|
549
|
+
and stripped.isalpha()
|
|
550
|
+
and " " not in stripped
|
|
551
|
+
)
|
|
552
|
+
|
|
553
|
+
|
|
554
|
+
def _agreement_tokens(names: list[str]) -> set[str] | None:
|
|
555
|
+
"""Order-free name tokens, or ``None`` when there is nothing to compare."""
|
|
556
|
+
tokens: set[str] = set()
|
|
557
|
+
for raw in names:
|
|
558
|
+
if not isinstance(raw, str) or not raw.strip():
|
|
559
|
+
continue
|
|
560
|
+
if _is_corporate_author(raw) or _is_acronym_author(raw):
|
|
561
|
+
return None
|
|
562
|
+
cleaned = _strip_diacritics(_fold_dashes(raw))
|
|
563
|
+
for apostrophe in _APOSTROPHES:
|
|
564
|
+
cleaned = cleaned.replace(apostrophe, "")
|
|
565
|
+
for token in re.split(r"[^A-Za-z]+", cleaned):
|
|
566
|
+
lowered = token.lower()
|
|
567
|
+
if len(token) > 2 and lowered not in _PARTICLES:
|
|
568
|
+
tokens.add(lowered)
|
|
569
|
+
return tokens or None
|
|
570
|
+
|
|
571
|
+
|
|
572
|
+
def _tokenize_given(given_str: str) -> list[str]:
|
|
573
|
+
"""Split a given-name string into tokens, preserving 'J.-C.' as one."""
|
|
574
|
+
if not given_str:
|
|
575
|
+
return []
|
|
576
|
+
# Split on whitespace; keep hyphenated initials intact.
|
|
577
|
+
parts = [p for p in given_str.split() if p]
|
|
578
|
+
return parts
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
# ---------------------------------------------------------------------------
|
|
582
|
+
# Clustering
|
|
583
|
+
# ---------------------------------------------------------------------------
|
|
584
|
+
|
|
585
|
+
|
|
586
|
+
@dataclass
|
|
587
|
+
class AuthorOccurrence:
|
|
588
|
+
"""One concrete appearance of an author on a canonical work.
|
|
589
|
+
|
|
590
|
+
The clustering output is grouped by canonical author id; each
|
|
591
|
+
occurrence keeps the work id so callers can join back onto
|
|
592
|
+
``works.csv`` (for rings) and ``citation_graph.csv`` (for citers).
|
|
593
|
+
"""
|
|
594
|
+
|
|
595
|
+
parsed: ParsedAuthor
|
|
596
|
+
record_id: str # 'w-…' canonical work id
|
|
597
|
+
position: int # 0-based index in the author list
|
|
598
|
+
co_author_keys: tuple[str, ...] = () # surname_norm of the other authors on the same record
|
|
599
|
+
openalex_id: str | None = None
|
|
600
|
+
orcid: str | None = None
|
|
601
|
+
cannot_link: set[tuple[str, int]] = field(default_factory=set)
|
|
602
|
+
ambiguous_identity_links: set[tuple[str, int]] = field(default_factory=set)
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
@dataclass
|
|
606
|
+
class AuthorClusterConfig:
|
|
607
|
+
"""Tunable knobs for :func:`normalize_authors`."""
|
|
608
|
+
|
|
609
|
+
# 'strict' (default) — precision-first. Merges 'Diamond, A.' into
|
|
610
|
+
# 'Diamond, Adele' only when 'Adele' is the *only* full first name
|
|
611
|
+
# observed for surname 'Diamond' in this corpus. Otherwise the
|
|
612
|
+
# initial-only record is held aside in its own cluster and flagged.
|
|
613
|
+
#
|
|
614
|
+
# 'loose' — recall-first. Collapses every (surname, first_initial)
|
|
615
|
+
# combination into one cluster regardless of full-name disagreement.
|
|
616
|
+
merge_mode: str = "strict"
|
|
617
|
+
|
|
618
|
+
|
|
619
|
+
@dataclass
|
|
620
|
+
class AuthorCluster:
|
|
621
|
+
"""One canonical author after corpus-wide normalization."""
|
|
622
|
+
|
|
623
|
+
id: str
|
|
624
|
+
display_name: str
|
|
625
|
+
surname: str
|
|
626
|
+
surname_norm: str
|
|
627
|
+
canonical_given: str
|
|
628
|
+
initials: str
|
|
629
|
+
openalex_id: str | None
|
|
630
|
+
orcid: str | None
|
|
631
|
+
occurrences: list[AuthorOccurrence] = field(default_factory=list)
|
|
632
|
+
review_reason: str | None = None # populated for low-confidence clusters
|
|
633
|
+
|
|
634
|
+
@property
|
|
635
|
+
def n_occurrences(self) -> int:
|
|
636
|
+
return len(self.occurrences)
|
|
637
|
+
|
|
638
|
+
|
|
639
|
+
def _make_author_id(surname_norm: str, given_token: str, n: int = 0) -> str:
|
|
640
|
+
"""Stable slug-style id for an author cluster.
|
|
641
|
+
|
|
642
|
+
``n`` is appended only to disambiguate within a corpus when the same
|
|
643
|
+
(surname, given-names) tuple genuinely refers to two distinct people
|
|
644
|
+
that the algorithm split apart. Corporate authors have no given
|
|
645
|
+
token and slug to just the normalized name.
|
|
646
|
+
"""
|
|
647
|
+
base = f"a-{slugify(surname_norm) or 'unknown'}"
|
|
648
|
+
given_slug = slugify(given_token) if given_token else ""
|
|
649
|
+
if given_slug:
|
|
650
|
+
base = f"{base}-{given_slug}"
|
|
651
|
+
if n:
|
|
652
|
+
base = f"{base}-{n}"
|
|
653
|
+
return base
|
|
654
|
+
|
|
655
|
+
|
|
656
|
+
def _given_units(parsed: ParsedAuthor) -> tuple[tuple[str, str], ...]:
|
|
657
|
+
"""Flatten given names into comparable units.
|
|
658
|
+
|
|
659
|
+
Each contiguous letter run becomes one unit: a full word ``("w",
|
|
660
|
+
"juan")`` or an initial ``("i", "j")``. Hyphens and periods dissolve,
|
|
661
|
+
so "Juan-Camilo", "Juan Camilo", and "J.C." become comparable:
|
|
662
|
+
``[w:juan, w:camilo]`` / ``[w:juan, w:camilo]`` / ``[i:j, i:c]``.
|
|
663
|
+
"""
|
|
664
|
+
units: list[tuple[str, str]] = []
|
|
665
|
+
for tok in parsed.given_names:
|
|
666
|
+
for run in _LETTER_RUN.findall(tok):
|
|
667
|
+
kind = "i" if len(run) == 1 else "w"
|
|
668
|
+
units.append((kind, _strip_diacritics(run).lower()))
|
|
669
|
+
return tuple(units)
|
|
670
|
+
|
|
671
|
+
|
|
672
|
+
def _w_equivalent(a: str, b: str) -> bool:
|
|
673
|
+
"""True when two full given-name words plausibly name the same person.
|
|
674
|
+
|
|
675
|
+
Exact match, or a near-identical spelling that reads as an OCR /
|
|
676
|
+
extraction typo ('camilo' vs 'camillo'): both words at least 4
|
|
677
|
+
letters, rapidfuzz ratio >= 90, and *neither a prefix of the other* —
|
|
678
|
+
the prefix guard is what keeps gendered pairs like Gabriel/Gabriela
|
|
679
|
+
and Daniel/Daniela (ratio > 90) firmly apart.
|
|
680
|
+
"""
|
|
681
|
+
if a == b:
|
|
682
|
+
return True
|
|
683
|
+
if len(a) < 4 or len(b) < 4:
|
|
684
|
+
return False
|
|
685
|
+
if a.startswith(b) or b.startswith(a):
|
|
686
|
+
return False
|
|
687
|
+
return fuzz.ratio(a, b) >= 90
|
|
688
|
+
|
|
689
|
+
|
|
690
|
+
def _units_compatible(
|
|
691
|
+
a: tuple[tuple[str, str], ...],
|
|
692
|
+
b: tuple[tuple[str, str], ...],
|
|
693
|
+
) -> bool:
|
|
694
|
+
"""True when two given-name unit sequences could name the same person.
|
|
695
|
+
|
|
696
|
+
Position-by-position over the shared prefix: word vs word must be
|
|
697
|
+
equivalent (exact or typo-close, see :func:`_w_equivalent`), initial
|
|
698
|
+
vs initial must be equal, and an initial matches a word starting
|
|
699
|
+
with it. The *whole* overlapping sequence must agree — this is what
|
|
700
|
+
keeps "J.P." out of a "Juan Camilo" cluster even though the first
|
|
701
|
+
initials match.
|
|
702
|
+
"""
|
|
703
|
+
# strict=False is the point: only the shared prefix is compared.
|
|
704
|
+
for (ka, va), (kb, vb) in zip(a, b, strict=False):
|
|
705
|
+
if ka == kb:
|
|
706
|
+
if ka == "w":
|
|
707
|
+
if not _w_equivalent(va, vb):
|
|
708
|
+
return False
|
|
709
|
+
elif va != vb:
|
|
710
|
+
return False
|
|
711
|
+
else:
|
|
712
|
+
word = va if ka == "w" else vb
|
|
713
|
+
initial = vb if ka == "w" else va
|
|
714
|
+
if not word.startswith(initial):
|
|
715
|
+
return False
|
|
716
|
+
return True
|
|
717
|
+
|
|
718
|
+
|
|
719
|
+
def _units_informativeness(units: tuple[tuple[str, str], ...]) -> tuple[int, int]:
|
|
720
|
+
return (sum(1 for kind, _ in units if kind == "w"), len(units))
|
|
721
|
+
|
|
722
|
+
|
|
723
|
+
def _cluster_best_units(
|
|
724
|
+
cluster: list[AuthorOccurrence],
|
|
725
|
+
) -> tuple[tuple[str, str], ...]:
|
|
726
|
+
"""The most informative given-name signature observed in a cluster."""
|
|
727
|
+
best: tuple[tuple[str, str], ...] = ()
|
|
728
|
+
best_key = (-1, -1)
|
|
729
|
+
for o in cluster:
|
|
730
|
+
units = _given_units(o.parsed)
|
|
731
|
+
key = _units_informativeness(units)
|
|
732
|
+
if key > best_key:
|
|
733
|
+
best, best_key = units, key
|
|
734
|
+
return best
|
|
735
|
+
|
|
736
|
+
|
|
737
|
+
def _canonical_given_for_cluster(occs: list[AuthorOccurrence]) -> tuple[str, str]:
|
|
738
|
+
"""Pick a (display-friendly given string, slug-friendly token) pair.
|
|
739
|
+
|
|
740
|
+
Prefer the fullest given-name *sequence* actually seen in the cluster
|
|
741
|
+
("Jose Alberto", not just "Jose"), so display names never drop
|
|
742
|
+
observed middle names. If the cluster only has initials, use the
|
|
743
|
+
canonical initial sequence. Corporate clusters return empty strings.
|
|
744
|
+
"""
|
|
745
|
+
frequency: dict[str, int] = defaultdict(int)
|
|
746
|
+
for o in occs:
|
|
747
|
+
frequency[" ".join(o.parsed.given_names)] += 1
|
|
748
|
+
|
|
749
|
+
best_key: tuple[int, int, int, int, str] | None = None
|
|
750
|
+
best_units: tuple[tuple[str, str], ...] = ()
|
|
751
|
+
best_given = ""
|
|
752
|
+
for o in occs:
|
|
753
|
+
if o.parsed.is_corporate:
|
|
754
|
+
continue
|
|
755
|
+
units = _given_units(o.parsed)
|
|
756
|
+
n_words = sum(1 for kind, _ in units if kind == "w")
|
|
757
|
+
if n_words == 0:
|
|
758
|
+
continue
|
|
759
|
+
given_str = " ".join(o.parsed.given_names)
|
|
760
|
+
# Fullest sequence first; among equally full ones, the spelling
|
|
761
|
+
# seen most often wins so a lone typo never names the cluster.
|
|
762
|
+
key = (n_words, len(units), frequency[given_str], len(given_str), given_str)
|
|
763
|
+
if best_key is None or key > best_key:
|
|
764
|
+
best_key, best_units, best_given = key, units, given_str
|
|
765
|
+
if best_given:
|
|
766
|
+
token = " ".join(v for kind, v in best_units if kind == "w")
|
|
767
|
+
return best_given, token
|
|
768
|
+
if any(o.parsed.is_corporate for o in occs):
|
|
769
|
+
return "", ""
|
|
770
|
+
# Fallback to the longest initial sequence we saw.
|
|
771
|
+
inits = max((o.parsed.initials for o in occs), key=len, default="")
|
|
772
|
+
return inits, inits.lower()
|
|
773
|
+
|
|
774
|
+
|
|
775
|
+
def _has_coauthor_overlap(
|
|
776
|
+
occ: AuthorOccurrence,
|
|
777
|
+
cluster: list[AuthorOccurrence],
|
|
778
|
+
) -> bool:
|
|
779
|
+
"""True if ``occ`` shares a co-author surname with any cluster member."""
|
|
780
|
+
if not occ.co_author_keys:
|
|
781
|
+
return False
|
|
782
|
+
cluster_keys: set[str] = set()
|
|
783
|
+
for c in cluster:
|
|
784
|
+
cluster_keys.update(c.co_author_keys)
|
|
785
|
+
return bool(set(occ.co_author_keys) & cluster_keys)
|
|
786
|
+
|
|
787
|
+
|
|
788
|
+
def _has_external_id(cluster: list[AuthorOccurrence]) -> bool:
|
|
789
|
+
return any(o.openalex_id or o.orcid for o in cluster)
|
|
790
|
+
|
|
791
|
+
|
|
792
|
+
def _cluster_signatures(
|
|
793
|
+
cluster: list[AuthorOccurrence],
|
|
794
|
+
) -> set[tuple[tuple[str, str], ...]]:
|
|
795
|
+
"""Every distinct non-empty given-name signature observed in a cluster."""
|
|
796
|
+
return {
|
|
797
|
+
units
|
|
798
|
+
for o in cluster
|
|
799
|
+
if (units := _given_units(o.parsed))
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
|
|
803
|
+
def _author_conflict_reasons(left: list[AuthorOccurrence], right: list[AuthorOccurrence]) -> list[str]:
|
|
804
|
+
"""Contradictions are independent of positive matching evidence."""
|
|
805
|
+
if any((r.record_id, r.position) in member.cannot_link for member in left for r in right):
|
|
806
|
+
return ["explicit_separate"]
|
|
807
|
+
if any((r.record_id, r.position) in member.ambiguous_identity_links for member in left for r in right):
|
|
808
|
+
return ["ambiguous_external_identity"]
|
|
809
|
+
lo = {o.orcid for o in left if o.orcid}
|
|
810
|
+
ro = {o.orcid for o in right if o.orcid}
|
|
811
|
+
if lo and ro and len(lo | ro) > 1:
|
|
812
|
+
return ["orcid_conflict"]
|
|
813
|
+
if any(a.parsed.is_corporate != b.parsed.is_corporate for a in left for b in right):
|
|
814
|
+
return ["corporate_person_conflict"]
|
|
815
|
+
li, ri = _cluster_external_ids(left), _cluster_external_ids(right)
|
|
816
|
+
if li & ri:
|
|
817
|
+
return [] # Consistent direct identity supports observed name variants.
|
|
818
|
+
reasons = []
|
|
819
|
+
if li and ri:
|
|
820
|
+
reasons.append("distinct_external_ids")
|
|
821
|
+
if {o.record_id for o in left} & {o.record_id for o in right}:
|
|
822
|
+
reasons.append("same_work_distinct_positions")
|
|
823
|
+
for left_occurrence in left:
|
|
824
|
+
for right_occurrence in right:
|
|
825
|
+
lu, ru = _given_units(left_occurrence.parsed), _given_units(right_occurrence.parsed)
|
|
826
|
+
if lu and ru and not (_units_compatible(lu, ru) or _given_subsequence(lu, ru)):
|
|
827
|
+
reasons.append("given_name_conflict")
|
|
828
|
+
return sorted(set(reasons))
|
|
829
|
+
|
|
830
|
+
|
|
831
|
+
def _author_clusters_conflict(left: list[AuthorOccurrence], right: list[AuthorOccurrence]) -> bool:
|
|
832
|
+
return bool(_author_conflict_reasons(left, right))
|
|
833
|
+
|
|
834
|
+
|
|
835
|
+
def _name_supported(left: list[AuthorOccurrence], right: list[AuthorOccurrence]) -> bool:
|
|
836
|
+
for left_occurrence in left:
|
|
837
|
+
for right_occurrence in right:
|
|
838
|
+
if left_occurrence.parsed.surname_norm != right_occurrence.parsed.surname_norm:
|
|
839
|
+
continue
|
|
840
|
+
if left_occurrence.parsed.is_corporate and right_occurrence.parsed.is_corporate:
|
|
841
|
+
return True
|
|
842
|
+
lu, ru = _given_units(left_occurrence.parsed), _given_units(right_occurrence.parsed)
|
|
843
|
+
if lu and ru and _units_compatible(lu, ru) and (
|
|
844
|
+
lu == ru or any(k == "w" for k, _ in lu + ru)
|
|
845
|
+
):
|
|
846
|
+
return True
|
|
847
|
+
return False
|
|
848
|
+
|
|
849
|
+
|
|
850
|
+
def _merge_unidentified_into_external_clusters(
|
|
851
|
+
clusters: list[list[AuthorOccurrence]],
|
|
852
|
+
) -> list[list[AuthorOccurrence]]:
|
|
853
|
+
external = [c for c in clusters if _has_external_id(c)]
|
|
854
|
+
unresolved = []
|
|
855
|
+
for cluster in clusters:
|
|
856
|
+
if _has_external_id(cluster):
|
|
857
|
+
continue
|
|
858
|
+
# Coauthor evidence belongs to each occurrence, never its signature bucket.
|
|
859
|
+
for occurrence in cluster:
|
|
860
|
+
candidates = [c for c in external if _name_supported([occurrence], c)
|
|
861
|
+
and not _author_clusters_conflict([occurrence], c)]
|
|
862
|
+
if len(candidates) > 1:
|
|
863
|
+
candidates = [c for c in candidates if _has_coauthor_overlap(occurrence, c)]
|
|
864
|
+
if len(candidates) == 1:
|
|
865
|
+
candidates[0].append(occurrence)
|
|
866
|
+
else:
|
|
867
|
+
unresolved.append([occurrence])
|
|
868
|
+
return external + unresolved
|
|
869
|
+
|
|
870
|
+
|
|
871
|
+
def _cluster_block(occs: list[AuthorOccurrence], cfg: AuthorClusterConfig,
|
|
872
|
+
seeds: list[list[AuthorOccurrence]] | None = None) -> list[list[AuthorOccurrence]]:
|
|
873
|
+
"""Resolve external identities globally before occurrence-specific name joins."""
|
|
874
|
+
seeds = seeds if seeds is not None else [[o] for o in occs]
|
|
875
|
+
external = _merge_clusters_by_external_id([c for c in seeds if len(c) > 1 or _has_external_id(c)])
|
|
876
|
+
unidentified = [c[0] for c in seeds if len(c) == 1 and not _has_external_id(c)]
|
|
877
|
+
# Full evidence comes first; stable tie ordering preserves deterministic IDs.
|
|
878
|
+
unidentified.sort(key=lambda o: (
|
|
879
|
+
tuple(-x for x in _units_informativeness(_given_units(o.parsed))),
|
|
880
|
+
" ".join(v for _, v in _given_units(o.parsed)), o.record_id, o.position,
|
|
881
|
+
))
|
|
882
|
+
clusters = external[:]
|
|
883
|
+
surname_candidates: dict[str, list[list[AuthorOccurrence]]] = defaultdict(list)
|
|
884
|
+
def index_cluster(cluster):
|
|
885
|
+
for surname in {o.parsed.surname_norm for o in cluster}:
|
|
886
|
+
surname_candidates[surname].append(cluster)
|
|
887
|
+
for cluster in clusters:
|
|
888
|
+
index_cluster(cluster)
|
|
889
|
+
held: list[list[AuthorOccurrence]] = []
|
|
890
|
+
for occurrence in unidentified:
|
|
891
|
+
one = [occurrence]
|
|
892
|
+
if cfg.merge_mode == "loose":
|
|
893
|
+
candidates = [c for c in surname_candidates[occurrence.parsed.surname_norm] if c[0].parsed.surname_norm == occurrence.parsed.surname_norm
|
|
894
|
+
and c[0].parsed.first_initial == occurrence.parsed.first_initial
|
|
895
|
+
and not _author_clusters_conflict(one, c)]
|
|
896
|
+
else:
|
|
897
|
+
candidates = [c for c in surname_candidates[occurrence.parsed.surname_norm] if _name_supported(one, c)
|
|
898
|
+
and not _author_clusters_conflict(one, c)]
|
|
899
|
+
if len(candidates) > 1:
|
|
900
|
+
overlap = [c for c in candidates if _has_coauthor_overlap(occurrence, c)]
|
|
901
|
+
if len(overlap) == 1:
|
|
902
|
+
candidates = overlap
|
|
903
|
+
if len(candidates) == 1:
|
|
904
|
+
candidates[0].append(occurrence)
|
|
905
|
+
elif candidates:
|
|
906
|
+
held.append(one)
|
|
907
|
+
else:
|
|
908
|
+
clusters.append(one)
|
|
909
|
+
index_cluster(one)
|
|
910
|
+
# Identical unresolved signatures can combine across works, without making
|
|
911
|
+
# an ambiguous initial cluster an anchor for other abbreviated signatures.
|
|
912
|
+
for one in held:
|
|
913
|
+
matches = [c for c in surname_candidates[one[0].parsed.surname_norm] if not _has_external_id(c)
|
|
914
|
+
and c[0].parsed.surname_norm == one[0].parsed.surname_norm
|
|
915
|
+
and _given_units(c[0].parsed) == _given_units(one[0].parsed)
|
|
916
|
+
and not _author_clusters_conflict(one, c)]
|
|
917
|
+
if len(matches) == 1:
|
|
918
|
+
matches[0].extend(one)
|
|
919
|
+
else:
|
|
920
|
+
clusters.append(one)
|
|
921
|
+
index_cluster(one)
|
|
922
|
+
return clusters
|
|
923
|
+
|
|
924
|
+
|
|
925
|
+
def _cluster_external_ids(cluster: list[AuthorOccurrence]) -> set[str]:
|
|
926
|
+
return {kind + value for o in cluster
|
|
927
|
+
for kind, value in (("oa:", o.openalex_id), ("orcid:", o.orcid)) if value}
|
|
928
|
+
|
|
929
|
+
|
|
930
|
+
def _mark_identity_ambiguities(clusters: list[list[AuthorOccurrence]]) -> None:
|
|
931
|
+
"""Find contradictory identity components before any order-dependent unions.
|
|
932
|
+
|
|
933
|
+
A seed compatible with both sides of a contradiction cannot choose either.
|
|
934
|
+
Keep that evidence unresolved across every later name/identity merge pass.
|
|
935
|
+
Explicitly forced groups are already seeds, so reviewed choices remain valid.
|
|
936
|
+
"""
|
|
937
|
+
parent = list(range(len(clusters)))
|
|
938
|
+
def root(i):
|
|
939
|
+
while parent[i] != i:
|
|
940
|
+
parent[i] = parent[parent[i]]
|
|
941
|
+
i = parent[i]
|
|
942
|
+
return i
|
|
943
|
+
owner: dict[str, int] = {}
|
|
944
|
+
for i, cluster in enumerate(clusters):
|
|
945
|
+
for key in _cluster_external_ids(cluster):
|
|
946
|
+
if key in owner:
|
|
947
|
+
parent[root(i)] = root(owner[key])
|
|
948
|
+
else:
|
|
949
|
+
owner[key] = i
|
|
950
|
+
components: dict[int, list[int]] = defaultdict(list)
|
|
951
|
+
for i in range(len(clusters)):
|
|
952
|
+
components[root(i)].append(i)
|
|
953
|
+
blocked: set[tuple[int, int]] = set()
|
|
954
|
+
for indices in components.values():
|
|
955
|
+
conflicts = [(i, j) for pos, i in enumerate(indices) for j in indices[pos + 1:]
|
|
956
|
+
if {"orcid_conflict", "explicit_separate"}
|
|
957
|
+
& set(_author_conflict_reasons(clusters[i], clusters[j]))]
|
|
958
|
+
for left, right in conflicts:
|
|
959
|
+
for i in indices:
|
|
960
|
+
if i in (left, right):
|
|
961
|
+
continue
|
|
962
|
+
# Membership of this component already supplies transitive ID
|
|
963
|
+
# evidence. Distinct direct IDs or name spellings are not hard
|
|
964
|
+
# contradictions to an anchor elsewhere in the component.
|
|
965
|
+
hard_conflicts = {"orcid_conflict", "explicit_separate", "corporate_person_conflict"}
|
|
966
|
+
if (not hard_conflicts.intersection(_author_conflict_reasons(clusters[i], clusters[left]))
|
|
967
|
+
and not hard_conflicts.intersection(_author_conflict_reasons(clusters[i], clusters[right]))):
|
|
968
|
+
blocked.update(((i, left), (i, right)))
|
|
969
|
+
for left, right in blocked:
|
|
970
|
+
for own, other in ((left, right), (right, left)):
|
|
971
|
+
coordinates = {(o.record_id, o.position) for o in clusters[other]}
|
|
972
|
+
for occurrence in clusters[own]:
|
|
973
|
+
occurrence.ambiguous_identity_links.update(coordinates)
|
|
974
|
+
|
|
975
|
+
|
|
976
|
+
def _merge_clusters_by_external_id(clusters: list[list[AuthorOccurrence]]) -> list[list[AuthorOccurrence]]:
|
|
977
|
+
"""Union direct identities only while every known ORCID remains consistent."""
|
|
978
|
+
result: list[list[AuthorOccurrence]] = []
|
|
979
|
+
for cluster in clusters:
|
|
980
|
+
targets = [c for c in result if _cluster_external_ids(c) & _cluster_external_ids(cluster)
|
|
981
|
+
and not _author_clusters_conflict(c, cluster)]
|
|
982
|
+
# An ORCID-free bridge shared by conflicting identities cannot choose one.
|
|
983
|
+
if len({o.orcid for c in targets + [cluster] for o in c if o.orcid}) > 1:
|
|
984
|
+
targets = []
|
|
985
|
+
if any('explicit_separate' in _author_conflict_reasons(a, b)
|
|
986
|
+
for i, a in enumerate(targets) for b in targets[i + 1:]):
|
|
987
|
+
targets = []
|
|
988
|
+
if targets:
|
|
989
|
+
primary = targets[0]
|
|
990
|
+
primary.extend(cluster)
|
|
991
|
+
for other in targets[1:]:
|
|
992
|
+
primary.extend(other)
|
|
993
|
+
result = [c for c in result if c is not other]
|
|
994
|
+
else:
|
|
995
|
+
result.append(cluster)
|
|
996
|
+
return result
|
|
997
|
+
|
|
998
|
+
|
|
999
|
+
def _cluster_surname(occs: list[AuthorOccurrence]) -> tuple[str, str]:
|
|
1000
|
+
"""Order-independent (surname_norm, display surname) for a cluster.
|
|
1001
|
+
|
|
1002
|
+
The most specific (most words), then most frequent, normalized
|
|
1003
|
+
surname wins — so a cluster merged across blocks by external ids or
|
|
1004
|
+
bridging is named by its compound form no matter which occurrence
|
|
1005
|
+
happened to land first. Display casing follows the chosen norm,
|
|
1006
|
+
preferring properly-cased spellings, then the most frequent one.
|
|
1007
|
+
"""
|
|
1008
|
+
norm_counts: dict[str, int] = defaultdict(int)
|
|
1009
|
+
for o in occs:
|
|
1010
|
+
norm_counts[o.parsed.surname_norm] += 1
|
|
1011
|
+
surname_norm = max(
|
|
1012
|
+
norm_counts,
|
|
1013
|
+
key=lambda n: (len(n.split()), norm_counts[n], n),
|
|
1014
|
+
)
|
|
1015
|
+
display_counts: dict[str, int] = defaultdict(int)
|
|
1016
|
+
for o in occs:
|
|
1017
|
+
if o.parsed.surname_norm == surname_norm:
|
|
1018
|
+
display_counts[o.parsed.surname] += 1
|
|
1019
|
+
surname_display = max(
|
|
1020
|
+
display_counts,
|
|
1021
|
+
key=lambda s: (any(c.isupper() for c in s), display_counts[s], s),
|
|
1022
|
+
)
|
|
1023
|
+
return surname_norm, surname_display
|
|
1024
|
+
|
|
1025
|
+
|
|
1026
|
+
def _bridge_compound_surnames(
|
|
1027
|
+
clusters: list[list[AuthorOccurrence]],
|
|
1028
|
+
) -> list[list[AuthorOccurrence]]:
|
|
1029
|
+
"""Merge single-surname clusters into compound-surname clusters.
|
|
1030
|
+
|
|
1031
|
+
'Reyes, Sandra' can join 'Polanía Reyes, Sandra' — but only on strong
|
|
1032
|
+
evidence: the compound surname starts or ends with the single one,
|
|
1033
|
+
the given-name signatures are fully compatible, *and* at least one
|
|
1034
|
+
exact full given name is shared. Clusters that both carry external
|
|
1035
|
+
ids never bridge (same-id cases were already merged by
|
|
1036
|
+
:func:`_merge_clusters_by_external_id`, so remaining id pairs
|
|
1037
|
+
disagree). Initials-only records never bridge; ambiguity (two
|
|
1038
|
+
possible compound targets) leaves the cluster where it is.
|
|
1039
|
+
"""
|
|
1040
|
+
|
|
1041
|
+
def block_key(cluster: list[AuthorOccurrence]) -> str:
|
|
1042
|
+
return _cluster_surname(cluster)[0]
|
|
1043
|
+
|
|
1044
|
+
by_key: dict[str, list[list[AuthorOccurrence]]] = defaultdict(list)
|
|
1045
|
+
for cluster in clusters:
|
|
1046
|
+
by_key[block_key(cluster)].append(cluster)
|
|
1047
|
+
compound_keys = sorted(k for k in by_key if " " in k)
|
|
1048
|
+
if not compound_keys:
|
|
1049
|
+
return clusters
|
|
1050
|
+
|
|
1051
|
+
absorbed: set[int] = set()
|
|
1052
|
+
for single_key in sorted(k for k in by_key if " " not in k):
|
|
1053
|
+
target_keys = [
|
|
1054
|
+
ck for ck in compound_keys
|
|
1055
|
+
if ck.split()[0] == single_key or ck.split()[-1] == single_key
|
|
1056
|
+
]
|
|
1057
|
+
if not target_keys:
|
|
1058
|
+
continue
|
|
1059
|
+
for single in by_key[single_key]:
|
|
1060
|
+
s_units = _cluster_best_units(single)
|
|
1061
|
+
s_words = {v for kind, v in s_units if kind == "w"}
|
|
1062
|
+
if not s_words or any(o.parsed.is_corporate for o in single):
|
|
1063
|
+
continue
|
|
1064
|
+
s_ids = _cluster_external_ids(single)
|
|
1065
|
+
targets: list[list[AuthorOccurrence]] = []
|
|
1066
|
+
for ck in target_keys:
|
|
1067
|
+
for compound in by_key[ck]:
|
|
1068
|
+
c_ids = _cluster_external_ids(compound)
|
|
1069
|
+
if s_ids and c_ids:
|
|
1070
|
+
# Both sides carry ids and (post id-merge) they
|
|
1071
|
+
# necessarily disagree — never bridge over that.
|
|
1072
|
+
continue
|
|
1073
|
+
c_units = _cluster_best_units(compound)
|
|
1074
|
+
c_words = {v for kind, v in c_units if kind == "w"}
|
|
1075
|
+
if (
|
|
1076
|
+
c_words
|
|
1077
|
+
and not _author_clusters_conflict(single, compound)
|
|
1078
|
+
and (s_words & c_words)
|
|
1079
|
+
and _units_compatible(s_units, c_units)
|
|
1080
|
+
):
|
|
1081
|
+
targets.append(compound)
|
|
1082
|
+
if len(targets) == 1:
|
|
1083
|
+
targets[0].extend(single)
|
|
1084
|
+
absorbed.add(id(single))
|
|
1085
|
+
return [c for c in clusters if id(c) not in absorbed]
|
|
1086
|
+
|
|
1087
|
+
|
|
1088
|
+
def normalize_authors(
|
|
1089
|
+
*,
|
|
1090
|
+
works: pd.DataFrame,
|
|
1091
|
+
enriched_works: pd.DataFrame | None = None,
|
|
1092
|
+
citation_edges: pd.DataFrame | None = None,
|
|
1093
|
+
cfg: AuthorClusterConfig | None = None,
|
|
1094
|
+
aliases: dict[str, str] | None = None,
|
|
1095
|
+
external_ids: dict[str, dict[str, str | None]] | None = None,
|
|
1096
|
+
constraints: list[dict] | None = None,
|
|
1097
|
+
audit: list[dict] | None = None,
|
|
1098
|
+
identity_state: dict | None = None,
|
|
1099
|
+
) -> tuple[pd.DataFrame, pd.DataFrame, list[dict]]:
|
|
1100
|
+
"""Cluster authors across the corpus.
|
|
1101
|
+
|
|
1102
|
+
Parameters
|
|
1103
|
+
----------
|
|
1104
|
+
works:
|
|
1105
|
+
Canonical works DataFrame (the output of
|
|
1106
|
+
:func:`citegraph.dedup.canonicalize_works`). Must be indexed by
|
|
1107
|
+
``id`` and contain an ``Authors_List`` column (or ``Authors``).
|
|
1108
|
+
A ``ring`` column, when present, feeds the ``n_core_works``
|
|
1109
|
+
metric; frames without it are accepted (the metric reads 0).
|
|
1110
|
+
enriched_works:
|
|
1111
|
+
Optional output of :func:`citegraph.enrich.enrich_works`.
|
|
1112
|
+
If present and contains the ``OpenAlex_Authors`` column (a list
|
|
1113
|
+
of ``{display_name, openalex_id, orcid}`` dicts per row), those
|
|
1114
|
+
identifiers are attached to the matching work authors so
|
|
1115
|
+
OpenAlex acts as ground truth — for core works too.
|
|
1116
|
+
citation_edges:
|
|
1117
|
+
Optional citation graph with ``citing_id`` and ``cited_id`` columns.
|
|
1118
|
+
When provided, ``authors.csv`` counts citations received by each
|
|
1119
|
+
canonical author's works and the distinct works citing them.
|
|
1120
|
+
cfg:
|
|
1121
|
+
Tunable :class:`AuthorClusterConfig`.
|
|
1122
|
+
aliases:
|
|
1123
|
+
Optional ``{cluster_id: canonical_id}`` mapping. Applied after
|
|
1124
|
+
clustering: every cluster whose id is a key in ``aliases`` is
|
|
1125
|
+
merged into the cluster keyed by the value. Lets the user fix
|
|
1126
|
+
the algorithm's mistakes without re-running.
|
|
1127
|
+
|
|
1128
|
+
Returns
|
|
1129
|
+
-------
|
|
1130
|
+
authors_df:
|
|
1131
|
+
One row per canonical author, indexed by ``id``.
|
|
1132
|
+
citations_df:
|
|
1133
|
+
Edge table linking each canonical author to every work they
|
|
1134
|
+
appear on (``author_id``, ``record_id``, ``position``,
|
|
1135
|
+
``raw_author``).
|
|
1136
|
+
review:
|
|
1137
|
+
List of dicts describing low-confidence clusters that the user
|
|
1138
|
+
may want to inspect.
|
|
1139
|
+
"""
|
|
1140
|
+
cfg = cfg or AuthorClusterConfig()
|
|
1141
|
+
aliases = aliases or {}
|
|
1142
|
+
|
|
1143
|
+
attachment_audit: list[dict] = []
|
|
1144
|
+
occurrences = _collect_occurrences(
|
|
1145
|
+
works=works,
|
|
1146
|
+
enriched_works=enriched_works,
|
|
1147
|
+
audit=attachment_audit,
|
|
1148
|
+
)
|
|
1149
|
+
|
|
1150
|
+
# Block by surname_norm and cluster within each block.
|
|
1151
|
+
blocks: dict[str, list[AuthorOccurrence]] = defaultdict(list)
|
|
1152
|
+
for o in occurrences:
|
|
1153
|
+
blocks[o.parsed.surname_norm].append(o)
|
|
1154
|
+
|
|
1155
|
+
from citegraph.author_overrides import prepare_constraints
|
|
1156
|
+
seeds = prepare_constraints(occurrences, constraints or [])
|
|
1157
|
+
_mark_identity_ambiguities(seeds)
|
|
1158
|
+
raw_clusters = _cluster_block(occurrences, cfg, seeds)
|
|
1159
|
+
|
|
1160
|
+
# Cross-block passes: shared external ids override blocking, then
|
|
1161
|
+
# single-surname clusters bridge into compound-surname clusters when
|
|
1162
|
+
# the evidence is strong enough (see the function docstrings).
|
|
1163
|
+
raw_clusters = _merge_clusters_by_external_id(raw_clusters)
|
|
1164
|
+
raw_clusters = _bridge_compound_surnames(raw_clusters)
|
|
1165
|
+
block_keys = frozenset(blocks)
|
|
1166
|
+
|
|
1167
|
+
# Build AuthorCluster records with stable ids.
|
|
1168
|
+
used_ids: set[str] = set()
|
|
1169
|
+
clusters: list[AuthorCluster] = []
|
|
1170
|
+
for occs in raw_clusters:
|
|
1171
|
+
if not occs:
|
|
1172
|
+
continue
|
|
1173
|
+
canonical_given, given_token = _canonical_given_for_cluster(occs)
|
|
1174
|
+
surname_norm, surname_display = _cluster_surname(occs)
|
|
1175
|
+
base_id = _make_author_id(surname_norm, given_token)
|
|
1176
|
+
cid = base_id
|
|
1177
|
+
suffix = 1
|
|
1178
|
+
while cid in used_ids:
|
|
1179
|
+
cid = _make_author_id(surname_norm, given_token, suffix)
|
|
1180
|
+
suffix += 1
|
|
1181
|
+
used_ids.add(cid)
|
|
1182
|
+
|
|
1183
|
+
oa_id = next((o.openalex_id for o in occs if o.openalex_id), None)
|
|
1184
|
+
orcid = next((o.orcid for o in occs if o.orcid), None)
|
|
1185
|
+
initials = max((o.parsed.initials for o in occs), key=len, default="")
|
|
1186
|
+
|
|
1187
|
+
display = (
|
|
1188
|
+
f"{canonical_given} {surname_display}".strip()
|
|
1189
|
+
if canonical_given and not canonical_given.isupper()
|
|
1190
|
+
else f"{surname_display}, {canonical_given}".strip(", ")
|
|
1191
|
+
)
|
|
1192
|
+
|
|
1193
|
+
clusters.append(
|
|
1194
|
+
AuthorCluster(
|
|
1195
|
+
id=cid,
|
|
1196
|
+
display_name=display,
|
|
1197
|
+
surname=surname_display,
|
|
1198
|
+
surname_norm=surname_norm,
|
|
1199
|
+
canonical_given=canonical_given,
|
|
1200
|
+
initials=initials,
|
|
1201
|
+
openalex_id=oa_id,
|
|
1202
|
+
orcid=orcid,
|
|
1203
|
+
occurrences=occs,
|
|
1204
|
+
review_reason=_review_reason(occs, cfg, block_keys),
|
|
1205
|
+
)
|
|
1206
|
+
)
|
|
1207
|
+
|
|
1208
|
+
# Post-hoc ambiguity flag: a no-id cluster whose name evidence is
|
|
1209
|
+
# compatible with two or more sibling clusters in the same surname
|
|
1210
|
+
# block was held apart only because the evidence could not decide —
|
|
1211
|
+
# surface that in the review file instead of leaving the split invisible.
|
|
1212
|
+
by_block: dict[str, list[AuthorCluster]] = defaultdict(list)
|
|
1213
|
+
for c in clusters:
|
|
1214
|
+
by_block[c.surname_norm].append(c)
|
|
1215
|
+
for block_clusters in by_block.values():
|
|
1216
|
+
if len(block_clusters) < 3:
|
|
1217
|
+
continue # fewer than two possible rivals — nothing ambiguous
|
|
1218
|
+
for c in block_clusters:
|
|
1219
|
+
if c.review_reason or c.openalex_id or c.orcid:
|
|
1220
|
+
continue
|
|
1221
|
+
units = _cluster_best_units(c.occurrences)
|
|
1222
|
+
if not units:
|
|
1223
|
+
continue
|
|
1224
|
+
n_compatible = sum(
|
|
1225
|
+
1
|
|
1226
|
+
for other in block_clusters
|
|
1227
|
+
if other is not c
|
|
1228
|
+
and (other_units := _cluster_best_units(other.occurrences))
|
|
1229
|
+
and _units_compatible(units, other_units)
|
|
1230
|
+
)
|
|
1231
|
+
if n_compatible >= 2:
|
|
1232
|
+
c.review_reason = (
|
|
1233
|
+
"ambiguous name evidence: compatible with multiple "
|
|
1234
|
+
"clusters in this surname block"
|
|
1235
|
+
)
|
|
1236
|
+
|
|
1237
|
+
identity_changes = []
|
|
1238
|
+
if identity_state is not None:
|
|
1239
|
+
from citegraph.identity import occurrence_key, reconcile, resolve_redirect
|
|
1240
|
+
|
|
1241
|
+
def memberships():
|
|
1242
|
+
return {c.id: {occurrence_key(o.record_id, o.position, o.parsed.raw)
|
|
1243
|
+
for o in c.occurrences} for c in clusters}
|
|
1244
|
+
|
|
1245
|
+
mapping, algorithm, changes = reconcile(memberships(), identity_state.get('algorithm'))
|
|
1246
|
+
for c in clusters:
|
|
1247
|
+
c.id = mapping[c.id]
|
|
1248
|
+
identity_changes.extend(changes)
|
|
1249
|
+
for source in aliases:
|
|
1250
|
+
resolve_redirect(source, aliases) # reject cycles before resolving history
|
|
1251
|
+
# A curated alias asserts that two spellings are one person. That
|
|
1252
|
+
# outranks both how the algorithm clustered them and how an earlier run
|
|
1253
|
+
# published them, so the merge is settled here — against stored
|
|
1254
|
+
# occurrences, before any published ID is assigned — and identity
|
|
1255
|
+
# assignment simply follows the membership the user chose.
|
|
1256
|
+
previous_entries = (identity_state.get('published') or {}).get('entries', {})
|
|
1257
|
+
current = memberships()
|
|
1258
|
+
parent = {cid: cid for cid in current}
|
|
1259
|
+
|
|
1260
|
+
def _find(cid: str) -> str:
|
|
1261
|
+
while parent[cid] != cid:
|
|
1262
|
+
parent[cid] = parent[parent[cid]]
|
|
1263
|
+
cid = parent[cid]
|
|
1264
|
+
return cid
|
|
1265
|
+
|
|
1266
|
+
def _union(left: str, right: str) -> None:
|
|
1267
|
+
left, right = _find(left), _find(right)
|
|
1268
|
+
if left != right:
|
|
1269
|
+
parent[right] = left
|
|
1270
|
+
|
|
1271
|
+
def _holders(public_id: str) -> list[str]:
|
|
1272
|
+
"""Clusters now holding a published ID's occurrences.
|
|
1273
|
+
|
|
1274
|
+
An ID may have been retired by a split, continued by one cluster, or
|
|
1275
|
+
spread over several; following its stored occurrences covers all
|
|
1276
|
+
three without the caller needing to know which happened. An ID with
|
|
1277
|
+
no stored occurrences is only usable if it names a cluster outright.
|
|
1278
|
+
"""
|
|
1279
|
+
members = set(previous_entries.get(public_id, ()))
|
|
1280
|
+
holders = sorted(cid for cid, held in current.items() if members & held)
|
|
1281
|
+
if holders:
|
|
1282
|
+
return holders
|
|
1283
|
+
return [public_id] if public_id in current else []
|
|
1284
|
+
|
|
1285
|
+
named: dict[str, set[str]] = defaultdict(set)
|
|
1286
|
+
stale: set[str] = set()
|
|
1287
|
+
bindings = {}
|
|
1288
|
+
for source, target in aliases.items():
|
|
1289
|
+
sources, targets = _holders(source), _holders(target)
|
|
1290
|
+
if not sources or not targets:
|
|
1291
|
+
stale.add(source if not sources else target)
|
|
1292
|
+
continue
|
|
1293
|
+
for cid in sources[1:] + targets:
|
|
1294
|
+
_union(sources[0], cid)
|
|
1295
|
+
for cid in targets:
|
|
1296
|
+
named[cid].add(target)
|
|
1297
|
+
bindings[source] = {'target': target, 'source_ids': sources, 'target_ids': targets}
|
|
1298
|
+
if stale:
|
|
1299
|
+
raise ValueError(
|
|
1300
|
+
f'Stale author aliases: {sorted(stale)}; review against current authors')
|
|
1301
|
+
|
|
1302
|
+
components: dict[str, list[str]] = defaultdict(list)
|
|
1303
|
+
for cid in current:
|
|
1304
|
+
components[_find(cid)].append(cid)
|
|
1305
|
+
migrated_aliases, pins = {}, {}
|
|
1306
|
+
for component in components.values():
|
|
1307
|
+
claims = {t for cid in component for t in named.get(cid, ())}
|
|
1308
|
+
if not claims:
|
|
1309
|
+
continue
|
|
1310
|
+
# Absorb into a cluster the user actually named, the largest one
|
|
1311
|
+
# first, so the survivor carries that person's display name instead
|
|
1312
|
+
# of an initials-only fragment's.
|
|
1313
|
+
canonical = max((cid for cid in component if named.get(cid)),
|
|
1314
|
+
key=lambda cid: (len(current[cid]), cid))
|
|
1315
|
+
for cid in component:
|
|
1316
|
+
if cid != canonical:
|
|
1317
|
+
migrated_aliases[cid] = canonical
|
|
1318
|
+
pins[canonical] = max(claims, key=lambda t: (len(previous_entries.get(t, ())), t))
|
|
1319
|
+
aliases = migrated_aliases
|
|
1320
|
+
|
|
1321
|
+
# Apply user-curated aliases: merge cluster B into cluster A.
|
|
1322
|
+
if aliases:
|
|
1323
|
+
clusters = _apply_aliases(clusters, aliases)
|
|
1324
|
+
|
|
1325
|
+
if identity_state is not None:
|
|
1326
|
+
# Membership is already what the user asked for; assigning IDs is all
|
|
1327
|
+
# that is left. The pins carry their merges through a split that would
|
|
1328
|
+
# otherwise retire the very IDs external crosswalks join on.
|
|
1329
|
+
mapping, published, changes = reconcile(
|
|
1330
|
+
memberships(), identity_state.get('published'), pinned=pins)
|
|
1331
|
+
for c in clusters:
|
|
1332
|
+
c.id = mapping[c.id]
|
|
1333
|
+
identity_changes.extend(changes)
|
|
1334
|
+
identity_state.update(schema_version=1, algorithm=algorithm, published=published,
|
|
1335
|
+
alias_bindings=bindings)
|
|
1336
|
+
|
|
1337
|
+
if external_ids:
|
|
1338
|
+
_apply_external_ids(clusters, external_ids)
|
|
1339
|
+
|
|
1340
|
+
authors_df = _clusters_to_authors_df(
|
|
1341
|
+
clusters, works=works, citation_edges=citation_edges
|
|
1342
|
+
)
|
|
1343
|
+
citations_df = _clusters_to_citations_df(clusters)
|
|
1344
|
+
review = [
|
|
1345
|
+
{
|
|
1346
|
+
"author_id": c.id,
|
|
1347
|
+
"display_name": c.display_name,
|
|
1348
|
+
"surname": c.surname_norm,
|
|
1349
|
+
"n_occurrences": c.n_occurrences,
|
|
1350
|
+
"raw_strings": sorted({o.parsed.raw for o in c.occurrences}),
|
|
1351
|
+
"reason": c.review_reason,
|
|
1352
|
+
}
|
|
1353
|
+
for c in clusters
|
|
1354
|
+
if c.review_reason
|
|
1355
|
+
]
|
|
1356
|
+
review.extend({'author_id': entry['old_id'], 'reason': 'identity_split', **entry}
|
|
1357
|
+
for entry in identity_changes if entry['decision'] == 'identity_split')
|
|
1358
|
+
|
|
1359
|
+
author_by_occurrence = {(row.record_id, row.position): row.author_id
|
|
1360
|
+
for row in citations_df.itertuples(index=False)}
|
|
1361
|
+
for entry in attachment_audit:
|
|
1362
|
+
entry["author_id"] = author_by_occurrence.get((entry["record_id"], entry["position"]))
|
|
1363
|
+
if entry["decision"] in {"ambiguous", "conflict"}:
|
|
1364
|
+
# A provider author nobody claimed is the mirror image of a
|
|
1365
|
+
# conflict, not a cluster to review: it has no position and no
|
|
1366
|
+
# author_id, so filing it as one inflates the backlog with rows
|
|
1367
|
+
# that cannot be acted on.
|
|
1368
|
+
review.append({**entry, "reason": "enrichment_provider_author_unused"
|
|
1369
|
+
if entry.get("detail") == "unassigned_provider_author"
|
|
1370
|
+
else "enrichment_" + entry["decision"]})
|
|
1371
|
+
resolution_audit: list[dict] = []
|
|
1372
|
+
for cluster in clusters:
|
|
1373
|
+
identifiers = {"openalex_ids": sorted({o.openalex_id for o in cluster.occurrences if o.openalex_id}),
|
|
1374
|
+
"orcids": sorted({o.orcid for o in cluster.occurrences if o.orcid})}
|
|
1375
|
+
resolution_audit.append({"decision": "cluster_identity", "author_id": cluster.id,
|
|
1376
|
+
"occurrences": [{"record_id": o.record_id, "position": o.position}
|
|
1377
|
+
for o in cluster.occurrences], **identifiers})
|
|
1378
|
+
reasons = []
|
|
1379
|
+
if len(identifiers["openalex_ids"]) > 1:
|
|
1380
|
+
reasons.append("multiple_openalex_ids")
|
|
1381
|
+
if len({o.record_id for o in cluster.occurrences}) < len(cluster.occurrences):
|
|
1382
|
+
reasons.append("duplicate_authorship")
|
|
1383
|
+
for reason in reasons:
|
|
1384
|
+
review.append({"author_id": cluster.id, "reason": reason, **identifiers})
|
|
1385
|
+
evidence_candidates: dict[str, set[int]] = defaultdict(set)
|
|
1386
|
+
for i, cluster in enumerate(clusters):
|
|
1387
|
+
for key in ({'surname:' + o.parsed.surname_norm for o in cluster.occurrences}
|
|
1388
|
+
| _cluster_external_ids(cluster.occurrences)):
|
|
1389
|
+
evidence_candidates[key].add(i)
|
|
1390
|
+
for i, left in enumerate(clusters):
|
|
1391
|
+
candidate_indices: set[int] = set()
|
|
1392
|
+
for key in ({'surname:' + o.parsed.surname_norm for o in left.occurrences}
|
|
1393
|
+
| _cluster_external_ids(left.occurrences)):
|
|
1394
|
+
candidate_indices.update(evidence_candidates[key])
|
|
1395
|
+
for j in sorted(index for index in candidate_indices if index > i):
|
|
1396
|
+
right = clusters[j]
|
|
1397
|
+
if not (_name_supported(left.occurrences, right.occurrences)
|
|
1398
|
+
or _cluster_external_ids(left.occurrences) & _cluster_external_ids(right.occurrences)):
|
|
1399
|
+
continue
|
|
1400
|
+
reasons = _author_conflict_reasons(left.occurrences, right.occurrences)
|
|
1401
|
+
if reasons:
|
|
1402
|
+
finding = {"author_id": left.id, "other_author_id": right.id,
|
|
1403
|
+
"reason": ",".join(reasons), "decision": "prevented_merge"}
|
|
1404
|
+
review.append(finding)
|
|
1405
|
+
resolution_audit.append(finding)
|
|
1406
|
+
if audit is not None:
|
|
1407
|
+
audit.extend(identity_changes)
|
|
1408
|
+
audit.extend(attachment_audit)
|
|
1409
|
+
audit.extend(resolution_audit)
|
|
1410
|
+
audit.extend({"decision": "manual_override", **row} for row in constraints or [])
|
|
1411
|
+
audit.extend({"decision": "legacy_alias", "cluster_id": source, "canonical_id": target}
|
|
1412
|
+
for source, target in aliases.items())
|
|
1413
|
+
|
|
1414
|
+
logger.info(
|
|
1415
|
+
"Author normalization: %d occurrences → %d clusters (%d flagged for review)",
|
|
1416
|
+
len(occurrences),
|
|
1417
|
+
len(clusters),
|
|
1418
|
+
len(review),
|
|
1419
|
+
)
|
|
1420
|
+
return authors_df, citations_df, review
|
|
1421
|
+
|
|
1422
|
+
|
|
1423
|
+
def _collect_occurrences(
|
|
1424
|
+
*,
|
|
1425
|
+
works: pd.DataFrame,
|
|
1426
|
+
enriched_works: pd.DataFrame | None,
|
|
1427
|
+
audit: list[dict] | None = None,
|
|
1428
|
+
) -> list[AuthorOccurrence]:
|
|
1429
|
+
"""Walk the works table and emit one AuthorOccurrence per author.
|
|
1430
|
+
|
|
1431
|
+
Runs in two passes: the first parse of every author string builds a
|
|
1432
|
+
corpus surname lexicon from trusted forms (comma forms, hyphenated
|
|
1433
|
+
and particle compounds, enrichment family names); the second parse
|
|
1434
|
+
uses that lexicon so comma-less names adopt attested compound
|
|
1435
|
+
surname boundaries. Co-author surname-norm keys are computed per
|
|
1436
|
+
record so the clustering step can use them as a tiebreak signal.
|
|
1437
|
+
"""
|
|
1438
|
+
occurrences: list[AuthorOccurrence] = []
|
|
1439
|
+
|
|
1440
|
+
# Build a per-work enrichment map: work_id -> [(display, oa_id, orcid), ...]
|
|
1441
|
+
enrich_map: dict[str, list[dict]] = {}
|
|
1442
|
+
if enriched_works is not None and not enriched_works.empty:
|
|
1443
|
+
if "OpenAlex_Authors" in enriched_works.columns:
|
|
1444
|
+
for work_id, row in enriched_works.iterrows():
|
|
1445
|
+
authors = row.get("OpenAlex_Authors")
|
|
1446
|
+
if isinstance(authors, list):
|
|
1447
|
+
enrich_map[str(work_id)] = authors
|
|
1448
|
+
|
|
1449
|
+
work_rows: list[tuple[str, list[str]]] = []
|
|
1450
|
+
if not works.empty:
|
|
1451
|
+
for work_id, row in works.iterrows():
|
|
1452
|
+
work_rows.append((str(work_id), _row_authors(row)))
|
|
1453
|
+
|
|
1454
|
+
lexicon = _build_surname_lexicon(
|
|
1455
|
+
(a for _wid, authors_list in work_rows for a in authors_list),
|
|
1456
|
+
enrich_map,
|
|
1457
|
+
)
|
|
1458
|
+
|
|
1459
|
+
for work_id, authors_list in work_rows:
|
|
1460
|
+
parsed = [parse_author(a, known_surnames=lexicon) for a in authors_list]
|
|
1461
|
+
co_keys = tuple(p.surname_norm if p is not None else "" for p in parsed)
|
|
1462
|
+
enrich_authors = enrich_map.get(work_id, [])
|
|
1463
|
+
work_audit: list[dict] = []
|
|
1464
|
+
assignments = _match_enrichment_authors(parsed, enrich_authors, lexicon, work_audit)
|
|
1465
|
+
if audit is not None:
|
|
1466
|
+
audit.extend({"record_id": work_id, **entry} for entry in work_audit)
|
|
1467
|
+
for pos, p in enumerate(parsed):
|
|
1468
|
+
if p is None:
|
|
1469
|
+
continue
|
|
1470
|
+
oa_id, orcid = assignments[pos]
|
|
1471
|
+
occurrences.append(
|
|
1472
|
+
AuthorOccurrence(
|
|
1473
|
+
parsed=p,
|
|
1474
|
+
record_id=work_id,
|
|
1475
|
+
position=pos,
|
|
1476
|
+
co_author_keys=tuple(k for i, k in enumerate(co_keys) if i != pos and k),
|
|
1477
|
+
openalex_id=oa_id,
|
|
1478
|
+
orcid=orcid,
|
|
1479
|
+
)
|
|
1480
|
+
)
|
|
1481
|
+
|
|
1482
|
+
return occurrences
|
|
1483
|
+
|
|
1484
|
+
|
|
1485
|
+
def _build_surname_lexicon(
|
|
1486
|
+
raw_strings: Iterable[str],
|
|
1487
|
+
enrich_map: dict[str, list[dict]],
|
|
1488
|
+
) -> frozenset[str]:
|
|
1489
|
+
"""Collect normalized multi-word surnames attested by trusted forms.
|
|
1490
|
+
|
|
1491
|
+
Trusted sources: comma forms ("Guerra Forero, J.A." states its own
|
|
1492
|
+
boundary), hyphenated surnames ("Polania-Reyes" attests "polania
|
|
1493
|
+
reyes"), particle compounds ("van der Berg"), the Spanish 'y'
|
|
1494
|
+
connector, and enrichment ``family`` fields. Single-word surnames are
|
|
1495
|
+
never collected — a bare "Knowles" elsewhere must not re-split
|
|
1496
|
+
"Caitlin Knowles Myers".
|
|
1497
|
+
"""
|
|
1498
|
+
lexicon: set[str] = set()
|
|
1499
|
+
for raw in raw_strings:
|
|
1500
|
+
p = parse_author(raw)
|
|
1501
|
+
if p is None or p.is_corporate:
|
|
1502
|
+
continue
|
|
1503
|
+
norm = p.surname_norm
|
|
1504
|
+
if " " not in norm:
|
|
1505
|
+
continue
|
|
1506
|
+
trusted = (
|
|
1507
|
+
"," in p.raw
|
|
1508
|
+
or "-" in p.surname
|
|
1509
|
+
or norm.split()[0] in _PARTICLES
|
|
1510
|
+
or " y " in f" {norm} "
|
|
1511
|
+
)
|
|
1512
|
+
if trusted:
|
|
1513
|
+
lexicon.add(norm)
|
|
1514
|
+
# A multi-word `family` redefines a surname boundary for the whole corpus,
|
|
1515
|
+
# so a provider record that *contradicts* another record for the same name
|
|
1516
|
+
# must not carry it: CrossRef returned family="Claudia Lopez"/given="Maria"
|
|
1517
|
+
# for María Claudia López on one work while others said "Lopez", and
|
|
1518
|
+
# adopting the longer boundary shattered her into three clusters (one of
|
|
1519
|
+
# them holding her OpenAlex id). When the corpus attests a shorter boundary
|
|
1520
|
+
# for the same display name, keep the conservative last-token parse — which
|
|
1521
|
+
# `author_review.json` already flags as a possible compound surname —
|
|
1522
|
+
# rather than trusting the longer one. An uncontradicted `family` is still
|
|
1523
|
+
# trusted on its own, so a lone "Guerra Forero" record still teaches it.
|
|
1524
|
+
families_by_display: dict[str, set[str]] = defaultdict(set)
|
|
1525
|
+
candidates: list[tuple[str, str]] = []
|
|
1526
|
+
for items in enrich_map.values():
|
|
1527
|
+
for item in items:
|
|
1528
|
+
family = item.get("family")
|
|
1529
|
+
if not isinstance(family, str):
|
|
1530
|
+
continue
|
|
1531
|
+
family_norm = _norm_surname(family)
|
|
1532
|
+
display = item.get("display_name")
|
|
1533
|
+
display_norm = _norm_surname(display) if isinstance(display, str) else ""
|
|
1534
|
+
families_by_display[display_norm].add(family_norm)
|
|
1535
|
+
if " " not in family_norm:
|
|
1536
|
+
continue
|
|
1537
|
+
# CrossRef sometimes stuffs the entire name into `family`
|
|
1538
|
+
# (with an empty `given`); a family that swallows the whole
|
|
1539
|
+
# display name attests nothing about the surname boundary.
|
|
1540
|
+
if display_norm and display_norm == family_norm:
|
|
1541
|
+
continue
|
|
1542
|
+
candidates.append((display_norm, family_norm))
|
|
1543
|
+
for display_norm, family_norm in candidates:
|
|
1544
|
+
shorter = {
|
|
1545
|
+
other
|
|
1546
|
+
for other in families_by_display[display_norm] - {family_norm}
|
|
1547
|
+
if other and family_norm.endswith(f" {other}")
|
|
1548
|
+
}
|
|
1549
|
+
if display_norm and shorter:
|
|
1550
|
+
continue
|
|
1551
|
+
lexicon.add(family_norm)
|
|
1552
|
+
return frozenset(lexicon)
|
|
1553
|
+
|
|
1554
|
+
|
|
1555
|
+
def _row_authors(row: pd.Series) -> list[str]:
|
|
1556
|
+
"""Pull a list of author strings from a DataFrame row.
|
|
1557
|
+
|
|
1558
|
+
``Authors_List`` is preferred (the canonical schema field) and round-
|
|
1559
|
+
trips through CSV as a Python repr that we parse back with
|
|
1560
|
+
:func:`ast.literal_eval`. When only the comma-joined ``Authors``
|
|
1561
|
+
string is available — e.g. a ``references.csv`` written before
|
|
1562
|
+
Authors_List was preserved through dedup — we fall back to
|
|
1563
|
+
:func:`_split_joined_authors`, which is safe against the
|
|
1564
|
+
``"Smith, J., García, A."`` pattern.
|
|
1565
|
+
"""
|
|
1566
|
+
val = row.get("Authors_List")
|
|
1567
|
+
if isinstance(val, list):
|
|
1568
|
+
return [str(a) if a is not None else "" for a in val]
|
|
1569
|
+
if isinstance(val, str):
|
|
1570
|
+
try:
|
|
1571
|
+
slots = ast.literal_eval(val)
|
|
1572
|
+
except (ValueError, SyntaxError):
|
|
1573
|
+
slots = None
|
|
1574
|
+
if isinstance(slots, list):
|
|
1575
|
+
return [str(a) if a is not None else "" for a in slots]
|
|
1576
|
+
parsed = parse_authors_list(val)
|
|
1577
|
+
if len(parsed) > 1:
|
|
1578
|
+
return parsed
|
|
1579
|
+
if parsed:
|
|
1580
|
+
return _split_joined_authors(parsed[0])
|
|
1581
|
+
fallback = row.get("Authors")
|
|
1582
|
+
if isinstance(fallback, str) and fallback.strip():
|
|
1583
|
+
return _split_joined_authors(fallback)
|
|
1584
|
+
return []
|
|
1585
|
+
|
|
1586
|
+
|
|
1587
|
+
def _split_joined_authors(s: str) -> list[str]:
|
|
1588
|
+
"""Split a delimiter-joined author string into individual names.
|
|
1589
|
+
|
|
1590
|
+
Prefers ``;`` (unambiguous) when present. Otherwise splits on ``,``
|
|
1591
|
+
and re-glues obvious ``Surname, Given`` pairs so that
|
|
1592
|
+
``"Smith, J., García, A."`` and ``"Smith, John, García, Ana"`` become
|
|
1593
|
+
two authors rather than four fragments. Strings without that pattern
|
|
1594
|
+
(e.g. ``"John Smith, Mary Jones"``) pass through unchanged.
|
|
1595
|
+
"""
|
|
1596
|
+
s = s.strip()
|
|
1597
|
+
if not s:
|
|
1598
|
+
return []
|
|
1599
|
+
if ";" in s:
|
|
1600
|
+
return [p.strip() for p in s.split(";") if p.strip()]
|
|
1601
|
+
|
|
1602
|
+
parts = [p.strip() for p in s.split(",") if p.strip()]
|
|
1603
|
+
out: list[str] = []
|
|
1604
|
+
i = 0
|
|
1605
|
+
while i < len(parts):
|
|
1606
|
+
cur = parts[i]
|
|
1607
|
+
nxt = parts[i + 1] if i + 1 < len(parts) else None
|
|
1608
|
+
if (
|
|
1609
|
+
nxt is not None
|
|
1610
|
+
and not _is_initial_only_token(cur)
|
|
1611
|
+
and (
|
|
1612
|
+
_is_initial_only_token(nxt)
|
|
1613
|
+
or _looks_like_surname_given_pair(cur, nxt)
|
|
1614
|
+
)
|
|
1615
|
+
):
|
|
1616
|
+
out.append(f"{cur}, {nxt}")
|
|
1617
|
+
i += 2
|
|
1618
|
+
else:
|
|
1619
|
+
out.append(cur)
|
|
1620
|
+
i += 1
|
|
1621
|
+
return out
|
|
1622
|
+
|
|
1623
|
+
|
|
1624
|
+
def _looks_like_surname_given_pair(surname_part: str, given_part: str) -> bool:
|
|
1625
|
+
"""Heuristic for comma-joined fallback strings.
|
|
1626
|
+
|
|
1627
|
+
This intentionally handles only high-confidence cases: one-word
|
|
1628
|
+
surnames (``Smith, John``) and particle compounds
|
|
1629
|
+
(``de la Cruz, Juan``). Multi-word chunks such as ``Talbot Page`` are
|
|
1630
|
+
likely already in first-last order, so pairing them with the next
|
|
1631
|
+
comma part would create a false author.
|
|
1632
|
+
"""
|
|
1633
|
+
surname_words = surname_part.split()
|
|
1634
|
+
given_words = given_part.split()
|
|
1635
|
+
if not surname_words or not given_words or len(given_words) > 2:
|
|
1636
|
+
return False
|
|
1637
|
+
if len(surname_words) == 1:
|
|
1638
|
+
return True
|
|
1639
|
+
return any(w.strip(".-").lower() in _PARTICLES for w in surname_words[:-1])
|
|
1640
|
+
|
|
1641
|
+
|
|
1642
|
+
def _given_subsequence(a: tuple, b: tuple) -> bool:
|
|
1643
|
+
"""Recognize omitted given names without admitting conflicting full words."""
|
|
1644
|
+
short, long = sorted((a, b), key=len)
|
|
1645
|
+
if not short or len(short) == len(long):
|
|
1646
|
+
return False
|
|
1647
|
+
remaining = iter(long)
|
|
1648
|
+
return all(any(_units_compatible((unit,), (other,)) for other in remaining)
|
|
1649
|
+
for unit in short)
|
|
1650
|
+
|
|
1651
|
+
|
|
1652
|
+
def _match_enrichment_authors(
|
|
1653
|
+
parsed: list[ParsedAuthor | None],
|
|
1654
|
+
enrichment: list[dict],
|
|
1655
|
+
known_surnames: frozenset[str],
|
|
1656
|
+
audit: list[dict],
|
|
1657
|
+
) -> list[tuple[str | None, str | None]]:
|
|
1658
|
+
"""Attach only mutually unique names, consuming each provider slot once."""
|
|
1659
|
+
providers = [parse_author(item.get("display_name") or "", known_surnames=known_surnames)
|
|
1660
|
+
for item in enrichment]
|
|
1661
|
+
pairs: dict[tuple[int, int], str] = {}
|
|
1662
|
+
evidence: list[list[dict]] = [[] for _ in parsed]
|
|
1663
|
+
for i, author in enumerate(parsed):
|
|
1664
|
+
for j, candidate in enumerate(providers):
|
|
1665
|
+
reason = "unusable_name"
|
|
1666
|
+
tier = None
|
|
1667
|
+
if author is not None and candidate is not None:
|
|
1668
|
+
au, cu = _given_units(author), _given_units(candidate)
|
|
1669
|
+
aw = {v for k, v in au if k == "w"}
|
|
1670
|
+
cw = {v for k, v in cu if k == "w"}
|
|
1671
|
+
single, compound = sorted((author.surname_norm, candidate.surname_norm),
|
|
1672
|
+
key=lambda name: len(name.split()))
|
|
1673
|
+
surname_ok = author.surname_norm == candidate.surname_norm or (
|
|
1674
|
+
len(single.split()) == 1 and single not in _PARTICLES
|
|
1675
|
+
and len(compound.split()) > 1
|
|
1676
|
+
and single in (compound.split()[0], compound.split()[-1])
|
|
1677
|
+
and bool(aw & cw)
|
|
1678
|
+
)
|
|
1679
|
+
if author.is_corporate != candidate.is_corporate:
|
|
1680
|
+
reason = "corporate_person_conflict"
|
|
1681
|
+
elif not surname_ok:
|
|
1682
|
+
reason = "surname_conflict"
|
|
1683
|
+
elif author.is_corporate:
|
|
1684
|
+
tier, reason = "full", "exact_corporate_name"
|
|
1685
|
+
elif not au or not cu:
|
|
1686
|
+
reason = "missing_given_name"
|
|
1687
|
+
elif not (_units_compatible(au, cu) or _given_subsequence(au, cu)):
|
|
1688
|
+
reason = "given_name_conflict"
|
|
1689
|
+
else:
|
|
1690
|
+
tier = "full" if aw and cw and aw == cw else "initial"
|
|
1691
|
+
reason = "compatible_" + tier
|
|
1692
|
+
if tier:
|
|
1693
|
+
pairs[i, j] = tier
|
|
1694
|
+
evidence[i].append({"provider_position": j, "reason": reason, "tier": tier})
|
|
1695
|
+
assigned: dict[int, int] = {}
|
|
1696
|
+
used: set[int] = set()
|
|
1697
|
+
for tier in ("full", "initial"):
|
|
1698
|
+
while True:
|
|
1699
|
+
candidates = {(i, j) for (i, j), t in pairs.items()
|
|
1700
|
+
if i not in assigned and j not in used
|
|
1701
|
+
and (tier == "initial" or t == "full")}
|
|
1702
|
+
round_pairs = [(i, j) for i, j in sorted(candidates)
|
|
1703
|
+
if sum(a == i for a, _ in candidates) == 1
|
|
1704
|
+
and sum(b == j for _, b in candidates) == 1]
|
|
1705
|
+
if not round_pairs:
|
|
1706
|
+
break
|
|
1707
|
+
assigned.update(round_pairs)
|
|
1708
|
+
used.update(j for _, j in round_pairs)
|
|
1709
|
+
result = []
|
|
1710
|
+
for i, author in enumerate(parsed):
|
|
1711
|
+
j = assigned.get(i)
|
|
1712
|
+
item = enrichment[j] if j is not None else {}
|
|
1713
|
+
ids = tuple(
|
|
1714
|
+
value.strip().rstrip("/").rsplit("/", 1)[-1] if isinstance(value, str) and value.strip() else None
|
|
1715
|
+
for value in (item.get("openalex_id"), item.get("orcid"))
|
|
1716
|
+
)
|
|
1717
|
+
result.append(ids)
|
|
1718
|
+
decision = ("assigned" if j is not None else "ambiguous"
|
|
1719
|
+
if any(a == i for a, _ in pairs) else "conflict" if enrichment else "unmatched")
|
|
1720
|
+
audit.append({"position": i, "raw_author": author.raw if author else None,
|
|
1721
|
+
"normalized_name": {"surname": author.surname_norm,
|
|
1722
|
+
"given_units": list(_given_units(author))} if author else None,
|
|
1723
|
+
"candidates": evidence[i], "provider_position": j,
|
|
1724
|
+
"openalex_id": ids[0], "orcid": ids[1], "decision": decision})
|
|
1725
|
+
for j, candidate in enumerate(providers):
|
|
1726
|
+
if j not in used:
|
|
1727
|
+
audit.append({"position": None, "raw_author": None,
|
|
1728
|
+
"provider_position": j,
|
|
1729
|
+
"provider_name": candidate.raw if candidate else None,
|
|
1730
|
+
"decision": "conflict", "detail": "unassigned_provider_author"})
|
|
1731
|
+
return result
|
|
1732
|
+
|
|
1733
|
+
|
|
1734
|
+
def _review_reason(
|
|
1735
|
+
occs: list[AuthorOccurrence],
|
|
1736
|
+
cfg: AuthorClusterConfig,
|
|
1737
|
+
block_keys: frozenset[str] = frozenset(),
|
|
1738
|
+
) -> str | None:
|
|
1739
|
+
"""Return a short reason string when the cluster looks low-confidence."""
|
|
1740
|
+
if any(o.parsed.is_corporate for o in occs):
|
|
1741
|
+
return "detected as corporate author (not a person)"
|
|
1742
|
+
has_external = any(o.openalex_id or o.orcid for o in occs)
|
|
1743
|
+
if has_external:
|
|
1744
|
+
return None
|
|
1745
|
+
for o in occs:
|
|
1746
|
+
p = o.parsed
|
|
1747
|
+
if "," in p.raw:
|
|
1748
|
+
continue
|
|
1749
|
+
words = [v for kind, v in _given_units(p) if kind == "w"]
|
|
1750
|
+
if len(words) >= 3:
|
|
1751
|
+
# 'Jose Alberto Guerra Forero' parsed conservatively — the
|
|
1752
|
+
# trailing given names may really be a compound surname the
|
|
1753
|
+
# corpus could not corroborate.
|
|
1754
|
+
return "possible compound surname: multiple full given names in no-comma form"
|
|
1755
|
+
if len(words) == 2 and words[-1] in block_keys and words[-1] != p.surname_norm:
|
|
1756
|
+
return "possible compound surname: a given name matches another surname block"
|
|
1757
|
+
has_full = any(o.parsed.has_full_first for o in occs)
|
|
1758
|
+
if not has_full and len(occs) >= 3:
|
|
1759
|
+
# Initial-only cluster with several citations — most worth checking.
|
|
1760
|
+
return "initial-only cluster with no full-name evidence"
|
|
1761
|
+
return None
|
|
1762
|
+
|
|
1763
|
+
|
|
1764
|
+
def _apply_aliases(
|
|
1765
|
+
clusters: list[AuthorCluster],
|
|
1766
|
+
aliases: dict[str, str],
|
|
1767
|
+
) -> list[AuthorCluster]:
|
|
1768
|
+
"""Merge clusters according to ``aliases``: {cluster_id: canonical_id}.
|
|
1769
|
+
|
|
1770
|
+
Aliases are followed transitively (A→B, B→C ⇒ A absorbed into C).
|
|
1771
|
+
Reject stale identifiers and cycles so corrections cannot silently disappear.
|
|
1772
|
+
"""
|
|
1773
|
+
by_id = {c.id: c for c in clusters}
|
|
1774
|
+
stale = (set(aliases) | set(aliases.values())) - set(by_id)
|
|
1775
|
+
if stale:
|
|
1776
|
+
raise ValueError(f'Stale author aliases: {sorted(stale)}; review against current authors')
|
|
1777
|
+
|
|
1778
|
+
def resolve(cid: str, seen: set[str]) -> str:
|
|
1779
|
+
if cid in seen:
|
|
1780
|
+
raise ValueError(f'Author alias cycle involving {cid}')
|
|
1781
|
+
if cid not in aliases:
|
|
1782
|
+
return cid
|
|
1783
|
+
seen.add(cid)
|
|
1784
|
+
return resolve(aliases[cid], seen)
|
|
1785
|
+
|
|
1786
|
+
groups: dict[str, list[AuthorOccurrence]] = defaultdict(list)
|
|
1787
|
+
for c in clusters:
|
|
1788
|
+
groups[resolve(c.id, set())].extend(c.occurrences)
|
|
1789
|
+
for target, group in groups.items():
|
|
1790
|
+
if len({o.orcid for o in group if o.orcid}) > 1:
|
|
1791
|
+
raise ValueError(f'Author alias {target} has conflicting ORCIDs')
|
|
1792
|
+
if any((r.record_id, r.position) in member.cannot_link for member in group for r in group):
|
|
1793
|
+
raise ValueError(f'Author alias {target} contradicts explicit separate constraint')
|
|
1794
|
+
merged: dict[str, AuthorCluster] = {}
|
|
1795
|
+
for c in clusters:
|
|
1796
|
+
target_id = resolve(c.id, set())
|
|
1797
|
+
if target_id == c.id:
|
|
1798
|
+
merged.setdefault(c.id, c)
|
|
1799
|
+
continue
|
|
1800
|
+
target = by_id.get(target_id)
|
|
1801
|
+
if target is None:
|
|
1802
|
+
merged.setdefault(c.id, c)
|
|
1803
|
+
continue
|
|
1804
|
+
absorber = merged.setdefault(target.id, target)
|
|
1805
|
+
absorber.occurrences.extend(c.occurrences)
|
|
1806
|
+
# Pull external ids forward if the absorber didn't have them.
|
|
1807
|
+
absorber.openalex_id = absorber.openalex_id or c.openalex_id
|
|
1808
|
+
absorber.orcid = absorber.orcid or c.orcid
|
|
1809
|
+
|
|
1810
|
+
return list(merged.values())
|
|
1811
|
+
|
|
1812
|
+
|
|
1813
|
+
def _clusters_to_authors_df(
|
|
1814
|
+
clusters: list[AuthorCluster],
|
|
1815
|
+
*,
|
|
1816
|
+
works: pd.DataFrame,
|
|
1817
|
+
citation_edges: pd.DataFrame | None = None,
|
|
1818
|
+
) -> pd.DataFrame:
|
|
1819
|
+
"""Render clusters as the on-disk ``authors.csv`` shape.
|
|
1820
|
+
|
|
1821
|
+
Metrics per canonical author:
|
|
1822
|
+
|
|
1823
|
+
- ``n_works`` — distinct canonical works the author appears on,
|
|
1824
|
+
- ``n_core_works`` — of those, how many are ring 0 (the user's corpus),
|
|
1825
|
+
- ``n_citations_received`` — citation edges into any of their works,
|
|
1826
|
+
- ``n_distinct_citing_works`` — distinct works doing that citing.
|
|
1827
|
+
"""
|
|
1828
|
+
ring_by_work = works["ring"].to_dict() if "ring" in works.columns else {}
|
|
1829
|
+
in_edges: dict[str, set[str]] = defaultdict(set)
|
|
1830
|
+
n_in_edges: dict[str, int] = defaultdict(int)
|
|
1831
|
+
if citation_edges is not None and not citation_edges.empty:
|
|
1832
|
+
require_columns(
|
|
1833
|
+
citation_edges,
|
|
1834
|
+
["citing_id", "cited_id"],
|
|
1835
|
+
artifact="citation graph",
|
|
1836
|
+
)
|
|
1837
|
+
for _, row in citation_edges.iterrows():
|
|
1838
|
+
cited_id = row.get("cited_id")
|
|
1839
|
+
citing_id = row.get("citing_id")
|
|
1840
|
+
if cited_id and citing_id:
|
|
1841
|
+
in_edges[str(cited_id)].add(str(citing_id))
|
|
1842
|
+
n_in_edges[str(cited_id)] += 1
|
|
1843
|
+
|
|
1844
|
+
rows = []
|
|
1845
|
+
for c in clusters:
|
|
1846
|
+
work_ids = {o.record_id for o in c.occurrences}
|
|
1847
|
+
citing: set[str] = set()
|
|
1848
|
+
for w in work_ids:
|
|
1849
|
+
citing |= in_edges.get(w, set())
|
|
1850
|
+
rows.append(
|
|
1851
|
+
{
|
|
1852
|
+
"id": c.id,
|
|
1853
|
+
"display_name": c.display_name,
|
|
1854
|
+
"surname": c.surname,
|
|
1855
|
+
"surname_norm": c.surname_norm,
|
|
1856
|
+
"canonical_given": c.canonical_given,
|
|
1857
|
+
"initials": c.initials,
|
|
1858
|
+
"openalex_id": c.openalex_id,
|
|
1859
|
+
"orcid": c.orcid,
|
|
1860
|
+
"n_works": len(work_ids),
|
|
1861
|
+
"n_core_works": sum(1 for w in work_ids if ring_by_work.get(w) == 0),
|
|
1862
|
+
"n_citations_received": sum(n_in_edges.get(w, 0) for w in work_ids),
|
|
1863
|
+
"n_distinct_citing_works": len(citing),
|
|
1864
|
+
}
|
|
1865
|
+
)
|
|
1866
|
+
df = pd.DataFrame(rows, columns=AUTHOR_COLUMNS)
|
|
1867
|
+
if df.empty:
|
|
1868
|
+
return df.set_index("id")
|
|
1869
|
+
# Stable two-key sort: without edges, authorship volume still yields a
|
|
1870
|
+
# deterministic, meaningful order.
|
|
1871
|
+
return df.set_index("id").sort_values(
|
|
1872
|
+
["n_citations_received", "n_works"],
|
|
1873
|
+
ascending=[False, False],
|
|
1874
|
+
kind="mergesort",
|
|
1875
|
+
)
|
|
1876
|
+
|
|
1877
|
+
|
|
1878
|
+
def _clusters_to_citations_df(clusters: list[AuthorCluster]) -> pd.DataFrame:
|
|
1879
|
+
"""Render the per-occurrence edges as ``author_citations.csv``."""
|
|
1880
|
+
rows = []
|
|
1881
|
+
for c in clusters:
|
|
1882
|
+
for o in c.occurrences:
|
|
1883
|
+
rows.append(
|
|
1884
|
+
{
|
|
1885
|
+
"author_id": c.id,
|
|
1886
|
+
"record_id": o.record_id,
|
|
1887
|
+
"position": o.position,
|
|
1888
|
+
"raw_author": o.parsed.raw,
|
|
1889
|
+
}
|
|
1890
|
+
)
|
|
1891
|
+
return pd.DataFrame(rows, columns=AUTHOR_CITATION_COLUMNS)
|
|
1892
|
+
|
|
1893
|
+
|
|
1894
|
+
def _apply_external_ids(
|
|
1895
|
+
clusters: list[AuthorCluster],
|
|
1896
|
+
external_ids: dict[str, dict[str, str | None]],
|
|
1897
|
+
) -> None:
|
|
1898
|
+
"""Stamp hand-curated identifiers onto canonical authors, in place.
|
|
1899
|
+
|
|
1900
|
+
Applied *after* identity reconciliation, because the file keys on the
|
|
1901
|
+
published ``a-`` id the user actually sees. A curated value wins over
|
|
1902
|
+
whatever the provider supplied: someone who opened the provider record
|
|
1903
|
+
and checked is better evidence than a name match that failed.
|
|
1904
|
+
|
|
1905
|
+
This deliberately does not merge anything. ``author_aliases.csv`` decides
|
|
1906
|
+
who is the same person; letting an identifier also force merges would put
|
|
1907
|
+
two curated files in charge of the same question.
|
|
1908
|
+
"""
|
|
1909
|
+
by_id = {c.id: c for c in clusters}
|
|
1910
|
+
stale = sorted(set(external_ids) - set(by_id))
|
|
1911
|
+
if stale:
|
|
1912
|
+
raise ValueError(
|
|
1913
|
+
f'Stale author external ids: {stale}; review against current authors.csv'
|
|
1914
|
+
)
|
|
1915
|
+
claimed: dict[tuple[str, str], str] = {}
|
|
1916
|
+
for author_id, ids in external_ids.items():
|
|
1917
|
+
for field_name in ("openalex_id", "orcid"):
|
|
1918
|
+
value = ids.get(field_name)
|
|
1919
|
+
if not value:
|
|
1920
|
+
continue
|
|
1921
|
+
owner = claimed.get((field_name, value))
|
|
1922
|
+
if owner is not None and owner != author_id:
|
|
1923
|
+
raise ValueError(
|
|
1924
|
+
f'External {field_name} {value} claimed by both {owner} and '
|
|
1925
|
+
f'{author_id}; one identifier cannot name two authors'
|
|
1926
|
+
)
|
|
1927
|
+
claimed[field_name, value] = author_id
|
|
1928
|
+
setattr(by_id[author_id], field_name, value)
|
|
1929
|
+
|
|
1930
|
+
|
|
1931
|
+
def load_external_ids(path: Path | str | None) -> dict[str, dict[str, str | None]]:
|
|
1932
|
+
"""Load ``author_id,openalex_id,orcid`` overrides keyed by author id.
|
|
1933
|
+
|
|
1934
|
+
Missing path or file returns ``{}``. Blank identifier cells mean "leave
|
|
1935
|
+
this one alone", so a row can set only an ORCID; a row that sets neither
|
|
1936
|
+
is a mistake and raises rather than being silently ignored. Any further
|
|
1937
|
+
columns (a ``note`` explaining the override) are read and discarded.
|
|
1938
|
+
"""
|
|
1939
|
+
if path is None:
|
|
1940
|
+
return {}
|
|
1941
|
+
p = Path(path)
|
|
1942
|
+
if not p.exists():
|
|
1943
|
+
return {}
|
|
1944
|
+
out: dict[str, dict[str, str | None]] = {}
|
|
1945
|
+
with p.open("r", encoding="utf-8", newline="") as handle:
|
|
1946
|
+
rows = list(csv.DictReader(_strip_comments(handle)))
|
|
1947
|
+
for row in rows:
|
|
1948
|
+
author_id = (row.get("author_id") or "").strip()
|
|
1949
|
+
if not author_id:
|
|
1950
|
+
continue
|
|
1951
|
+
openalex_id = (row.get("openalex_id") or "").strip() or None
|
|
1952
|
+
orcid = (row.get("orcid") or "").strip() or None
|
|
1953
|
+
if not openalex_id and not orcid:
|
|
1954
|
+
raise ValueError(
|
|
1955
|
+
f'Author external id row for {author_id} sets no identifier; '
|
|
1956
|
+
'give an openalex_id or an orcid, or delete the row'
|
|
1957
|
+
)
|
|
1958
|
+
if author_id in out:
|
|
1959
|
+
raise ValueError(f'Duplicate author external id row for {author_id}')
|
|
1960
|
+
out[author_id] = {"openalex_id": openalex_id, "orcid": orcid}
|
|
1961
|
+
return out
|
|
1962
|
+
|
|
1963
|
+
|
|
1964
|
+
def _strip_comments(lines):
|
|
1965
|
+
for line in lines:
|
|
1966
|
+
if not line.lstrip().startswith("#"):
|
|
1967
|
+
yield line
|
|
1968
|
+
|
|
1969
|
+
|
|
1970
|
+
def load_aliases(path: Path | str | None) -> dict[str, str]:
|
|
1971
|
+
"""Load a ``cluster_id,canonical_id`` CSV into a dict.
|
|
1972
|
+
|
|
1973
|
+
Empty / missing path returns ``{}``. Header row optional. Comments
|
|
1974
|
+
(lines starting with ``#``) are ignored.
|
|
1975
|
+
"""
|
|
1976
|
+
if path is None:
|
|
1977
|
+
return {}
|
|
1978
|
+
p = Path(path)
|
|
1979
|
+
if not p.exists():
|
|
1980
|
+
return {}
|
|
1981
|
+
out: dict[str, str] = {}
|
|
1982
|
+
with p.open("r", encoding="utf-8") as f:
|
|
1983
|
+
for line in f:
|
|
1984
|
+
line = line.strip()
|
|
1985
|
+
if not line or line.startswith("#"):
|
|
1986
|
+
continue
|
|
1987
|
+
parts = [x.strip() for x in line.split(",")]
|
|
1988
|
+
if len(parts) < 2:
|
|
1989
|
+
continue
|
|
1990
|
+
src, dst = parts[0], parts[1]
|
|
1991
|
+
if src.lower() == "cluster_id" and dst.lower() == "canonical_id":
|
|
1992
|
+
continue # header
|
|
1993
|
+
if src and dst:
|
|
1994
|
+
if src in out and out[src] != dst:
|
|
1995
|
+
raise ValueError(f'Author alias conflict for {src}: {out[src]} versus {dst}')
|
|
1996
|
+
out[src] = dst
|
|
1997
|
+
return out
|