mjpn 6.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
mjpn/__init__.py ADDED
@@ -0,0 +1,40 @@
1
+ """MJPN — Moroccan Jewish Phonetic Names.
2
+
3
+ Computes the phonetic root of Moroccan Jewish surnames, exactly the
4
+ pipeline behind yahasra.org's burials.phonetic_index (normalize_to_phonetic
5
+ v3 stages -> v4 subroots -> compound dual indexing).
6
+
7
+ Typical use:
8
+
9
+ >>> from mjpn import normalize_to_phonetic
10
+ >>> normalize_to_phonetic('OUAKNINE')
11
+ 'AKNN'
12
+
13
+ Or from the command line:
14
+
15
+ mjpn myfile.xlsx # xlsx with a "Names" column
16
+ mjpn --name COHEN OUAKNINE
17
+ """
18
+ from .mjpn_tool import (
19
+ __version__ as engine_version,
20
+ normalize_to_phonetic,
21
+ normalize_to_phonetic_v4,
22
+ compute_compound_indices,
23
+ compute_compound_indices_v4,
24
+ check_special_mapping,
25
+ process_name,
26
+ main,
27
+ )
28
+
29
+ __version__ = "6.0.0"
30
+
31
+ __all__ = [
32
+ "engine_version",
33
+ "normalize_to_phonetic",
34
+ "normalize_to_phonetic_v4",
35
+ "compute_compound_indices",
36
+ "compute_compound_indices_v4",
37
+ "check_special_mapping",
38
+ "process_name",
39
+ "main",
40
+ ]
mjpn/mjpn_tool.py ADDED
@@ -0,0 +1,1035 @@
1
+ #!/usr/bin/env python3
2
+ """
3
+ MJPN xlsx tool — GENERATED, DO NOT EDIT BY HAND.
4
+
5
+ Regenerate with: python3 scripts/build_mjpn_tool.py
6
+ Generated 2026-11-16 from app/phonetic_normalizer.py (engine 6.0.0, v4 chain)
7
+ with 98 exact + 1 regex special mappings from the DB.
8
+
9
+ Moroccan Jewish Phonetic Names (MJPN): computes the phonetic root of
10
+ Moroccan Jewish surnames, exactly the pipeline behind yahasra.org's
11
+ burials.phonetic_index (normalize_to_phonetic v3 stages -> v4 subroots ->
12
+ compound dual indexing).
13
+
14
+ Usage:
15
+ python3 mjpn_tool.py # first .xlsx in cwd / next to this file
16
+ python3 mjpn_tool.py myfile.xlsx # explicit file
17
+ python3 mjpn_tool.py --name COHEN OUAKNINE # ad-hoc names, no xlsx
18
+
19
+ Input: an .xlsx with a column named "Names" (case-insensitive).
20
+ Output: <input>_mjpn.xlsx — sheet "MJPN" (name, root, root2, subroot,
21
+ subroot2, special_mapping, status) + sheet "Stats"; stats are also
22
+ printed to stdout.
23
+
24
+ Only non-stdlib dependency: openpyxl.
25
+ """
26
+ import argparse
27
+ import logging
28
+ import re
29
+ import sys
30
+ from collections import Counter, defaultdict
31
+ from pathlib import Path
32
+
33
+ __version__ = "6.0.0+v4"
34
+
35
+ logger = logging.getLogger("mjpn_tool")
36
+
37
+ VOWELS = 'AEIOU'
38
+ ACCENT_MAP = {'À': 'A', 'Á': 'A', 'Â': 'A', 'Ã': 'A', 'Ä': 'A', 'Å': 'A', 'È': 'E', 'É': 'E', 'Ê': 'E', 'Ë': 'E', 'Ì': 'I', 'Í': 'I', 'Î': 'I', 'Ï': 'I', 'Ò': 'O', 'Ó': 'O', 'Ô': 'O', 'Õ': 'O', 'Ö': 'O', 'Ù': 'U', 'Ú': 'U', 'Û': 'U', 'Ü': 'U', 'Ý': 'Y', 'Ÿ': 'Y', 'Ñ': 'N', 'Ç': 'S'}
39
+ LATIN_PREFIXES = ['ABEN', 'BEN', 'IBN', 'ABN', 'BAR', 'OU', 'BN', 'W', 'DE', 'DI']
40
+ HA_STRUCTURAL_NAMES = {'HAZAN', 'HAYON', 'AIM', 'HALIOUA', 'ASSAN', 'DAD', 'HAMU', 'HAZZAN', 'AMU', 'HASSANI', 'AZAN', 'HAYM', 'HAYOUN', 'AYM', 'HAMOU', 'ALIWA', 'HADAD', 'HAYUN', 'HALIWA', 'AMOU', 'HASSAN', 'HAIM'}
41
+ HA_KNOWN_SURNAMES = ['COHEN', 'KOHEN', 'KOHN', 'COEN', 'CAHEN', 'LEVY', 'LEVI', 'LEVIE', 'LEWY', 'NAGID', 'NASI']
42
+ HA_ARTICLE_NAMES = {'HASSARFATI', 'HAZERFATY', 'HASSERFATY OZIEL', 'HASSARFATY', 'HASSERFATY', 'HASSERFATI'}
43
+ EL_STRUCTURAL_NAMES = {'EL FASSY', 'EL-FASSY', 'ELFASSY'}
44
+ OU_STRUCTURAL_NAMES = {'WAHNOUNOU', 'OUZIELLE', 'OUAHNOUN', 'OUANOUNOU', 'OUZIELE', 'OUAHNON', 'OUZIELI', 'OUZIEL', 'OUAZIEL', 'WAHNOUN', 'WAHNON', 'OUHANOUNOU'}
45
+ AL_STRUCTURAL_NAMES = set()
46
+ BAR_STRUCTURAL_NAMES = {'BARZILAH', 'BARZILLAY', 'BARZILA', 'BARZILAI', 'BARZILLAI', 'BARSILLAI', 'BARSILAI', 'BARZILAY'}
47
+ GUA_EXCEPTIONS = {'LUGUASSY', 'LUGUASI', 'LUGUASSI'}
48
+ WA_EXCEPTIONS = {'GRUNWALD', 'HARWAY', 'GRUNEWALD', 'SCHWARTZ', 'SCHWARZ', 'SCHWARTZ SZERENA', 'SHWARTZ', 'BOHMERWALD'}
49
+ VA_TO_OUA_NAMES = {'VAQNINE', 'VAQNIN', 'VAKNIN', 'VAKNINE'}
50
+ INVALID_SURNAMES = {'BAR', 'NOM', 'HA', 'VIDE', 'ANONYME', 'BEN…', 'BAT', 'OU', '???', 'NN', 'N.N.', 'NEANT', 'IBN', 'EL', 'INCONNU', 'BEN...', 'FECHA .', 'FECHA .', 'ILLISIBLE', 'ABEN', 'AL', 'FECHA', 'BEN', 'SANS NOM', '...'}
51
+ NON_NAME_MARKERS = frozenset({'TEST', 'ESSAI', 'HEBREU', 'SANS INSCRIPTION', 'ILLISIBLE', 'VIDE', 'ANONYME', 'JUIF', 'INCONNU'})
52
+ _VOWEL_CLASS = {'A': 'A', 'E': 'A', 'O': 'O', 'U': 'O', 'I': 'I', 'Y': 'I'}
53
+ _INITIAL_VOWEL_CLASS = {'E': 'A', 'U': 'O', 'Y': 'I'}
54
+ _COMPOUND_PARTICLES = {'BAR', 'W', 'HA', 'OU', 'BN', 'DI', 'IBN', 'EL', 'LA', 'DA', 'LE', 'DE', 'ABI', 'ABN', 'ABEN', 'AL', 'BEN'}
55
+ _COMPOUND_NOISE_WORDS = {'INHUMATION', "D'AGADIR", 'ESTIMES', 'TOMBE', 'YESHIVA', 'FAMILLE', 'COLLECTIVE', 'ELEVES'}
56
+
57
+ # Embedded special mappings (from the mjpn_special_mappings DB table).
58
+ EXACT_MAPPINGS = {'LAGRABLI': 'GRBL', 'ABOUDAHM': 'ABDRHM', 'TADGI': 'ATG', 'MOZNINO': 'MSNN', 'ABISEROR': 'SRR', 'RIBO': 'RBB', 'ASSEDOUN': 'SDN', 'ASBAG': 'SBG', 'CHARROUT': 'RSH', 'BENHOYAL': 'LL', 'HÉBREU': 'LBR', 'ASSEBAN': 'SBN', 'ELGABAS': 'GB', 'SUSSAN': 'SN', 'ASSEDON': 'SDN', 'ELE...': 'LL', 'ZELL': 'ZLR', 'ZALLAG': 'AMZLG', 'BERTINI': 'BTN', 'LABBOUZ': 'LBZ', 'SAAD': 'ASD', 'BEHE': 'BEHE', 'AUBADIA': 'OBD', 'ATBOL': 'TBL', 'RABIN': 'NRBN', 'GHARBIT': 'SHRBT', 'ENSALEM': 'AMSLM', 'HAFFAF': 'ALF', 'BENSRMOUN': 'SMN', 'COUAZIZ': 'BZZ', 'ANSELLEM': 'AMSLM', 'BENCHEL': 'TL', 'DOUAZIZ': 'ODZ', 'BOLAZIZ': 'BZZ', 'OUBADIA': 'OBD', 'BOFFE': 'RF', 'TOUA': 'HT', 'D': 'HDD', 'ROHBOT': 'RBT', 'AKRICHE': 'KRS', 'CUENOUN': 'KHN', 'BELLICHAH': 'BLSH', 'ENCAQUA': 'ANK', 'MARTINEZ': 'MRTN', 'BEADUGO': 'BRDG', 'LANGRY': 'ANKR', 'ABOUHATZEIRA': 'ABHSR', 'TOPDJMAN': 'TRGMN', 'ALBAZ': 'BZ', 'GHOUKROUN': 'SHKRN', 'BENY': 'BNN', 'YAZERI': 'ZHR', 'BOUKHANA': 'AKNN', 'BENSOT': 'IST', 'DIWAN': 'DHN', 'SERELLY': 'ISRL', 'MIRELLY': 'MRHL', 'ELOLYO': 'LL', 'ZERAH': 'ZHR', 'BOUZACLO': 'BZGL', 'BENAUIS': 'BENAUIS', 'MOYA': 'MH', 'NAMIAS': 'NHM', 'ABOUAZIZ': 'BZ', 'OUAZIZ': 'ZZ', 'BERSAMOUD': 'ABRMD', 'BENSAMOUD': 'SMN', 'CHOCHRON': 'SHKRN', 'ABOUHATZIRA': 'ABHSR', 'BENSADOU': 'SDN', 'SAAS': 'ASS', 'HENRI': 'NHR', 'ARRAR': 'HRR', 'MAIMON': 'MYMN', 'MAIMOUN': 'MYMN', 'MAYMON': 'MYMN', 'MEIMOUN': 'MYMN', 'BENMAIMON': 'MYMN', 'BEN MAIMON': 'MYMN', 'BOUHSIRA': 'ABHSR', 'BOUHSSIRA': 'ABHSR', 'BOHSIRA': 'ABHSR', 'ALMOSNINO': 'MSNN', 'ALMOZNINO': 'MSNN', 'ALMOSNINOS': 'MSNN', 'HALEOUI': 'LB', 'BATTINO': 'BETN', 'BETTINO': 'BETN', 'ALMALEH': 'MLH', 'ALMALIAH': 'MLH', 'ABTTANE': 'ABTN', 'MOYE?': 'HM', 'MAR YOSEF': 'ISF', 'MARYOSEF': 'ISF', 'MOR YOSSEF': 'ISF', 'MOR YUSSEF': 'ISF', 'MAHEODA': 'MD', 'BAREBAY': 'BB'}
59
+ REGEX_MAPPINGS = [(re.compile(p, re.IGNORECASE), r) for p, r in [('(ABU|ABOU|ABI|ABEH|ABIH|BOU|BOD|BO|BU)H?S+[EIY]R+A?', 'ABHSR')]]
60
+ REGEX_MAPPINGS.sort(key=lambda pr: len(pr[0].pattern), reverse=True)
61
+
62
+
63
+ def check_special_mapping(name):
64
+ """DB-free port of the engine's check_special_mapping (v4)."""
65
+ if not name:
66
+ return None
67
+ name_upper = name.upper()
68
+ if name_upper in EXACT_MAPPINGS:
69
+ return EXACT_MAPPINGS[name_upper]
70
+ no_sep = re.sub(r"[\s\-]+", "", name_upper)
71
+ if no_sep != name_upper and no_sep in EXACT_MAPPINGS:
72
+ return EXACT_MAPPINGS[no_sep]
73
+ for pat, root in REGEX_MAPPINGS:
74
+ if pat.fullmatch(name_upper) or (no_sep != name_upper and pat.fullmatch(no_sep)):
75
+ return root
76
+ return None
77
+
78
+
79
+ def normalize_accents(text):
80
+ """Remove accented characters, replacing with ASCII equivalents."""
81
+ if not text:
82
+ return text
83
+ result = []
84
+ for char in text.upper():
85
+ result.append(ACCENT_MAP.get(char, char))
86
+ return ''.join(result)
87
+
88
+ def is_fecha_artifact(value: str) -> bool:
89
+ """Return True if the value is a 'Fecha …' date artifact from the
90
+ Larache import (Phase 2e). Matches 'Fecha .', 'Fecha .', 'Fecha 1914'
91
+ etc., case-insensitive, any whitespace."""
92
+ if not value:
93
+ return False
94
+ s = value.strip().upper()
95
+ return s.startswith('FECHA') and (
96
+ s == 'FECHA' or s[5:].lstrip(' .0123456789') == ''
97
+ )
98
+
99
+ def _effective_consonant_count(text):
100
+ """E2: count consonants, collapsing a run of the same consonant to one, so
101
+ ABBOU counts 1 (BB), not 2. Keeps BENABBOU aligned with BENABOU."""
102
+ count = 0
103
+ prev = None
104
+ for c in text:
105
+ if c.isalpha() and c not in VOWELS:
106
+ if c != prev:
107
+ count += 1
108
+ prev = c
109
+ else:
110
+ prev = None
111
+ return count
112
+
113
+ def normalize_start_patterns(text):
114
+ """
115
+ STAGE 1: Normalize start-of-word patterns BEFORE prefix stripping.
116
+
117
+ This handles patterns like:
118
+ - OI at start → A (OIKNINE → AKNINE, then OAKNINE after full process)
119
+
120
+ This MUST happen before prefix stripping, otherwise:
121
+ - OIKNINE → O stripped → IKNINE → OI pattern never matches!
122
+ """
123
+ if not text or len(text) < 2:
124
+ return text
125
+
126
+ text = text.upper()
127
+
128
+ # OI at start → A (Berber variant of OU)
129
+ # OIKNINE should become AKNINE to match OUAKNINE→AKNINE
130
+ if text.startswith('OI'):
131
+ text = 'A' + text[2:]
132
+
133
+ # GUA / WA / OUA are one sound (owner rule, 2026-08-01). The Judeo-Arabic
134
+ # /wa/ was written OUA by a French clerk, WA by an English or Hebrew one,
135
+ # and GUA by a Spanish one, where <gu> before a vowel is exactly /w/.
136
+ # Collapsing them here, before prefix stripping, is what makes
137
+ # HALIWA = HALIOUA = HALIGUA
138
+ # BENOUALID = BENWALID = BENGUALID
139
+ # one family each. Left unnormalized these produced three roots apiece
140
+ # (HLB/HL/HLG and OLD/BLD/GLD), so the Spanish and English spellings of a
141
+ # family were unreachable from its French one.
142
+ #
143
+ # OUA is the target rather than WA because OU- already has a structural
144
+ # exception list (OUAHNON, OUANOUNOU) that the other spellings must land
145
+ # inside: GUAHNON has to reach the same protection as OUAHNON.
146
+ # GU-not-/w/: LUGUASSY is the LUGASSY family with an inserted u, not a
147
+ # Spanish spelling of /wa/. Checked before the rewrite, same mechanism as
148
+ # the HA-/OU-/BAR- exception lists.
149
+ # ONLY before A. Spanish <gu> is /gw/ before a and o, but a hard /g/
150
+ # before e and i (guerra, guitarra). A first draft of this rule matched
151
+ # GU before any vowel and destroyed 100+ names: GUIGUI lost its root
152
+ # entirely, BENGUIGUI became BN, MERGUI became MR, RODRIGUES became RDRS.
153
+ if text.upper() not in GUA_EXCEPTIONS:
154
+ text = re.sub(r'GUA', 'OUA', text)
155
+ # Medial WA counts too (HALIWA is HALI + /wa/), except in the Ashkenazi
156
+ # names in this corpus where <wa> is not the Judeo-Arabic /wa/ at all.
157
+ if text.upper() not in WA_EXCEPTIONS:
158
+ text = re.sub(r'WA', 'OUA', text)
159
+
160
+ # V before A at word-start is the Israeli transliteration of the same
161
+ # Berber wa- prefix that OUA/WA/GUA collapse to OUA above (Section 52.2,
162
+ # docs/MJPN_MONOGRAPH.md; verified defect 2026-08-08: VAKNIN alone reached
163
+ # BKNN, via the later V->B consonant-equivalence rule (Beider Table 2.7),
164
+ # while OUAKNINE/WAKNIN/AKNIN/OIKNINE/AKNINE all reach AKNN).
165
+ #
166
+ # This is a WHITELIST, unlike WA_EXCEPTIONS' blocklist: a corpus-wide dry
167
+ # run (data/reports/vaknin_rule_dryrun_20260808.csv) showed every OTHER
168
+ # VA-initial spelling in this archive (VALENCY, VALENCI, VALENZY, VATURI)
169
+ # -- and VALENSI/VIDAL/VITAL, checked explicitly though only VALENSI
170
+ # starts with VA -- is genuine Latin V (Beider: bet/vet) and must not
171
+ # move. A blocklist would require naming every future VA- Latin surname
172
+ # to protect it; a whitelist of the one attested family name does not.
173
+ if text in VA_TO_OUA_NAMES:
174
+ text = 'OUA' + text[2:]
175
+
176
+ return text
177
+
178
+ def strip_latin_prefix(text):
179
+ """
180
+ STAGE 2: Remove Latin patronymic prefix.
181
+
182
+ v3.5.0 CHANGES:
183
+ - Added: Generalized HA- stripping with HA_STRUCTURAL_NAMES exception list
184
+ - Added: OU_STRUCTURAL_NAMES exception list (OUANOUNOU, etc.)
185
+ - Added: EL_STRUCTURAL_NAMES exception list (ELFASSY)
186
+
187
+ NOTE: Special handling for:
188
+ - 'O': NOT stripped when followed by 'I' (OI→A already handled)
189
+ - 'HA': Generalized stripping EXCEPT for HA_STRUCTURAL_NAMES
190
+ - 'OU': Standard stripping EXCEPT for OU_STRUCTURAL_NAMES
191
+ - 'EL': Stripped EXCEPT for EL_STRUCTURAL_NAMES
192
+ """
193
+ if not text:
194
+ return text
195
+
196
+ text_upper = text.upper()
197
+
198
+ # v3.5.0: Check EL- structural names BEFORE prefix stripping
199
+ for el_name in EL_STRUCTURAL_NAMES:
200
+ clean_name = el_name.replace('-', '').replace(' ', '')
201
+ if text_upper == clean_name or text_upper == el_name:
202
+ return text_upper # Preserve EL- prefix
203
+
204
+ # v3.5.0: Check OU- structural names
205
+ for ou_name in OU_STRUCTURAL_NAMES:
206
+ if text_upper == ou_name or text_upper.startswith(ou_name):
207
+ return text_upper # Preserve OU- prefix
208
+
209
+ # BAR- structural names (BARZILAI, BARUCH): integral, never a patronymic
210
+ for bar_name in BAR_STRUCTURAL_NAMES:
211
+ if text_upper == bar_name or text_upper.startswith(bar_name):
212
+ return text_upper # Preserve BAR-
213
+
214
+ # Sort by length (longest first) to match correctly
215
+ sorted_prefixes = sorted(LATIN_PREFIXES, key=len, reverse=True)
216
+
217
+ for prefix in sorted_prefixes:
218
+ if text_upper.startswith(prefix):
219
+ remainder = text_upper[len(prefix):]
220
+ # Only strip if remainder has enough substance (2+ consonants)
221
+ consonants_in_remainder = _effective_consonant_count(remainder)
222
+ if consonants_in_remainder >= 2 and len(remainder) >= 2:
223
+ return remainder
224
+
225
+ # v3.5.0: GENERALIZED HA- stripping with exception list
226
+ # Strip HA- unless:
227
+ # 1. Name is in HA_STRUCTURAL_NAMES (HAMOU, HAIM, HADAD, HAZAN, HAYON, HAYOUN)
228
+ # 2. Remainder matches HA_KNOWN_SURNAMES (COHEN, LEVY, etc.)
229
+ # 3. Remainder is too short
230
+ if text_upper.startswith('HA') and len(text_upper) > 3:
231
+ remainder = text_upper[2:]
232
+
233
+ # HA- is the article here despite the structural list: strip it.
234
+ for article_name in HA_ARTICLE_NAMES:
235
+ if text_upper == article_name or text_upper.startswith(article_name):
236
+ return text_upper[2:]
237
+
238
+ # Check if this is a structural HA- name (HAMOU, HAIM, etc.)
239
+ for structural in HA_STRUCTURAL_NAMES:
240
+ if text_upper == structural or text_upper.startswith(structural[:4]):
241
+ return text_upper # Keep HA- prefix
242
+
243
+ # Check if remainder is a known surname (COHEN, LEVY)
244
+ for known in HA_KNOWN_SURNAMES:
245
+ if remainder.startswith(known) or remainder == known:
246
+ return remainder # HACOHEN -> COHEN
247
+
248
+ # v3.5.0: Generalized HA- stripping for OTHER names
249
+ # Strip HA- if remainder has enough substance
250
+ consonants_in_remainder = _effective_consonant_count(remainder)
251
+ if consonants_in_remainder >= 2 and len(remainder) >= 3:
252
+ return remainder # Strip HA- (e.g., HARARI -> ARARI)
253
+
254
+ # Handle single 'O' prefix specially
255
+ # O is a Berber patronymic prefix (variant of OU/W)
256
+ if text_upper.startswith('O') and len(text_upper) > 3:
257
+ remainder = text_upper[1:]
258
+ consonants_in_remainder = _effective_consonant_count(remainder)
259
+
260
+ # Only strip O if:
261
+ # - Remainder has enough consonants (2+)
262
+ # - AND NOT followed by 'I' (OI patterns handled in stage 1)
263
+ if consonants_in_remainder >= 2 and not remainder.startswith('I'):
264
+ # Pattern 1: O + H + vowel (OHAYON, OHANA)
265
+ if remainder.startswith('H') and len(remainder) > 1 and remainder[1] in VOWELS:
266
+ return remainder # OHAYON → HAYON
267
+
268
+ # Pattern 2: O + vowel + consonants (OAKNIN → AKNIN)
269
+ # This is a Berber variant where O replaces OU/W
270
+ if remainder[0] in VOWELS and len(remainder) > 2:
271
+ # Check that it's not a double vowel that forms a different sound
272
+ # OA-KNIN → AKNIN (strip O)
273
+ # But be careful with patterns like OOZE where OO is meaningful
274
+ if remainder[0] != 'O': # Avoid stripping O from OO-
275
+ return remainder # OAKNIN → AKNIN
276
+
277
+ # v3.5.0: EL-/AL- stripping (unless structural)
278
+ if text_upper.startswith('EL') and len(text_upper) > 4:
279
+ # Check if this is a structural EL- name
280
+ is_structural = False
281
+ for el_name in EL_STRUCTURAL_NAMES:
282
+ clean_name = el_name.replace('-', '').replace(' ', '')
283
+ if text_upper == clean_name:
284
+ is_structural = True
285
+ break
286
+
287
+ if not is_structural:
288
+ remainder = text_upper[2:]
289
+ consonants_in_remainder = _effective_consonant_count(remainder)
290
+ if consonants_in_remainder >= 2 and len(remainder) >= 3:
291
+ return remainder # Strip EL-
292
+
293
+ # E6 NOT applied as a blanket rule: the dry-run showed a generic AL- strip
294
+ # false-merges names where AL is integral (Allouche, Allouf, Alexandre,
295
+ # Alster, ...). That repeats the v3.4 Z->S over-merge (rolled back, I2), so
296
+ # AL- families are handled per-name via mjpn_special_mappings / adjudication.
297
+ # AL_STRUCTURAL_NAMES remains for a future targeted exception list.
298
+
299
+ return text_upper
300
+
301
+ def apply_context_c_rule(text):
302
+ """
303
+ Apply context-dependent C transformation (v3.4).
304
+
305
+ Per Beider Table 2.5 (Spanish/French orthographic equivalence):
306
+ - C before E, I, Y → S (ce, ci → se, si)
307
+ - C elsewhere → K
308
+
309
+ Examples:
310
+ MARCIANO → MARSIANO (C before I → S)
311
+ COHEN → KOHEN (C before O → K)
312
+ CAHEN → KAHEN (C before A → K)
313
+
314
+ Returns:
315
+ str: Text with C replaced appropriately
316
+ """
317
+ if not text or 'C' not in text:
318
+ return text
319
+
320
+ result = []
321
+ text_upper = text.upper()
322
+
323
+ for i, char in enumerate(text_upper):
324
+ if char == 'C':
325
+ # Check what follows C
326
+ next_char = text_upper[i + 1] if i + 1 < len(text_upper) else ''
327
+ if next_char in 'EIY':
328
+ # C before E, I, Y → S (Spanish/French soft C)
329
+ result.append('S')
330
+ else:
331
+ # C elsewhere → K
332
+ result.append('K')
333
+ else:
334
+ result.append(char)
335
+
336
+ return ''.join(result)
337
+
338
+ def strip_spanish_plural_s(text):
339
+ """
340
+ Strip final -S from Spanish plural/patronymic suffixes (v3.4.2).
341
+
342
+ Per Beider p. 148-149: Final -S is a Spanish plural or patronymic marker.
343
+ Same family names exist with and without the -S:
344
+ - ATTIA ↔ ATTIAS (285+36 = 321 records in database)
345
+ - NAHMIA ↔ NAHMIAS (41 records)
346
+ - CORCIA ↔ CORCIAS (28 records)
347
+ - SORIA ↔ SORIAS (3 records)
348
+
349
+ Rule: Strip final -S when name ends in -IAS pattern.
350
+ This is a CONSERVATIVE rule targeting only the well-attested Spanish pattern.
351
+
352
+ DO NOT strip:
353
+ - -EZ names (PEREZ, SANCHEZ) — -EZ is a different suffix
354
+ - -AS names where S is part of root (BIBAS, MESSAS, SASPORTAS)
355
+ - -OS names (too risky, many false positives)
356
+
357
+ Returns:
358
+ str: Text with Spanish plural -S stripped if applicable
359
+ """
360
+ if not text or len(text) < 4:
361
+ return text
362
+
363
+ text_upper = text.upper()
364
+
365
+ # Only strip -S from -IAS pattern (Spanish feminine patronymic)
366
+ # This is the safest rule with clear etymological basis
367
+ if text_upper.endswith('IAS'):
368
+ return text_upper[:-1] # ATTIAS → ATTIA
369
+
370
+ return text_upper
371
+
372
+ def y_is_consonant(text, position):
373
+ """
374
+ Determine if Y at given position should be treated as consonant.
375
+
376
+ Y is consonant when:
377
+ - At start of name (YOSEF, YAAKOV)
378
+ - Followed by vowel (YA, YO, YE, YI, YU)
379
+ """
380
+ if position >= len(text):
381
+ return False
382
+
383
+ # Y at start is consonant
384
+ if position == 0:
385
+ return True
386
+
387
+ # Y followed by vowel is consonant
388
+ if position + 1 < len(text) and text[position + 1] in VOWELS:
389
+ return True
390
+
391
+ return False
392
+
393
+ def apply_smart_vowel_rules(text):
394
+ """
395
+ Apply vowel combination rules smartly.
396
+
397
+ KEY FIX: AY→I and EY→I only apply when Y is NOT followed by a vowel.
398
+
399
+ When Y is followed by a vowel, Y is a consonant and should be preserved:
400
+ - HAYON: H-A-Y-O-N → Y followed by O → Y is consonant → keep as HAYON
401
+ - LAYEM: L-A-Y-E-M → Y followed by E → Y is consonant → keep as LAYEM
402
+
403
+ When Y is NOT followed by a vowel, Y is a vowel sound:
404
+ - MARHAYI: ends in AYI → AY is vowel pair → could become MARHI (but careful)
405
+ - BENHALAY: ends in AY → Y is final vowel → AY→I could apply
406
+ """
407
+ if not text:
408
+ return text
409
+
410
+ result = list(text)
411
+ i = 0
412
+
413
+ while i < len(result) - 1:
414
+ pair = result[i] + result[i + 1]
415
+
416
+ # Check vowel pairs
417
+ if pair in ['OU', 'AU', 'EI']:
418
+ # These always get simplified (when not at start)
419
+ if i > 0:
420
+ replacements = {'OU': 'U', 'AU': 'O', 'EI': 'I'}
421
+ if pair in replacements:
422
+ result[i] = replacements[pair]
423
+ result[i + 1] = '' # Mark for removal
424
+ i += 2
425
+ continue
426
+
427
+ # AI - similar logic to AY: check if I is followed by vowel (making I consonant-like)
428
+ # OHAION: A-I-O → I followed by O → I acts like Y → keep as AION → later becomes AYON-like
429
+ if pair == 'AI':
430
+ if i > 0: # Not at start
431
+ next_pos = i + 2
432
+ if next_pos < len(result) and result[next_pos] in VOWELS:
433
+ # I is followed by vowel → I acts like consonant Y → convert AI to AY
434
+ result[i + 1] = 'Y' # Change I to Y
435
+ i += 1
436
+ continue
437
+ else:
438
+ # I is NOT followed by vowel → AI is vowel pair → AI→I
439
+ result[i] = 'I'
440
+ result[i + 1] = '' # Mark for removal
441
+ i += 2
442
+ continue
443
+
444
+ # AY and EY - check if Y is consonant (followed by vowel)
445
+ if pair in ['AY', 'EY']:
446
+ if i > 0: # Not at start
447
+ # Check what follows Y
448
+ next_pos = i + 2
449
+ if next_pos < len(result) and result[next_pos] in VOWELS:
450
+ # Y is followed by vowel → Y is consonant → KEEP
451
+ i += 1
452
+ continue
453
+ else:
454
+ # Y is NOT followed by vowel → Y is vowel → AY/EY→I
455
+ result[i] = 'I'
456
+ result[i + 1] = '' # Mark for removal
457
+ i += 2
458
+ continue
459
+
460
+ # OI - only apply when not at start (start handled earlier)
461
+ if pair == 'OI' and i > 0:
462
+ result[i] = 'I'
463
+ result[i + 1] = '' # Mark for removal
464
+ i += 2
465
+ continue
466
+
467
+ i += 1
468
+
469
+ # Remove empty strings
470
+ return ''.join(c for c in result if c)
471
+
472
+ def apply_phonetic_rules(text):
473
+ """
474
+ STAGE 3: Apply phonetic transformation rules.
475
+
476
+ v3.4.2 CHANGES:
477
+ - Added: Strip final -S from -IAS suffix (Beider p. 148-149: Spanish plural)
478
+ - Merges: ATTIAS↔ATTIA, NAHMIAS↔NAHMIA, CORCIAS↔CORCIA
479
+
480
+ v3.4.1 CHANGES:
481
+ - Added W→B (Beider Table 2.7: waw/bet interchange)
482
+ - C is now context-dependent: C→S before i,e,y; C→K otherwise (Beider Table 2.5)
483
+ - REMOVED Z→S: Caused 724 false merges (HAZAN≠HASSAN, etc.) affecting 14K records
484
+
485
+ KEY FIX v3.0: AY/EY rules now check if Y is consonant (followed by vowel)
486
+ - HAYON: A-Y-O-N → Y is consonant (followed by O) → NOT replaced
487
+ - MARHAYI: A-Y-I → AY at end-ish, Y is vowel → replaced with I
488
+ """
489
+ if not text:
490
+ return ''
491
+
492
+ result = text.upper()
493
+
494
+ # Digraphs and trigraphs (order matters - longer first!)
495
+ digraph_rules = [
496
+ ('TCH', 'SH'),
497
+ ('SCH', 'SH'),
498
+ ('CH', 'SH'), # French CH = SH
499
+ ('PH', 'F'),
500
+ ('TZ', 'TS'),
501
+ ('DJ', 'G'), # French style → G
502
+ ('DG', 'G'),
503
+ ('QU', 'K'),
504
+ ('CK', 'K'),
505
+ ('GN', 'NY'), # French sound
506
+ ('GH', 'G'), # v3.6.1: Amghar=Amgar, Sebagh=Sebag (13 merges, 0 splits)
507
+ ('TH', 'T'), # v3.6.1: Nathan=Natan, Wakrath=Wakrat (12 merges, 0 splits)
508
+ ]
509
+
510
+ for old, new in digraph_rules:
511
+ result = result.replace(old, new)
512
+
513
+ # Context-dependent C rule (v3.4):
514
+ # C→S before i,e,y (Spanish/French: ce, ci → se, si)
515
+ # C→K otherwise
516
+ # This must be applied BEFORE other char_rules that might conflict
517
+ result = apply_context_c_rule(result)
518
+
519
+ # Single character substitutions
520
+ # NOTE: C is handled above by apply_context_c_rule(), not here
521
+ # NOTE: Z→S was removed in v3.4.1 - too aggressive, creates 724 false merges
522
+ # (HAZAN≠HASSAN, ZAZON≠SOUSSAN, etc.) affecting 14K records
523
+ char_rules = [
524
+ ('J', 'G'), # J → G
525
+ ('Q', 'K'),
526
+ ('X', 'KS'),
527
+ ('V', 'B'), # V ≈ B in Moroccan (Beider: bet/vet)
528
+ ('W', 'B'), # W ≈ B (Beider Table 2.7: waw/bet interchange)
529
+ ]
530
+
531
+ for old, new in char_rules:
532
+ result = result.replace(old, new)
533
+
534
+ # Strip Spanish plural -S from -IAS suffix (v3.4.2)
535
+ # Per Beider p. 148-149: ATTIAS↔ATTIA, NAHMIAS↔NAHMIA are same family
536
+ result = strip_spanish_plural_s(result)
537
+
538
+ # Vowel combinations - SMART APPLICATION
539
+ # Key insight: Only apply AY/EY→I when Y is acting as a vowel (not followed by vowel)
540
+ result = apply_smart_vowel_rules(result)
541
+
542
+ return result
543
+
544
+ def extract_consonant_skeleton(text):
545
+ """
546
+ STAGE 4: Extract consonant skeleton with preserved structure.
547
+
548
+ KEY FEATURES:
549
+ - Preserves consonants separated by vowels (AKNINE → AKNN, two Ns)
550
+ - Only merges CONSECUTIVE duplicate consonants (MANN → MN)
551
+ - Handles Y specially based on context
552
+ - Keeps first vowel if at start (AKNINE → AKNN starts with A)
553
+ """
554
+ if not text:
555
+ return ''
556
+
557
+ text = text.upper()
558
+ text = ''.join(c for c in text if c.isalpha())
559
+
560
+ if not text:
561
+ return ''
562
+
563
+ # Remove trailing vowels
564
+ while text and text[-1] in VOWELS:
565
+ text = text[:-1]
566
+
567
+ if not text:
568
+ return ''
569
+
570
+ # Build skeleton - track vowels between consonants
571
+ result = []
572
+ prev_consonant = ''
573
+ vowel_since_last_consonant = False
574
+
575
+ for i, char in enumerate(text):
576
+ # Handle Y specially
577
+ if char == 'Y':
578
+ if y_is_consonant(text, i):
579
+ # Y is consonant - keep if different or vowel between
580
+ if vowel_since_last_consonant or char != prev_consonant:
581
+ result.append(char)
582
+ prev_consonant = char
583
+ vowel_since_last_consonant = False
584
+ else:
585
+ # Y is vowel
586
+ vowel_since_last_consonant = True
587
+ continue
588
+
589
+ # Handle vowels
590
+ if char in VOWELS:
591
+ # Keep first vowel if at start
592
+ if i == 0:
593
+ result.append(char)
594
+ vowel_since_last_consonant = True
595
+ continue
596
+
597
+ # Handle consonants
598
+ if char == prev_consonant and not vowel_since_last_consonant:
599
+ # Same consonant, NO vowel between → skip (MM → M)
600
+ continue
601
+
602
+ # Different consonant, OR same but vowel between (N-vowel-N → NN)
603
+ result.append(char)
604
+ prev_consonant = char
605
+ vowel_since_last_consonant = False
606
+
607
+ return ''.join(result)
608
+
609
+ def normalize_to_phonetic(name):
610
+ """
611
+ Main normalization function - COMPLETE REWRITE v3.0
612
+
613
+ Converts any surname variant to its phonetic root through 4 stages:
614
+
615
+ STAGE 1: Start patterns (OI→A) - BEFORE prefix stripping!
616
+ STAGE 2: Prefix stripping (OU, W, BEN, etc.)
617
+ STAGE 3: Phonetic rules (digraphs, vowel combinations)
618
+ STAGE 4: Consonant skeleton extraction
619
+
620
+ Args:
621
+ name: Surname in Latin script
622
+
623
+ Returns:
624
+ str: Phonetic root (consonant skeleton)
625
+
626
+ Examples:
627
+ normalize_to_phonetic('OUAKNINE') → 'AKNN'
628
+ normalize_to_phonetic('OIKNINE') → 'AKNN' # Fixed in v3.0!
629
+ normalize_to_phonetic('AKOUN') → 'AKN' # Different!
630
+ normalize_to_phonetic('OHAYON') → 'HYN' # Fixed in v3.0!
631
+ normalize_to_phonetic('HAYON') → 'HYN' # Fixed in v3.0!
632
+ normalize_to_phonetic('OHANA') → 'HN' # Different from OHAYON!
633
+ normalize_to_phonetic('HANOUNA') → 'HNN'
634
+ normalize_to_phonetic('BENARROCH') → 'ARSH' (rules-only, v3.7.0)
635
+ """
636
+ if not name:
637
+ return ''
638
+
639
+ name = name.strip().upper()
640
+ name = normalize_accents(name) # Remove accented characters (é→E, etc.)
641
+
642
+ # A marker is not a name, and must never become a root (owner, 2026-07-26).
643
+ if name in NON_NAME_MARKERS:
644
+ return ''
645
+
646
+ # E5: reject junk — any digit present, or fewer than 2 letters.
647
+ if any(c.isdigit() for c in name) or sum(c.isalpha() for c in name) < 2:
648
+ return ''
649
+
650
+ # v3.4.6: Reject invalid surnames (incomplete/prefix-only entries)
651
+ # These are data quality issues, not valid surnames
652
+ if name in INVALID_SURNAMES:
653
+ return ''
654
+
655
+ # Check special mappings BEFORE normal processing
656
+ special = check_special_mapping(name)
657
+ if special:
658
+ return special
659
+
660
+ # STAGE 1: Normalize start patterns BEFORE prefix stripping
661
+ # This ensures OI→A happens before O is stripped
662
+ stage1 = normalize_start_patterns(name)
663
+
664
+ # Check special mapping after stage 1
665
+ special = check_special_mapping(stage1)
666
+ if special:
667
+ return special
668
+
669
+ # STAGE 2: Strip Latin prefix
670
+ stage2 = strip_latin_prefix(stage1)
671
+
672
+ # Check special mapping on stripped version
673
+ special = check_special_mapping(stage2)
674
+ if special:
675
+ return special
676
+
677
+ # STAGE 3: Apply phonetic rules (with smart vowel handling)
678
+ stage3 = apply_phonetic_rules(stage2)
679
+
680
+ # STAGE 4: Extract consonant skeleton
681
+ phonetic_root = extract_consonant_skeleton(stage3)
682
+
683
+ # (STAGE 5 post-normalization corrections removed in v3.7.1: pure rules only.)
684
+
685
+ # STAGE 6 (v3.6.0): root-initial vowel classing, A/E -> A, O/U -> O,
686
+ # I/Y -> I. Promoted from adjudications after a corpus-wide dry-run
687
+ # (42 merges, 0 splits) and per-merge review on 2026-07-11: Alfassi =
688
+ # Elfassy, Assayag = Esayag, Oziel = Uziel, Ifrah = Yfrah, etc.
689
+ # Special-mapping outputs bypass this stage (they are authoritative).
690
+ # Evidence: docs/mjpn/mapping_analysis_2026_07_11.txt
691
+ if phonetic_root and phonetic_root[0] in _INITIAL_VOWEL_CLASS:
692
+ phonetic_root = _INITIAL_VOWEL_CLASS[phonetic_root[0]] + phonetic_root[1:]
693
+
694
+ return phonetic_root
695
+
696
+ def vowel_signature(text):
697
+ """Vowel-class signature of `text`: map A/E->A, O/U->O, I/Y->I over the
698
+ form with trailing vowels trimmed, then collapse adjacent duplicates."""
699
+ if not text:
700
+ return ''
701
+ trimmed = text.rstrip('AEIOUY')
702
+ sig = []
703
+ for ch in trimmed:
704
+ cls = _VOWEL_CLASS.get(ch)
705
+ if cls and (not sig or sig[-1] != cls):
706
+ sig.append(cls)
707
+ return ''.join(sig)
708
+
709
+ def _post_stage2_form(name):
710
+ """The name after Stage 1 (start patterns) and Stage 2 (prefix strip),
711
+ which is where the distinguishing vowels still live (BEN not yet gone)."""
712
+ if not name:
713
+ return ''
714
+ cleaned = normalize_accents(name.strip().upper())
715
+ if not cleaned or cleaned in INVALID_SURNAMES:
716
+ return cleaned
717
+ return strip_latin_prefix(normalize_start_patterns(cleaned))
718
+
719
+ def normalize_to_phonetic_v4(name):
720
+ """Return (root, subroot). root is exactly normalize_to_phonetic(name);
721
+ subroot = root + '~' + vowel_signature(post-Stage-2 form). Empty root -> ('','').
722
+
723
+ Subroot definition (I12, the single authority): vowel classes A/E->A, O/U->O,
724
+ I/Y->I, adjacent duplicates collapsed, computed on the post-prefix-strip form
725
+ with trailing vowels INCLUDING Y trimmed. An EMPTY signature renders as
726
+ `root~-` (never the bare root), so every attested subroot carries a `~`. The
727
+ special-mapping path computes the signature from the input form (via
728
+ _post_stage2_form) exactly like every other path."""
729
+ root = normalize_to_phonetic(name)
730
+ if not root:
731
+ return ('', '')
732
+ sig = vowel_signature(_post_stage2_form(name))
733
+ return (root, f"{root}~{sig}" if sig else f"{root}~-")
734
+
735
+ def _is_non_name(nom):
736
+ """Whole-field marker test, before any tokenizing: 'Sans inscription'
737
+ must not be split into two tokens and given a root."""
738
+ return normalize_accents((nom or '').strip().upper()) in NON_NAME_MARKERS
739
+
740
+ def _compound_tokens(nom):
741
+ """Preprocess a Nom into the final surname tokens (parens dropped,
742
+ separators normalised, particle+name rejoined). Shared by
743
+ compute_compound_indices and compute_compound_indices_v4 so they cannot drift."""
744
+ if not nom or not nom.strip():
745
+ return []
746
+ s = re.sub(r'\s*\([^)]*\)\s*', ' ', nom.strip()) # drop parenthesised comments
747
+ s = re.sub(r'\s*-\s*', ' ', s) # normalise separators
748
+ s = re.sub(r'\s+', ' ', s).strip().upper()
749
+ if not s:
750
+ return []
751
+ tokens = [t for t in s.split(' ') if t]
752
+ merged, i = [], 0
753
+ while i < len(tokens):
754
+ t = tokens[i]
755
+ if t in _COMPOUND_PARTICLES and i + 1 < len(tokens) and len(tokens[i + 1]) >= 3:
756
+ merged.append(t + tokens[i + 1])
757
+ i += 2
758
+ else:
759
+ merged.append(t)
760
+ i += 1
761
+ return merged
762
+
763
+ def compute_compound_indices(nom):
764
+ """
765
+ Compute (primary_root, secondary_root) for a Nom value.
766
+
767
+ For single-word Nom: returns (root, None).
768
+ For multi-word Nom representing a real compound (e.g., "levy bencheton",
769
+ "bittan serfaty"): returns the MJPN root of each surname part in order.
770
+ Tokens that look like noise (FAMILLE, articles, parentheticals) or are
771
+ not normalizable are skipped.
772
+
773
+ Returns:
774
+ tuple (str|None, str|None) — both may be None if nothing normalizes
775
+ (e.g., "eleves yeshiva d'agadir").
776
+
777
+ Examples:
778
+ >>> compute_compound_indices('cohen')
779
+ ('KHN', None)
780
+ >>> compute_compound_indices('levy bencheton')
781
+ ('LB', 'SHTN')
782
+ >>> compute_compound_indices('bendavid ohayon')
783
+ ('DBD', 'HYN')
784
+ >>> compute_compound_indices('leb lugasi')
785
+ ('LB', 'LGS')
786
+ """
787
+ if _is_non_name(nom):
788
+ return (None, None)
789
+ tokens = _compound_tokens(nom)
790
+ if not tokens:
791
+ return (None, None)
792
+
793
+ # Single token → just normalize
794
+ if len(tokens) == 1:
795
+ root = normalize_to_phonetic(tokens[0])
796
+ return (root or None, None)
797
+
798
+ # Multi token — normalize each, drop empties / noise duplicates
799
+ roots = []
800
+ for tok in tokens:
801
+ if tok in _COMPOUND_NOISE_WORDS:
802
+ continue
803
+ r = normalize_to_phonetic(tok)
804
+ if r and r not in roots:
805
+ roots.append(r)
806
+ if len(roots) >= 2:
807
+ break
808
+
809
+ if not roots:
810
+ return (None, None)
811
+ if len(roots) == 1:
812
+ return (roots[0], None)
813
+ return (roots[0], roots[1])
814
+
815
+ def compute_compound_indices_v4(nom):
816
+ """(root, root2, subroot, subroot2) — v4 dual index plus per-root subroots.
817
+
818
+ Same token selection as compute_compound_indices; each selected token also
819
+ yields its subroot via normalize_to_phonetic_v4.
820
+
821
+ Examples:
822
+ >>> compute_compound_indices_v4('cohen')
823
+ ('KHN', None, 'KHN~OA', None)
824
+ >>> compute_compound_indices_v4('levy bencheton')[:2]
825
+ ('LB', 'SHTN')
826
+ """
827
+ if _is_non_name(nom):
828
+ return (None, None, None, None)
829
+ tokens = _compound_tokens(nom)
830
+ if not tokens:
831
+ return (None, None, None, None)
832
+ if len(tokens) == 1:
833
+ root, subroot = normalize_to_phonetic_v4(tokens[0])
834
+ return (root or None, None, subroot or None, None)
835
+
836
+ pairs, seen = [], []
837
+ for tok in tokens:
838
+ if tok in _COMPOUND_NOISE_WORDS:
839
+ continue
840
+ root, subroot = normalize_to_phonetic_v4(tok)
841
+ if root and root not in seen:
842
+ seen.append(root)
843
+ pairs.append((root, subroot))
844
+ if len(pairs) >= 2:
845
+ break
846
+ if not pairs:
847
+ return (None, None, None, None)
848
+ r1, s1 = pairs[0]
849
+ r2, s2 = pairs[1] if len(pairs) > 1 else (None, None)
850
+ return (r1, r2, s1, s2)
851
+
852
+
853
+ # =============================================================================
854
+ # XLSX RUNNER + STATS (tool layer — the engine above is generated verbatim)
855
+ # =============================================================================
856
+
857
+ def process_name(name):
858
+ """Run one raw Name through the full pipeline.
859
+
860
+ Returns dict with root/root2/subroot/subroot2, whether a special mapping
861
+ fired, and a status: ok | compound | invalid | empty.
862
+ """
863
+ if name is None or not str(name).strip():
864
+ return {"name": "" if name is None else str(name),
865
+ "root": None, "root2": None, "subroot": None, "subroot2": None,
866
+ "special": False, "status": "empty"}
867
+ raw = str(name).strip()
868
+ root, root2, subroot, subroot2 = compute_compound_indices_v4(raw)
869
+ special = any(check_special_mapping(normalize_accents(t))
870
+ for t in _compound_tokens(raw))
871
+ if root is None:
872
+ status = "invalid"
873
+ elif root2:
874
+ status = "compound"
875
+ else:
876
+ status = "ok"
877
+ return {"name": raw, "root": root, "root2": root2,
878
+ "subroot": subroot, "subroot2": subroot2,
879
+ "special": special, "status": status}
880
+
881
+
882
+ def compute_stats(rows):
883
+ """Aggregate stats over process_name() rows."""
884
+ total = len(rows)
885
+ by_status = Counter(r["status"] for r in rows)
886
+ special_hits = sum(1 for r in rows if r["special"])
887
+ families = defaultdict(list) # root -> [names]
888
+ for r in rows:
889
+ for root in (r["root"], r["root2"]):
890
+ if root:
891
+ families[root].append(r["name"])
892
+ sizes = Counter(len(v) for v in families.values())
893
+ dist = {
894
+ "singleton (1 name)": sum(c for s, c in sizes.items() if s == 1),
895
+ "small (2-5)": sum(c for s, c in sizes.items() if 2 <= s <= 5),
896
+ "medium (6-20)": sum(c for s, c in sizes.items() if 6 <= s <= 20),
897
+ "large (>20)": sum(c for s, c in sizes.items() if s > 20),
898
+ }
899
+ top = sorted(families.items(), key=lambda kv: -len(kv[1]))[:20]
900
+ subroots_per_root = defaultdict(set)
901
+ for r in rows:
902
+ if r["root"] and r["subroot"]:
903
+ subroots_per_root[r["root"]].add(r["subroot"])
904
+ split_roots = {k: sorted(v) for k, v in subroots_per_root.items() if len(v) > 1}
905
+ return {
906
+ "total names": total,
907
+ "normalized": by_status.get("ok", 0) + by_status.get("compound", 0),
908
+ "compound names": by_status.get("compound", 0),
909
+ "invalid / not normalizable": by_status.get("invalid", 0),
910
+ "empty cells": by_status.get("empty", 0),
911
+ "special-mapping hits": special_hits,
912
+ "unique roots": len(families),
913
+ "family size distribution": dist,
914
+ "top families": [(root, len(names), ", ".join(sorted(set(names))[:5]))
915
+ for root, names in top],
916
+ "roots split by subroot": {k: v for k, v in sorted(split_roots.items())},
917
+ }
918
+
919
+
920
+ def read_names_xlsx(path):
921
+ """Read the 'Names' column (case-insensitive header) from an xlsx."""
922
+ from openpyxl import load_workbook
923
+ wb = load_workbook(path, read_only=True, data_only=True)
924
+ ws = wb.active
925
+ rows = ws.iter_rows(values_only=True)
926
+ header = next(rows, None)
927
+ if header is None:
928
+ raise SystemExit(f"{path}: empty sheet")
929
+ idx = None
930
+ for i, h in enumerate(header):
931
+ if h is not None and str(h).strip().lower() == "names":
932
+ idx = i
933
+ break
934
+ if idx is None:
935
+ raise SystemExit(f'{path}: no "Names" column (headers: {list(header)})')
936
+ names = [row[idx] for row in rows if row is not None and len(row) > idx]
937
+ wb.close()
938
+ return names
939
+
940
+
941
+ def write_output_xlsx(path, rows, stats):
942
+ from openpyxl import Workbook
943
+ wb = Workbook()
944
+ ws = wb.active
945
+ ws.title = "MJPN"
946
+ ws.append(["name", "root", "root2", "subroot", "subroot2",
947
+ "special_mapping", "status"])
948
+ for r in rows:
949
+ ws.append([r["name"], r["root"], r["root2"], r["subroot"],
950
+ r["subroot2"], "yes" if r["special"] else "", r["status"]])
951
+ st = wb.create_sheet("Stats")
952
+ for key in ("total names", "normalized", "compound names",
953
+ "invalid / not normalizable", "empty cells",
954
+ "special-mapping hits", "unique roots"):
955
+ st.append([key, stats[key]])
956
+ st.append([])
957
+ st.append(["family size distribution"])
958
+ for k, v in stats["family size distribution"].items():
959
+ st.append([k, v])
960
+ st.append([])
961
+ st.append(["top families", "count", "example names"])
962
+ for root, count, examples in stats["top families"]:
963
+ st.append([root, count, examples])
964
+ if stats["roots split by subroot"]:
965
+ st.append([])
966
+ st.append(["roots split by subroot (v4)", "subroots"])
967
+ for root, subs in stats["roots split by subroot"].items():
968
+ st.append([root, ", ".join(subs)])
969
+ wb.save(path)
970
+
971
+
972
+ def print_stats(stats):
973
+ for key in ("total names", "normalized", "compound names",
974
+ "invalid / not normalizable", "empty cells",
975
+ "special-mapping hits", "unique roots"):
976
+ print(f" {key}: {stats[key]}")
977
+ print(" family size distribution:")
978
+ for k, v in stats["family size distribution"].items():
979
+ print(f" {k}: {v}")
980
+ print(" top families:")
981
+ for root, count, examples in stats["top families"][:10]:
982
+ print(f" {root:<8} {count:>4} ({examples})")
983
+
984
+
985
+ def find_default_xlsx():
986
+ # Current directory first (the natural place when installed as a
987
+ # package), then the script's own folder (one-file standalone usage).
988
+ for folder in (Path.cwd(), Path(__file__).resolve().parent):
989
+ candidates = [p for p in sorted(folder.glob("*.xlsx"))
990
+ if not p.stem.endswith("_mjpn") and not p.name.startswith("~$")]
991
+ if candidates:
992
+ return candidates[0]
993
+ raise SystemExit("no .xlsx found in the current directory or next to "
994
+ f"{Path(__file__).name}; pass a path explicitly")
995
+
996
+
997
+ def main():
998
+ ap = argparse.ArgumentParser(description="MJPN root computation for an "
999
+ 'xlsx list of names (column "Names")')
1000
+ ap.add_argument("xlsx", nargs="?", help="input .xlsx (default: first one "
1001
+ "in this script's folder)")
1002
+ ap.add_argument("--name", nargs="+", help="ad-hoc names instead of a file")
1003
+ ap.add_argument("--selftest", action="store_true")
1004
+ args = ap.parse_args()
1005
+
1006
+ if args.selftest:
1007
+ checks = [("BENHAIM", "HM"), ("MARCIANO", "MRSN"), ("OUAKNINE", "AKNN"),
1008
+ ("SASSON", "SSN"), ("COHEN", "KHN"), ("BENARROCH", "ARSH")]
1009
+ ok = True
1010
+ for name, expected in checks:
1011
+ got = normalize_to_phonetic(name)
1012
+ print(f" {'OK ' if got == expected else 'XX '}{name} -> {got}")
1013
+ ok = ok and got == expected
1014
+ sys.exit(0 if ok else 1)
1015
+
1016
+ if args.name:
1017
+ for n in args.name:
1018
+ r = process_name(n)
1019
+ print(f"{n} -> root={r['root']} root2={r['root2']} "
1020
+ f"subroot={r['subroot']} subroot2={r['subroot2']}")
1021
+ return
1022
+
1023
+ src = Path(args.xlsx) if args.xlsx else find_default_xlsx()
1024
+ print(f"input: {src}")
1025
+ names = read_names_xlsx(src)
1026
+ rows = [process_name(n) for n in names]
1027
+ stats = compute_stats(rows)
1028
+ out = src.with_name(src.stem + "_mjpn.xlsx")
1029
+ write_output_xlsx(out, rows, stats)
1030
+ print(f"output: {out}")
1031
+ print_stats(stats)
1032
+
1033
+
1034
+ if __name__ == "__main__":
1035
+ main()
@@ -0,0 +1,120 @@
1
+ Metadata-Version: 2.4
2
+ Name: mjpn
3
+ Version: 6.0.0
4
+ Summary: Moroccan Jewish Phonetic Names (MJPN): phonetic root computation for Moroccan Jewish surnames, the engine behind yahasra.org
5
+ Author-email: Dr Yohanan Ouaknine <contact@yahasra.org>
6
+ License: MIT
7
+ Project-URL: Homepage, https://yahasra.org
8
+ Project-URL: Documentation, https://yahasra.org/about
9
+ Keywords: onomastics,phonetic,surnames,genealogy,jewish,morocco,sephardic
10
+ Classifier: Development Status :: 5 - Production/Stable
11
+ Classifier: Intended Audience :: Science/Research
12
+ Classifier: License :: OSI Approved :: MIT License
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Topic :: Sociology :: Genealogy
15
+ Classifier: Topic :: Text Processing :: Linguistic
16
+ Requires-Python: >=3.8
17
+ Description-Content-Type: text/markdown
18
+ License-File: LICENSE
19
+ Requires-Dist: openpyxl>=3.0
20
+ Dynamic: license-file
21
+
22
+ # MJPN — Moroccan Jewish Phonetic Names
23
+
24
+ Phonetic root computation for Moroccan Jewish surnames — the exact engine
25
+ behind [yahasra.org](https://yahasra.org)'s search across 32,000+ burial
26
+ records from Jewish cemeteries in Morocco.
27
+
28
+ Moroccan Jewish surnames were written down by French, Spanish and Hebrew
29
+ scribes, each with their own orthography: the same family appears as
30
+ *Ouaknine, Vaknin, Aknin, Boaknin, Oiknine…* MJPN reduces every variant to a
31
+ stable phonetic root (`AKNN`), so records that belong together can be found
32
+ together.
33
+
34
+ ## Install
35
+
36
+ ```
37
+ pip install mjpn
38
+ ```
39
+
40
+ ## Command line
41
+
42
+ ```
43
+ mjpn myfile.xlsx # xlsx with a "Names" column (case-insensitive)
44
+ mjpn # first .xlsx found in the current script folder
45
+ mjpn --name COHEN OUAKNINE "LEVY BENCHETON"
46
+ mjpn --selftest
47
+ ```
48
+
49
+ The xlsx run writes `<input>_mjpn.xlsx` next to the input, with:
50
+
51
+ - sheet **MJPN** — `name, root, root2, subroot, subroot2, special_mapping, status`
52
+ (`status` is `ok` / `compound` / `invalid` / `empty`)
53
+ - sheet **Stats** — totals, family-size distribution, top families,
54
+ roots split by v4 subroot
55
+
56
+ and prints the stats summary to stdout.
57
+
58
+ ## Python API
59
+
60
+ ```python
61
+ from mjpn import normalize_to_phonetic, compute_compound_indices_v4
62
+
63
+ normalize_to_phonetic("OUAKNINE") # 'AKNN'
64
+ normalize_to_phonetic("BENARROCH") # 'HRSH'
65
+ compute_compound_indices_v4("levy bencheton")
66
+ # ('LB', 'SHTN', 'LB~A', 'SHTN~AO') — compound dual index + v4 subroots
67
+ ```
68
+
69
+ ## How it works
70
+
71
+ Four normalization stages (engine v3.5.2, v4 subroot chain):
72
+
73
+ 1. **Start patterns** — Berber orthographic variants (OI→A) before anything else.
74
+ 2. **Prefix stripping** — patronymic prefixes (BEN, IBN, OU, HA, EL…) with
75
+ structural-name exception lists so *Hamou* or *Elfassy* keep their prefix.
76
+ 3. **Phonetic rules** — digraphs (CH→SH…), context-dependent C, V/W→B,
77
+ Spanish plural -IAS→-IA, smart vowel-pair handling (based on Beider's
78
+ *A Dictionary of Jewish Surnames from Maghreb*, 2017).
79
+ 4. **Consonant skeleton** — the root; a vowel-class signature on top yields
80
+ the v4 *subroot* that keeps genuinely different families apart.
81
+
82
+ Plus 149 exact and 2 regex special mappings for families whose variants defy
83
+ the rules (Abuhatzeira, Azoulay, Zrihen…), curated from the yahasra.org
84
+ adjudication process.
85
+
86
+ `mjpn/mjpn_tool.py` is **generated** from the production engine and verified
87
+ against it: identical output on all 3,933 distinct surnames of the yahasra.org
88
+ burials database (0 mismatches).
89
+
90
+ ## Citation
91
+
92
+ If you use MJPN in academic work, please cite:
93
+
94
+ > Ouaknine, Y. (2026). MJPN: Moroccan Jewish Phonetic Names — the phonetic
95
+ > indexing engine of yahasra.org, the Moroccan Jewish cemeteries database.
96
+ > https://yahasra.org
97
+
98
+ ## License
99
+
100
+ MIT © Dr Yohanan Ouaknine
101
+
102
+ ## Provenance and AI disclosure
103
+
104
+ The engine (`app/phonetic_normalizer.py` in the Yahasra codebase) encodes
105
+ orthographic rules authored and adjudicated by Dr Yohanan Ouaknine, tuned
106
+ against Beider (2017) with page references recorded per rule. This package is
107
+ generated mechanically from that engine by `scripts/build_mjpn_tool.py` — no
108
+ hand-maintained copy exists — and each release is parity-checked against the
109
+ full live corpus (5,314 distinct surnames, zero mismatches required).
110
+
111
+ Portions of the engine's implementation code and this package's build
112
+ tooling were produced with assistive AI (Anthropic Claude: Sonnet 4.x and
113
+ Opus 4.x/5, 2025-06 through 2026-08) under human direction and review; all
114
+ linguistic rules, special mappings and adjudications are human decisions.
115
+ This disclosure follows the AI policy of the Journal of Cultural Analytics.
116
+
117
+ ## Version
118
+
119
+ Package 6.0.0 wraps engine 6.0.0 (v4 chain: subroots + compound dual
120
+ indexing), generated 2026-11-16 with 98 exact + 1 regex special mappings.
@@ -0,0 +1,8 @@
1
+ mjpn/__init__.py,sha256=IDHRQ3Z-iU2nSyO9aSZrDiAoceZ9AmVRiANVVZnzAx0,965
2
+ mjpn/mjpn_tool.py,sha256=gNpBSke6psHURz8aU3XxJ1HapvM1lc_WAXWixivJ6Pg,42559
3
+ mjpn-6.0.0.dist-info/licenses/LICENSE,sha256=q1mDwNHs4ZwsDeHe39N2-lC0BWyTLi_MK5H5wozSzLE,1076
4
+ mjpn-6.0.0.dist-info/METADATA,sha256=R4UstybdoCr9bHW1pgNtBVxWgr9f_wZXEKbBboUykRs,4684
5
+ mjpn-6.0.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
6
+ mjpn-6.0.0.dist-info/entry_points.txt,sha256=H01WCt3GBTU7R4Dfx09pElqbu-KW_o26FIVeZ9C_Mzg,45
7
+ mjpn-6.0.0.dist-info/top_level.txt,sha256=6q558vgYhSThTq81kf9HGDPuQWYxpXPGBn_X7HSADkY,5
8
+ mjpn-6.0.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (82.0.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ mjpn = mjpn.mjpn_tool:main
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Dr Yohanan Ouaknine
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1 @@
1
+ mjpn