mjpn 6.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mjpn/__init__.py +40 -0
- mjpn/mjpn_tool.py +1035 -0
- mjpn-6.0.0.dist-info/METADATA +120 -0
- mjpn-6.0.0.dist-info/RECORD +8 -0
- mjpn-6.0.0.dist-info/WHEEL +5 -0
- mjpn-6.0.0.dist-info/entry_points.txt +2 -0
- mjpn-6.0.0.dist-info/licenses/LICENSE +21 -0
- mjpn-6.0.0.dist-info/top_level.txt +1 -0
mjpn/__init__.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""MJPN — Moroccan Jewish Phonetic Names.
|
|
2
|
+
|
|
3
|
+
Computes the phonetic root of Moroccan Jewish surnames, exactly the
|
|
4
|
+
pipeline behind yahasra.org's burials.phonetic_index (normalize_to_phonetic
|
|
5
|
+
v3 stages -> v4 subroots -> compound dual indexing).
|
|
6
|
+
|
|
7
|
+
Typical use:
|
|
8
|
+
|
|
9
|
+
>>> from mjpn import normalize_to_phonetic
|
|
10
|
+
>>> normalize_to_phonetic('OUAKNINE')
|
|
11
|
+
'AKNN'
|
|
12
|
+
|
|
13
|
+
Or from the command line:
|
|
14
|
+
|
|
15
|
+
mjpn myfile.xlsx # xlsx with a "Names" column
|
|
16
|
+
mjpn --name COHEN OUAKNINE
|
|
17
|
+
"""
|
|
18
|
+
from .mjpn_tool import (
|
|
19
|
+
__version__ as engine_version,
|
|
20
|
+
normalize_to_phonetic,
|
|
21
|
+
normalize_to_phonetic_v4,
|
|
22
|
+
compute_compound_indices,
|
|
23
|
+
compute_compound_indices_v4,
|
|
24
|
+
check_special_mapping,
|
|
25
|
+
process_name,
|
|
26
|
+
main,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
__version__ = "6.0.0"
|
|
30
|
+
|
|
31
|
+
__all__ = [
|
|
32
|
+
"engine_version",
|
|
33
|
+
"normalize_to_phonetic",
|
|
34
|
+
"normalize_to_phonetic_v4",
|
|
35
|
+
"compute_compound_indices",
|
|
36
|
+
"compute_compound_indices_v4",
|
|
37
|
+
"check_special_mapping",
|
|
38
|
+
"process_name",
|
|
39
|
+
"main",
|
|
40
|
+
]
|
mjpn/mjpn_tool.py
ADDED
|
@@ -0,0 +1,1035 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
MJPN xlsx tool — GENERATED, DO NOT EDIT BY HAND.
|
|
4
|
+
|
|
5
|
+
Regenerate with: python3 scripts/build_mjpn_tool.py
|
|
6
|
+
Generated 2026-11-16 from app/phonetic_normalizer.py (engine 6.0.0, v4 chain)
|
|
7
|
+
with 98 exact + 1 regex special mappings from the DB.
|
|
8
|
+
|
|
9
|
+
Moroccan Jewish Phonetic Names (MJPN): computes the phonetic root of
|
|
10
|
+
Moroccan Jewish surnames, exactly the pipeline behind yahasra.org's
|
|
11
|
+
burials.phonetic_index (normalize_to_phonetic v3 stages -> v4 subroots ->
|
|
12
|
+
compound dual indexing).
|
|
13
|
+
|
|
14
|
+
Usage:
|
|
15
|
+
python3 mjpn_tool.py # first .xlsx in cwd / next to this file
|
|
16
|
+
python3 mjpn_tool.py myfile.xlsx # explicit file
|
|
17
|
+
python3 mjpn_tool.py --name COHEN OUAKNINE # ad-hoc names, no xlsx
|
|
18
|
+
|
|
19
|
+
Input: an .xlsx with a column named "Names" (case-insensitive).
|
|
20
|
+
Output: <input>_mjpn.xlsx — sheet "MJPN" (name, root, root2, subroot,
|
|
21
|
+
subroot2, special_mapping, status) + sheet "Stats"; stats are also
|
|
22
|
+
printed to stdout.
|
|
23
|
+
|
|
24
|
+
Only non-stdlib dependency: openpyxl.
|
|
25
|
+
"""
|
|
26
|
+
import argparse
|
|
27
|
+
import logging
|
|
28
|
+
import re
|
|
29
|
+
import sys
|
|
30
|
+
from collections import Counter, defaultdict
|
|
31
|
+
from pathlib import Path
|
|
32
|
+
|
|
33
|
+
__version__ = "6.0.0+v4"
|
|
34
|
+
|
|
35
|
+
logger = logging.getLogger("mjpn_tool")
|
|
36
|
+
|
|
37
|
+
VOWELS = 'AEIOU'
|
|
38
|
+
ACCENT_MAP = {'À': 'A', 'Á': 'A', 'Â': 'A', 'Ã': 'A', 'Ä': 'A', 'Å': 'A', 'È': 'E', 'É': 'E', 'Ê': 'E', 'Ë': 'E', 'Ì': 'I', 'Í': 'I', 'Î': 'I', 'Ï': 'I', 'Ò': 'O', 'Ó': 'O', 'Ô': 'O', 'Õ': 'O', 'Ö': 'O', 'Ù': 'U', 'Ú': 'U', 'Û': 'U', 'Ü': 'U', 'Ý': 'Y', 'Ÿ': 'Y', 'Ñ': 'N', 'Ç': 'S'}
|
|
39
|
+
LATIN_PREFIXES = ['ABEN', 'BEN', 'IBN', 'ABN', 'BAR', 'OU', 'BN', 'W', 'DE', 'DI']
|
|
40
|
+
HA_STRUCTURAL_NAMES = {'HAZAN', 'HAYON', 'AIM', 'HALIOUA', 'ASSAN', 'DAD', 'HAMU', 'HAZZAN', 'AMU', 'HASSANI', 'AZAN', 'HAYM', 'HAYOUN', 'AYM', 'HAMOU', 'ALIWA', 'HADAD', 'HAYUN', 'HALIWA', 'AMOU', 'HASSAN', 'HAIM'}
|
|
41
|
+
HA_KNOWN_SURNAMES = ['COHEN', 'KOHEN', 'KOHN', 'COEN', 'CAHEN', 'LEVY', 'LEVI', 'LEVIE', 'LEWY', 'NAGID', 'NASI']
|
|
42
|
+
HA_ARTICLE_NAMES = {'HASSARFATI', 'HAZERFATY', 'HASSERFATY OZIEL', 'HASSARFATY', 'HASSERFATY', 'HASSERFATI'}
|
|
43
|
+
EL_STRUCTURAL_NAMES = {'EL FASSY', 'EL-FASSY', 'ELFASSY'}
|
|
44
|
+
OU_STRUCTURAL_NAMES = {'WAHNOUNOU', 'OUZIELLE', 'OUAHNOUN', 'OUANOUNOU', 'OUZIELE', 'OUAHNON', 'OUZIELI', 'OUZIEL', 'OUAZIEL', 'WAHNOUN', 'WAHNON', 'OUHANOUNOU'}
|
|
45
|
+
AL_STRUCTURAL_NAMES = set()
|
|
46
|
+
BAR_STRUCTURAL_NAMES = {'BARZILAH', 'BARZILLAY', 'BARZILA', 'BARZILAI', 'BARZILLAI', 'BARSILLAI', 'BARSILAI', 'BARZILAY'}
|
|
47
|
+
GUA_EXCEPTIONS = {'LUGUASSY', 'LUGUASI', 'LUGUASSI'}
|
|
48
|
+
WA_EXCEPTIONS = {'GRUNWALD', 'HARWAY', 'GRUNEWALD', 'SCHWARTZ', 'SCHWARZ', 'SCHWARTZ SZERENA', 'SHWARTZ', 'BOHMERWALD'}
|
|
49
|
+
VA_TO_OUA_NAMES = {'VAQNINE', 'VAQNIN', 'VAKNIN', 'VAKNINE'}
|
|
50
|
+
INVALID_SURNAMES = {'BAR', 'NOM', 'HA', 'VIDE', 'ANONYME', 'BEN…', 'BAT', 'OU', '???', 'NN', 'N.N.', 'NEANT', 'IBN', 'EL', 'INCONNU', 'BEN...', 'FECHA .', 'FECHA .', 'ILLISIBLE', 'ABEN', 'AL', 'FECHA', 'BEN', 'SANS NOM', '...'}
|
|
51
|
+
NON_NAME_MARKERS = frozenset({'TEST', 'ESSAI', 'HEBREU', 'SANS INSCRIPTION', 'ILLISIBLE', 'VIDE', 'ANONYME', 'JUIF', 'INCONNU'})
|
|
52
|
+
_VOWEL_CLASS = {'A': 'A', 'E': 'A', 'O': 'O', 'U': 'O', 'I': 'I', 'Y': 'I'}
|
|
53
|
+
_INITIAL_VOWEL_CLASS = {'E': 'A', 'U': 'O', 'Y': 'I'}
|
|
54
|
+
_COMPOUND_PARTICLES = {'BAR', 'W', 'HA', 'OU', 'BN', 'DI', 'IBN', 'EL', 'LA', 'DA', 'LE', 'DE', 'ABI', 'ABN', 'ABEN', 'AL', 'BEN'}
|
|
55
|
+
_COMPOUND_NOISE_WORDS = {'INHUMATION', "D'AGADIR", 'ESTIMES', 'TOMBE', 'YESHIVA', 'FAMILLE', 'COLLECTIVE', 'ELEVES'}
|
|
56
|
+
|
|
57
|
+
# Embedded special mappings (from the mjpn_special_mappings DB table).
|
|
58
|
+
EXACT_MAPPINGS = {'LAGRABLI': 'GRBL', 'ABOUDAHM': 'ABDRHM', 'TADGI': 'ATG', 'MOZNINO': 'MSNN', 'ABISEROR': 'SRR', 'RIBO': 'RBB', 'ASSEDOUN': 'SDN', 'ASBAG': 'SBG', 'CHARROUT': 'RSH', 'BENHOYAL': 'LL', 'HÉBREU': 'LBR', 'ASSEBAN': 'SBN', 'ELGABAS': 'GB', 'SUSSAN': 'SN', 'ASSEDON': 'SDN', 'ELE...': 'LL', 'ZELL': 'ZLR', 'ZALLAG': 'AMZLG', 'BERTINI': 'BTN', 'LABBOUZ': 'LBZ', 'SAAD': 'ASD', 'BEHE': 'BEHE', 'AUBADIA': 'OBD', 'ATBOL': 'TBL', 'RABIN': 'NRBN', 'GHARBIT': 'SHRBT', 'ENSALEM': 'AMSLM', 'HAFFAF': 'ALF', 'BENSRMOUN': 'SMN', 'COUAZIZ': 'BZZ', 'ANSELLEM': 'AMSLM', 'BENCHEL': 'TL', 'DOUAZIZ': 'ODZ', 'BOLAZIZ': 'BZZ', 'OUBADIA': 'OBD', 'BOFFE': 'RF', 'TOUA': 'HT', 'D': 'HDD', 'ROHBOT': 'RBT', 'AKRICHE': 'KRS', 'CUENOUN': 'KHN', 'BELLICHAH': 'BLSH', 'ENCAQUA': 'ANK', 'MARTINEZ': 'MRTN', 'BEADUGO': 'BRDG', 'LANGRY': 'ANKR', 'ABOUHATZEIRA': 'ABHSR', 'TOPDJMAN': 'TRGMN', 'ALBAZ': 'BZ', 'GHOUKROUN': 'SHKRN', 'BENY': 'BNN', 'YAZERI': 'ZHR', 'BOUKHANA': 'AKNN', 'BENSOT': 'IST', 'DIWAN': 'DHN', 'SERELLY': 'ISRL', 'MIRELLY': 'MRHL', 'ELOLYO': 'LL', 'ZERAH': 'ZHR', 'BOUZACLO': 'BZGL', 'BENAUIS': 'BENAUIS', 'MOYA': 'MH', 'NAMIAS': 'NHM', 'ABOUAZIZ': 'BZ', 'OUAZIZ': 'ZZ', 'BERSAMOUD': 'ABRMD', 'BENSAMOUD': 'SMN', 'CHOCHRON': 'SHKRN', 'ABOUHATZIRA': 'ABHSR', 'BENSADOU': 'SDN', 'SAAS': 'ASS', 'HENRI': 'NHR', 'ARRAR': 'HRR', 'MAIMON': 'MYMN', 'MAIMOUN': 'MYMN', 'MAYMON': 'MYMN', 'MEIMOUN': 'MYMN', 'BENMAIMON': 'MYMN', 'BEN MAIMON': 'MYMN', 'BOUHSIRA': 'ABHSR', 'BOUHSSIRA': 'ABHSR', 'BOHSIRA': 'ABHSR', 'ALMOSNINO': 'MSNN', 'ALMOZNINO': 'MSNN', 'ALMOSNINOS': 'MSNN', 'HALEOUI': 'LB', 'BATTINO': 'BETN', 'BETTINO': 'BETN', 'ALMALEH': 'MLH', 'ALMALIAH': 'MLH', 'ABTTANE': 'ABTN', 'MOYE?': 'HM', 'MAR YOSEF': 'ISF', 'MARYOSEF': 'ISF', 'MOR YOSSEF': 'ISF', 'MOR YUSSEF': 'ISF', 'MAHEODA': 'MD', 'BAREBAY': 'BB'}
|
|
59
|
+
REGEX_MAPPINGS = [(re.compile(p, re.IGNORECASE), r) for p, r in [('(ABU|ABOU|ABI|ABEH|ABIH|BOU|BOD|BO|BU)H?S+[EIY]R+A?', 'ABHSR')]]
|
|
60
|
+
REGEX_MAPPINGS.sort(key=lambda pr: len(pr[0].pattern), reverse=True)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def check_special_mapping(name):
|
|
64
|
+
"""DB-free port of the engine's check_special_mapping (v4)."""
|
|
65
|
+
if not name:
|
|
66
|
+
return None
|
|
67
|
+
name_upper = name.upper()
|
|
68
|
+
if name_upper in EXACT_MAPPINGS:
|
|
69
|
+
return EXACT_MAPPINGS[name_upper]
|
|
70
|
+
no_sep = re.sub(r"[\s\-]+", "", name_upper)
|
|
71
|
+
if no_sep != name_upper and no_sep in EXACT_MAPPINGS:
|
|
72
|
+
return EXACT_MAPPINGS[no_sep]
|
|
73
|
+
for pat, root in REGEX_MAPPINGS:
|
|
74
|
+
if pat.fullmatch(name_upper) or (no_sep != name_upper and pat.fullmatch(no_sep)):
|
|
75
|
+
return root
|
|
76
|
+
return None
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def normalize_accents(text):
|
|
80
|
+
"""Remove accented characters, replacing with ASCII equivalents."""
|
|
81
|
+
if not text:
|
|
82
|
+
return text
|
|
83
|
+
result = []
|
|
84
|
+
for char in text.upper():
|
|
85
|
+
result.append(ACCENT_MAP.get(char, char))
|
|
86
|
+
return ''.join(result)
|
|
87
|
+
|
|
88
|
+
def is_fecha_artifact(value: str) -> bool:
|
|
89
|
+
"""Return True if the value is a 'Fecha …' date artifact from the
|
|
90
|
+
Larache import (Phase 2e). Matches 'Fecha .', 'Fecha .', 'Fecha 1914'
|
|
91
|
+
etc., case-insensitive, any whitespace."""
|
|
92
|
+
if not value:
|
|
93
|
+
return False
|
|
94
|
+
s = value.strip().upper()
|
|
95
|
+
return s.startswith('FECHA') and (
|
|
96
|
+
s == 'FECHA' or s[5:].lstrip(' .0123456789') == ''
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
def _effective_consonant_count(text):
|
|
100
|
+
"""E2: count consonants, collapsing a run of the same consonant to one, so
|
|
101
|
+
ABBOU counts 1 (BB), not 2. Keeps BENABBOU aligned with BENABOU."""
|
|
102
|
+
count = 0
|
|
103
|
+
prev = None
|
|
104
|
+
for c in text:
|
|
105
|
+
if c.isalpha() and c not in VOWELS:
|
|
106
|
+
if c != prev:
|
|
107
|
+
count += 1
|
|
108
|
+
prev = c
|
|
109
|
+
else:
|
|
110
|
+
prev = None
|
|
111
|
+
return count
|
|
112
|
+
|
|
113
|
+
def normalize_start_patterns(text):
|
|
114
|
+
"""
|
|
115
|
+
STAGE 1: Normalize start-of-word patterns BEFORE prefix stripping.
|
|
116
|
+
|
|
117
|
+
This handles patterns like:
|
|
118
|
+
- OI at start → A (OIKNINE → AKNINE, then OAKNINE after full process)
|
|
119
|
+
|
|
120
|
+
This MUST happen before prefix stripping, otherwise:
|
|
121
|
+
- OIKNINE → O stripped → IKNINE → OI pattern never matches!
|
|
122
|
+
"""
|
|
123
|
+
if not text or len(text) < 2:
|
|
124
|
+
return text
|
|
125
|
+
|
|
126
|
+
text = text.upper()
|
|
127
|
+
|
|
128
|
+
# OI at start → A (Berber variant of OU)
|
|
129
|
+
# OIKNINE should become AKNINE to match OUAKNINE→AKNINE
|
|
130
|
+
if text.startswith('OI'):
|
|
131
|
+
text = 'A' + text[2:]
|
|
132
|
+
|
|
133
|
+
# GUA / WA / OUA are one sound (owner rule, 2026-08-01). The Judeo-Arabic
|
|
134
|
+
# /wa/ was written OUA by a French clerk, WA by an English or Hebrew one,
|
|
135
|
+
# and GUA by a Spanish one, where <gu> before a vowel is exactly /w/.
|
|
136
|
+
# Collapsing them here, before prefix stripping, is what makes
|
|
137
|
+
# HALIWA = HALIOUA = HALIGUA
|
|
138
|
+
# BENOUALID = BENWALID = BENGUALID
|
|
139
|
+
# one family each. Left unnormalized these produced three roots apiece
|
|
140
|
+
# (HLB/HL/HLG and OLD/BLD/GLD), so the Spanish and English spellings of a
|
|
141
|
+
# family were unreachable from its French one.
|
|
142
|
+
#
|
|
143
|
+
# OUA is the target rather than WA because OU- already has a structural
|
|
144
|
+
# exception list (OUAHNON, OUANOUNOU) that the other spellings must land
|
|
145
|
+
# inside: GUAHNON has to reach the same protection as OUAHNON.
|
|
146
|
+
# GU-not-/w/: LUGUASSY is the LUGASSY family with an inserted u, not a
|
|
147
|
+
# Spanish spelling of /wa/. Checked before the rewrite, same mechanism as
|
|
148
|
+
# the HA-/OU-/BAR- exception lists.
|
|
149
|
+
# ONLY before A. Spanish <gu> is /gw/ before a and o, but a hard /g/
|
|
150
|
+
# before e and i (guerra, guitarra). A first draft of this rule matched
|
|
151
|
+
# GU before any vowel and destroyed 100+ names: GUIGUI lost its root
|
|
152
|
+
# entirely, BENGUIGUI became BN, MERGUI became MR, RODRIGUES became RDRS.
|
|
153
|
+
if text.upper() not in GUA_EXCEPTIONS:
|
|
154
|
+
text = re.sub(r'GUA', 'OUA', text)
|
|
155
|
+
# Medial WA counts too (HALIWA is HALI + /wa/), except in the Ashkenazi
|
|
156
|
+
# names in this corpus where <wa> is not the Judeo-Arabic /wa/ at all.
|
|
157
|
+
if text.upper() not in WA_EXCEPTIONS:
|
|
158
|
+
text = re.sub(r'WA', 'OUA', text)
|
|
159
|
+
|
|
160
|
+
# V before A at word-start is the Israeli transliteration of the same
|
|
161
|
+
# Berber wa- prefix that OUA/WA/GUA collapse to OUA above (Section 52.2,
|
|
162
|
+
# docs/MJPN_MONOGRAPH.md; verified defect 2026-08-08: VAKNIN alone reached
|
|
163
|
+
# BKNN, via the later V->B consonant-equivalence rule (Beider Table 2.7),
|
|
164
|
+
# while OUAKNINE/WAKNIN/AKNIN/OIKNINE/AKNINE all reach AKNN).
|
|
165
|
+
#
|
|
166
|
+
# This is a WHITELIST, unlike WA_EXCEPTIONS' blocklist: a corpus-wide dry
|
|
167
|
+
# run (data/reports/vaknin_rule_dryrun_20260808.csv) showed every OTHER
|
|
168
|
+
# VA-initial spelling in this archive (VALENCY, VALENCI, VALENZY, VATURI)
|
|
169
|
+
# -- and VALENSI/VIDAL/VITAL, checked explicitly though only VALENSI
|
|
170
|
+
# starts with VA -- is genuine Latin V (Beider: bet/vet) and must not
|
|
171
|
+
# move. A blocklist would require naming every future VA- Latin surname
|
|
172
|
+
# to protect it; a whitelist of the one attested family name does not.
|
|
173
|
+
if text in VA_TO_OUA_NAMES:
|
|
174
|
+
text = 'OUA' + text[2:]
|
|
175
|
+
|
|
176
|
+
return text
|
|
177
|
+
|
|
178
|
+
def strip_latin_prefix(text):
|
|
179
|
+
"""
|
|
180
|
+
STAGE 2: Remove Latin patronymic prefix.
|
|
181
|
+
|
|
182
|
+
v3.5.0 CHANGES:
|
|
183
|
+
- Added: Generalized HA- stripping with HA_STRUCTURAL_NAMES exception list
|
|
184
|
+
- Added: OU_STRUCTURAL_NAMES exception list (OUANOUNOU, etc.)
|
|
185
|
+
- Added: EL_STRUCTURAL_NAMES exception list (ELFASSY)
|
|
186
|
+
|
|
187
|
+
NOTE: Special handling for:
|
|
188
|
+
- 'O': NOT stripped when followed by 'I' (OI→A already handled)
|
|
189
|
+
- 'HA': Generalized stripping EXCEPT for HA_STRUCTURAL_NAMES
|
|
190
|
+
- 'OU': Standard stripping EXCEPT for OU_STRUCTURAL_NAMES
|
|
191
|
+
- 'EL': Stripped EXCEPT for EL_STRUCTURAL_NAMES
|
|
192
|
+
"""
|
|
193
|
+
if not text:
|
|
194
|
+
return text
|
|
195
|
+
|
|
196
|
+
text_upper = text.upper()
|
|
197
|
+
|
|
198
|
+
# v3.5.0: Check EL- structural names BEFORE prefix stripping
|
|
199
|
+
for el_name in EL_STRUCTURAL_NAMES:
|
|
200
|
+
clean_name = el_name.replace('-', '').replace(' ', '')
|
|
201
|
+
if text_upper == clean_name or text_upper == el_name:
|
|
202
|
+
return text_upper # Preserve EL- prefix
|
|
203
|
+
|
|
204
|
+
# v3.5.0: Check OU- structural names
|
|
205
|
+
for ou_name in OU_STRUCTURAL_NAMES:
|
|
206
|
+
if text_upper == ou_name or text_upper.startswith(ou_name):
|
|
207
|
+
return text_upper # Preserve OU- prefix
|
|
208
|
+
|
|
209
|
+
# BAR- structural names (BARZILAI, BARUCH): integral, never a patronymic
|
|
210
|
+
for bar_name in BAR_STRUCTURAL_NAMES:
|
|
211
|
+
if text_upper == bar_name or text_upper.startswith(bar_name):
|
|
212
|
+
return text_upper # Preserve BAR-
|
|
213
|
+
|
|
214
|
+
# Sort by length (longest first) to match correctly
|
|
215
|
+
sorted_prefixes = sorted(LATIN_PREFIXES, key=len, reverse=True)
|
|
216
|
+
|
|
217
|
+
for prefix in sorted_prefixes:
|
|
218
|
+
if text_upper.startswith(prefix):
|
|
219
|
+
remainder = text_upper[len(prefix):]
|
|
220
|
+
# Only strip if remainder has enough substance (2+ consonants)
|
|
221
|
+
consonants_in_remainder = _effective_consonant_count(remainder)
|
|
222
|
+
if consonants_in_remainder >= 2 and len(remainder) >= 2:
|
|
223
|
+
return remainder
|
|
224
|
+
|
|
225
|
+
# v3.5.0: GENERALIZED HA- stripping with exception list
|
|
226
|
+
# Strip HA- unless:
|
|
227
|
+
# 1. Name is in HA_STRUCTURAL_NAMES (HAMOU, HAIM, HADAD, HAZAN, HAYON, HAYOUN)
|
|
228
|
+
# 2. Remainder matches HA_KNOWN_SURNAMES (COHEN, LEVY, etc.)
|
|
229
|
+
# 3. Remainder is too short
|
|
230
|
+
if text_upper.startswith('HA') and len(text_upper) > 3:
|
|
231
|
+
remainder = text_upper[2:]
|
|
232
|
+
|
|
233
|
+
# HA- is the article here despite the structural list: strip it.
|
|
234
|
+
for article_name in HA_ARTICLE_NAMES:
|
|
235
|
+
if text_upper == article_name or text_upper.startswith(article_name):
|
|
236
|
+
return text_upper[2:]
|
|
237
|
+
|
|
238
|
+
# Check if this is a structural HA- name (HAMOU, HAIM, etc.)
|
|
239
|
+
for structural in HA_STRUCTURAL_NAMES:
|
|
240
|
+
if text_upper == structural or text_upper.startswith(structural[:4]):
|
|
241
|
+
return text_upper # Keep HA- prefix
|
|
242
|
+
|
|
243
|
+
# Check if remainder is a known surname (COHEN, LEVY)
|
|
244
|
+
for known in HA_KNOWN_SURNAMES:
|
|
245
|
+
if remainder.startswith(known) or remainder == known:
|
|
246
|
+
return remainder # HACOHEN -> COHEN
|
|
247
|
+
|
|
248
|
+
# v3.5.0: Generalized HA- stripping for OTHER names
|
|
249
|
+
# Strip HA- if remainder has enough substance
|
|
250
|
+
consonants_in_remainder = _effective_consonant_count(remainder)
|
|
251
|
+
if consonants_in_remainder >= 2 and len(remainder) >= 3:
|
|
252
|
+
return remainder # Strip HA- (e.g., HARARI -> ARARI)
|
|
253
|
+
|
|
254
|
+
# Handle single 'O' prefix specially
|
|
255
|
+
# O is a Berber patronymic prefix (variant of OU/W)
|
|
256
|
+
if text_upper.startswith('O') and len(text_upper) > 3:
|
|
257
|
+
remainder = text_upper[1:]
|
|
258
|
+
consonants_in_remainder = _effective_consonant_count(remainder)
|
|
259
|
+
|
|
260
|
+
# Only strip O if:
|
|
261
|
+
# - Remainder has enough consonants (2+)
|
|
262
|
+
# - AND NOT followed by 'I' (OI patterns handled in stage 1)
|
|
263
|
+
if consonants_in_remainder >= 2 and not remainder.startswith('I'):
|
|
264
|
+
# Pattern 1: O + H + vowel (OHAYON, OHANA)
|
|
265
|
+
if remainder.startswith('H') and len(remainder) > 1 and remainder[1] in VOWELS:
|
|
266
|
+
return remainder # OHAYON → HAYON
|
|
267
|
+
|
|
268
|
+
# Pattern 2: O + vowel + consonants (OAKNIN → AKNIN)
|
|
269
|
+
# This is a Berber variant where O replaces OU/W
|
|
270
|
+
if remainder[0] in VOWELS and len(remainder) > 2:
|
|
271
|
+
# Check that it's not a double vowel that forms a different sound
|
|
272
|
+
# OA-KNIN → AKNIN (strip O)
|
|
273
|
+
# But be careful with patterns like OOZE where OO is meaningful
|
|
274
|
+
if remainder[0] != 'O': # Avoid stripping O from OO-
|
|
275
|
+
return remainder # OAKNIN → AKNIN
|
|
276
|
+
|
|
277
|
+
# v3.5.0: EL-/AL- stripping (unless structural)
|
|
278
|
+
if text_upper.startswith('EL') and len(text_upper) > 4:
|
|
279
|
+
# Check if this is a structural EL- name
|
|
280
|
+
is_structural = False
|
|
281
|
+
for el_name in EL_STRUCTURAL_NAMES:
|
|
282
|
+
clean_name = el_name.replace('-', '').replace(' ', '')
|
|
283
|
+
if text_upper == clean_name:
|
|
284
|
+
is_structural = True
|
|
285
|
+
break
|
|
286
|
+
|
|
287
|
+
if not is_structural:
|
|
288
|
+
remainder = text_upper[2:]
|
|
289
|
+
consonants_in_remainder = _effective_consonant_count(remainder)
|
|
290
|
+
if consonants_in_remainder >= 2 and len(remainder) >= 3:
|
|
291
|
+
return remainder # Strip EL-
|
|
292
|
+
|
|
293
|
+
# E6 NOT applied as a blanket rule: the dry-run showed a generic AL- strip
|
|
294
|
+
# false-merges names where AL is integral (Allouche, Allouf, Alexandre,
|
|
295
|
+
# Alster, ...). That repeats the v3.4 Z->S over-merge (rolled back, I2), so
|
|
296
|
+
# AL- families are handled per-name via mjpn_special_mappings / adjudication.
|
|
297
|
+
# AL_STRUCTURAL_NAMES remains for a future targeted exception list.
|
|
298
|
+
|
|
299
|
+
return text_upper
|
|
300
|
+
|
|
301
|
+
def apply_context_c_rule(text):
|
|
302
|
+
"""
|
|
303
|
+
Apply context-dependent C transformation (v3.4).
|
|
304
|
+
|
|
305
|
+
Per Beider Table 2.5 (Spanish/French orthographic equivalence):
|
|
306
|
+
- C before E, I, Y → S (ce, ci → se, si)
|
|
307
|
+
- C elsewhere → K
|
|
308
|
+
|
|
309
|
+
Examples:
|
|
310
|
+
MARCIANO → MARSIANO (C before I → S)
|
|
311
|
+
COHEN → KOHEN (C before O → K)
|
|
312
|
+
CAHEN → KAHEN (C before A → K)
|
|
313
|
+
|
|
314
|
+
Returns:
|
|
315
|
+
str: Text with C replaced appropriately
|
|
316
|
+
"""
|
|
317
|
+
if not text or 'C' not in text:
|
|
318
|
+
return text
|
|
319
|
+
|
|
320
|
+
result = []
|
|
321
|
+
text_upper = text.upper()
|
|
322
|
+
|
|
323
|
+
for i, char in enumerate(text_upper):
|
|
324
|
+
if char == 'C':
|
|
325
|
+
# Check what follows C
|
|
326
|
+
next_char = text_upper[i + 1] if i + 1 < len(text_upper) else ''
|
|
327
|
+
if next_char in 'EIY':
|
|
328
|
+
# C before E, I, Y → S (Spanish/French soft C)
|
|
329
|
+
result.append('S')
|
|
330
|
+
else:
|
|
331
|
+
# C elsewhere → K
|
|
332
|
+
result.append('K')
|
|
333
|
+
else:
|
|
334
|
+
result.append(char)
|
|
335
|
+
|
|
336
|
+
return ''.join(result)
|
|
337
|
+
|
|
338
|
+
def strip_spanish_plural_s(text):
|
|
339
|
+
"""
|
|
340
|
+
Strip final -S from Spanish plural/patronymic suffixes (v3.4.2).
|
|
341
|
+
|
|
342
|
+
Per Beider p. 148-149: Final -S is a Spanish plural or patronymic marker.
|
|
343
|
+
Same family names exist with and without the -S:
|
|
344
|
+
- ATTIA ↔ ATTIAS (285+36 = 321 records in database)
|
|
345
|
+
- NAHMIA ↔ NAHMIAS (41 records)
|
|
346
|
+
- CORCIA ↔ CORCIAS (28 records)
|
|
347
|
+
- SORIA ↔ SORIAS (3 records)
|
|
348
|
+
|
|
349
|
+
Rule: Strip final -S when name ends in -IAS pattern.
|
|
350
|
+
This is a CONSERVATIVE rule targeting only the well-attested Spanish pattern.
|
|
351
|
+
|
|
352
|
+
DO NOT strip:
|
|
353
|
+
- -EZ names (PEREZ, SANCHEZ) — -EZ is a different suffix
|
|
354
|
+
- -AS names where S is part of root (BIBAS, MESSAS, SASPORTAS)
|
|
355
|
+
- -OS names (too risky, many false positives)
|
|
356
|
+
|
|
357
|
+
Returns:
|
|
358
|
+
str: Text with Spanish plural -S stripped if applicable
|
|
359
|
+
"""
|
|
360
|
+
if not text or len(text) < 4:
|
|
361
|
+
return text
|
|
362
|
+
|
|
363
|
+
text_upper = text.upper()
|
|
364
|
+
|
|
365
|
+
# Only strip -S from -IAS pattern (Spanish feminine patronymic)
|
|
366
|
+
# This is the safest rule with clear etymological basis
|
|
367
|
+
if text_upper.endswith('IAS'):
|
|
368
|
+
return text_upper[:-1] # ATTIAS → ATTIA
|
|
369
|
+
|
|
370
|
+
return text_upper
|
|
371
|
+
|
|
372
|
+
def y_is_consonant(text, position):
|
|
373
|
+
"""
|
|
374
|
+
Determine if Y at given position should be treated as consonant.
|
|
375
|
+
|
|
376
|
+
Y is consonant when:
|
|
377
|
+
- At start of name (YOSEF, YAAKOV)
|
|
378
|
+
- Followed by vowel (YA, YO, YE, YI, YU)
|
|
379
|
+
"""
|
|
380
|
+
if position >= len(text):
|
|
381
|
+
return False
|
|
382
|
+
|
|
383
|
+
# Y at start is consonant
|
|
384
|
+
if position == 0:
|
|
385
|
+
return True
|
|
386
|
+
|
|
387
|
+
# Y followed by vowel is consonant
|
|
388
|
+
if position + 1 < len(text) and text[position + 1] in VOWELS:
|
|
389
|
+
return True
|
|
390
|
+
|
|
391
|
+
return False
|
|
392
|
+
|
|
393
|
+
def apply_smart_vowel_rules(text):
|
|
394
|
+
"""
|
|
395
|
+
Apply vowel combination rules smartly.
|
|
396
|
+
|
|
397
|
+
KEY FIX: AY→I and EY→I only apply when Y is NOT followed by a vowel.
|
|
398
|
+
|
|
399
|
+
When Y is followed by a vowel, Y is a consonant and should be preserved:
|
|
400
|
+
- HAYON: H-A-Y-O-N → Y followed by O → Y is consonant → keep as HAYON
|
|
401
|
+
- LAYEM: L-A-Y-E-M → Y followed by E → Y is consonant → keep as LAYEM
|
|
402
|
+
|
|
403
|
+
When Y is NOT followed by a vowel, Y is a vowel sound:
|
|
404
|
+
- MARHAYI: ends in AYI → AY is vowel pair → could become MARHI (but careful)
|
|
405
|
+
- BENHALAY: ends in AY → Y is final vowel → AY→I could apply
|
|
406
|
+
"""
|
|
407
|
+
if not text:
|
|
408
|
+
return text
|
|
409
|
+
|
|
410
|
+
result = list(text)
|
|
411
|
+
i = 0
|
|
412
|
+
|
|
413
|
+
while i < len(result) - 1:
|
|
414
|
+
pair = result[i] + result[i + 1]
|
|
415
|
+
|
|
416
|
+
# Check vowel pairs
|
|
417
|
+
if pair in ['OU', 'AU', 'EI']:
|
|
418
|
+
# These always get simplified (when not at start)
|
|
419
|
+
if i > 0:
|
|
420
|
+
replacements = {'OU': 'U', 'AU': 'O', 'EI': 'I'}
|
|
421
|
+
if pair in replacements:
|
|
422
|
+
result[i] = replacements[pair]
|
|
423
|
+
result[i + 1] = '' # Mark for removal
|
|
424
|
+
i += 2
|
|
425
|
+
continue
|
|
426
|
+
|
|
427
|
+
# AI - similar logic to AY: check if I is followed by vowel (making I consonant-like)
|
|
428
|
+
# OHAION: A-I-O → I followed by O → I acts like Y → keep as AION → later becomes AYON-like
|
|
429
|
+
if pair == 'AI':
|
|
430
|
+
if i > 0: # Not at start
|
|
431
|
+
next_pos = i + 2
|
|
432
|
+
if next_pos < len(result) and result[next_pos] in VOWELS:
|
|
433
|
+
# I is followed by vowel → I acts like consonant Y → convert AI to AY
|
|
434
|
+
result[i + 1] = 'Y' # Change I to Y
|
|
435
|
+
i += 1
|
|
436
|
+
continue
|
|
437
|
+
else:
|
|
438
|
+
# I is NOT followed by vowel → AI is vowel pair → AI→I
|
|
439
|
+
result[i] = 'I'
|
|
440
|
+
result[i + 1] = '' # Mark for removal
|
|
441
|
+
i += 2
|
|
442
|
+
continue
|
|
443
|
+
|
|
444
|
+
# AY and EY - check if Y is consonant (followed by vowel)
|
|
445
|
+
if pair in ['AY', 'EY']:
|
|
446
|
+
if i > 0: # Not at start
|
|
447
|
+
# Check what follows Y
|
|
448
|
+
next_pos = i + 2
|
|
449
|
+
if next_pos < len(result) and result[next_pos] in VOWELS:
|
|
450
|
+
# Y is followed by vowel → Y is consonant → KEEP
|
|
451
|
+
i += 1
|
|
452
|
+
continue
|
|
453
|
+
else:
|
|
454
|
+
# Y is NOT followed by vowel → Y is vowel → AY/EY→I
|
|
455
|
+
result[i] = 'I'
|
|
456
|
+
result[i + 1] = '' # Mark for removal
|
|
457
|
+
i += 2
|
|
458
|
+
continue
|
|
459
|
+
|
|
460
|
+
# OI - only apply when not at start (start handled earlier)
|
|
461
|
+
if pair == 'OI' and i > 0:
|
|
462
|
+
result[i] = 'I'
|
|
463
|
+
result[i + 1] = '' # Mark for removal
|
|
464
|
+
i += 2
|
|
465
|
+
continue
|
|
466
|
+
|
|
467
|
+
i += 1
|
|
468
|
+
|
|
469
|
+
# Remove empty strings
|
|
470
|
+
return ''.join(c for c in result if c)
|
|
471
|
+
|
|
472
|
+
def apply_phonetic_rules(text):
|
|
473
|
+
"""
|
|
474
|
+
STAGE 3: Apply phonetic transformation rules.
|
|
475
|
+
|
|
476
|
+
v3.4.2 CHANGES:
|
|
477
|
+
- Added: Strip final -S from -IAS suffix (Beider p. 148-149: Spanish plural)
|
|
478
|
+
- Merges: ATTIAS↔ATTIA, NAHMIAS↔NAHMIA, CORCIAS↔CORCIA
|
|
479
|
+
|
|
480
|
+
v3.4.1 CHANGES:
|
|
481
|
+
- Added W→B (Beider Table 2.7: waw/bet interchange)
|
|
482
|
+
- C is now context-dependent: C→S before i,e,y; C→K otherwise (Beider Table 2.5)
|
|
483
|
+
- REMOVED Z→S: Caused 724 false merges (HAZAN≠HASSAN, etc.) affecting 14K records
|
|
484
|
+
|
|
485
|
+
KEY FIX v3.0: AY/EY rules now check if Y is consonant (followed by vowel)
|
|
486
|
+
- HAYON: A-Y-O-N → Y is consonant (followed by O) → NOT replaced
|
|
487
|
+
- MARHAYI: A-Y-I → AY at end-ish, Y is vowel → replaced with I
|
|
488
|
+
"""
|
|
489
|
+
if not text:
|
|
490
|
+
return ''
|
|
491
|
+
|
|
492
|
+
result = text.upper()
|
|
493
|
+
|
|
494
|
+
# Digraphs and trigraphs (order matters - longer first!)
|
|
495
|
+
digraph_rules = [
|
|
496
|
+
('TCH', 'SH'),
|
|
497
|
+
('SCH', 'SH'),
|
|
498
|
+
('CH', 'SH'), # French CH = SH
|
|
499
|
+
('PH', 'F'),
|
|
500
|
+
('TZ', 'TS'),
|
|
501
|
+
('DJ', 'G'), # French style → G
|
|
502
|
+
('DG', 'G'),
|
|
503
|
+
('QU', 'K'),
|
|
504
|
+
('CK', 'K'),
|
|
505
|
+
('GN', 'NY'), # French sound
|
|
506
|
+
('GH', 'G'), # v3.6.1: Amghar=Amgar, Sebagh=Sebag (13 merges, 0 splits)
|
|
507
|
+
('TH', 'T'), # v3.6.1: Nathan=Natan, Wakrath=Wakrat (12 merges, 0 splits)
|
|
508
|
+
]
|
|
509
|
+
|
|
510
|
+
for old, new in digraph_rules:
|
|
511
|
+
result = result.replace(old, new)
|
|
512
|
+
|
|
513
|
+
# Context-dependent C rule (v3.4):
|
|
514
|
+
# C→S before i,e,y (Spanish/French: ce, ci → se, si)
|
|
515
|
+
# C→K otherwise
|
|
516
|
+
# This must be applied BEFORE other char_rules that might conflict
|
|
517
|
+
result = apply_context_c_rule(result)
|
|
518
|
+
|
|
519
|
+
# Single character substitutions
|
|
520
|
+
# NOTE: C is handled above by apply_context_c_rule(), not here
|
|
521
|
+
# NOTE: Z→S was removed in v3.4.1 - too aggressive, creates 724 false merges
|
|
522
|
+
# (HAZAN≠HASSAN, ZAZON≠SOUSSAN, etc.) affecting 14K records
|
|
523
|
+
char_rules = [
|
|
524
|
+
('J', 'G'), # J → G
|
|
525
|
+
('Q', 'K'),
|
|
526
|
+
('X', 'KS'),
|
|
527
|
+
('V', 'B'), # V ≈ B in Moroccan (Beider: bet/vet)
|
|
528
|
+
('W', 'B'), # W ≈ B (Beider Table 2.7: waw/bet interchange)
|
|
529
|
+
]
|
|
530
|
+
|
|
531
|
+
for old, new in char_rules:
|
|
532
|
+
result = result.replace(old, new)
|
|
533
|
+
|
|
534
|
+
# Strip Spanish plural -S from -IAS suffix (v3.4.2)
|
|
535
|
+
# Per Beider p. 148-149: ATTIAS↔ATTIA, NAHMIAS↔NAHMIA are same family
|
|
536
|
+
result = strip_spanish_plural_s(result)
|
|
537
|
+
|
|
538
|
+
# Vowel combinations - SMART APPLICATION
|
|
539
|
+
# Key insight: Only apply AY/EY→I when Y is acting as a vowel (not followed by vowel)
|
|
540
|
+
result = apply_smart_vowel_rules(result)
|
|
541
|
+
|
|
542
|
+
return result
|
|
543
|
+
|
|
544
|
+
def extract_consonant_skeleton(text):
|
|
545
|
+
"""
|
|
546
|
+
STAGE 4: Extract consonant skeleton with preserved structure.
|
|
547
|
+
|
|
548
|
+
KEY FEATURES:
|
|
549
|
+
- Preserves consonants separated by vowels (AKNINE → AKNN, two Ns)
|
|
550
|
+
- Only merges CONSECUTIVE duplicate consonants (MANN → MN)
|
|
551
|
+
- Handles Y specially based on context
|
|
552
|
+
- Keeps first vowel if at start (AKNINE → AKNN starts with A)
|
|
553
|
+
"""
|
|
554
|
+
if not text:
|
|
555
|
+
return ''
|
|
556
|
+
|
|
557
|
+
text = text.upper()
|
|
558
|
+
text = ''.join(c for c in text if c.isalpha())
|
|
559
|
+
|
|
560
|
+
if not text:
|
|
561
|
+
return ''
|
|
562
|
+
|
|
563
|
+
# Remove trailing vowels
|
|
564
|
+
while text and text[-1] in VOWELS:
|
|
565
|
+
text = text[:-1]
|
|
566
|
+
|
|
567
|
+
if not text:
|
|
568
|
+
return ''
|
|
569
|
+
|
|
570
|
+
# Build skeleton - track vowels between consonants
|
|
571
|
+
result = []
|
|
572
|
+
prev_consonant = ''
|
|
573
|
+
vowel_since_last_consonant = False
|
|
574
|
+
|
|
575
|
+
for i, char in enumerate(text):
|
|
576
|
+
# Handle Y specially
|
|
577
|
+
if char == 'Y':
|
|
578
|
+
if y_is_consonant(text, i):
|
|
579
|
+
# Y is consonant - keep if different or vowel between
|
|
580
|
+
if vowel_since_last_consonant or char != prev_consonant:
|
|
581
|
+
result.append(char)
|
|
582
|
+
prev_consonant = char
|
|
583
|
+
vowel_since_last_consonant = False
|
|
584
|
+
else:
|
|
585
|
+
# Y is vowel
|
|
586
|
+
vowel_since_last_consonant = True
|
|
587
|
+
continue
|
|
588
|
+
|
|
589
|
+
# Handle vowels
|
|
590
|
+
if char in VOWELS:
|
|
591
|
+
# Keep first vowel if at start
|
|
592
|
+
if i == 0:
|
|
593
|
+
result.append(char)
|
|
594
|
+
vowel_since_last_consonant = True
|
|
595
|
+
continue
|
|
596
|
+
|
|
597
|
+
# Handle consonants
|
|
598
|
+
if char == prev_consonant and not vowel_since_last_consonant:
|
|
599
|
+
# Same consonant, NO vowel between → skip (MM → M)
|
|
600
|
+
continue
|
|
601
|
+
|
|
602
|
+
# Different consonant, OR same but vowel between (N-vowel-N → NN)
|
|
603
|
+
result.append(char)
|
|
604
|
+
prev_consonant = char
|
|
605
|
+
vowel_since_last_consonant = False
|
|
606
|
+
|
|
607
|
+
return ''.join(result)
|
|
608
|
+
|
|
609
|
+
def normalize_to_phonetic(name):
|
|
610
|
+
"""
|
|
611
|
+
Main normalization function - COMPLETE REWRITE v3.0
|
|
612
|
+
|
|
613
|
+
Converts any surname variant to its phonetic root through 4 stages:
|
|
614
|
+
|
|
615
|
+
STAGE 1: Start patterns (OI→A) - BEFORE prefix stripping!
|
|
616
|
+
STAGE 2: Prefix stripping (OU, W, BEN, etc.)
|
|
617
|
+
STAGE 3: Phonetic rules (digraphs, vowel combinations)
|
|
618
|
+
STAGE 4: Consonant skeleton extraction
|
|
619
|
+
|
|
620
|
+
Args:
|
|
621
|
+
name: Surname in Latin script
|
|
622
|
+
|
|
623
|
+
Returns:
|
|
624
|
+
str: Phonetic root (consonant skeleton)
|
|
625
|
+
|
|
626
|
+
Examples:
|
|
627
|
+
normalize_to_phonetic('OUAKNINE') → 'AKNN'
|
|
628
|
+
normalize_to_phonetic('OIKNINE') → 'AKNN' # Fixed in v3.0!
|
|
629
|
+
normalize_to_phonetic('AKOUN') → 'AKN' # Different!
|
|
630
|
+
normalize_to_phonetic('OHAYON') → 'HYN' # Fixed in v3.0!
|
|
631
|
+
normalize_to_phonetic('HAYON') → 'HYN' # Fixed in v3.0!
|
|
632
|
+
normalize_to_phonetic('OHANA') → 'HN' # Different from OHAYON!
|
|
633
|
+
normalize_to_phonetic('HANOUNA') → 'HNN'
|
|
634
|
+
normalize_to_phonetic('BENARROCH') → 'ARSH' (rules-only, v3.7.0)
|
|
635
|
+
"""
|
|
636
|
+
if not name:
|
|
637
|
+
return ''
|
|
638
|
+
|
|
639
|
+
name = name.strip().upper()
|
|
640
|
+
name = normalize_accents(name) # Remove accented characters (é→E, etc.)
|
|
641
|
+
|
|
642
|
+
# A marker is not a name, and must never become a root (owner, 2026-07-26).
|
|
643
|
+
if name in NON_NAME_MARKERS:
|
|
644
|
+
return ''
|
|
645
|
+
|
|
646
|
+
# E5: reject junk — any digit present, or fewer than 2 letters.
|
|
647
|
+
if any(c.isdigit() for c in name) or sum(c.isalpha() for c in name) < 2:
|
|
648
|
+
return ''
|
|
649
|
+
|
|
650
|
+
# v3.4.6: Reject invalid surnames (incomplete/prefix-only entries)
|
|
651
|
+
# These are data quality issues, not valid surnames
|
|
652
|
+
if name in INVALID_SURNAMES:
|
|
653
|
+
return ''
|
|
654
|
+
|
|
655
|
+
# Check special mappings BEFORE normal processing
|
|
656
|
+
special = check_special_mapping(name)
|
|
657
|
+
if special:
|
|
658
|
+
return special
|
|
659
|
+
|
|
660
|
+
# STAGE 1: Normalize start patterns BEFORE prefix stripping
|
|
661
|
+
# This ensures OI→A happens before O is stripped
|
|
662
|
+
stage1 = normalize_start_patterns(name)
|
|
663
|
+
|
|
664
|
+
# Check special mapping after stage 1
|
|
665
|
+
special = check_special_mapping(stage1)
|
|
666
|
+
if special:
|
|
667
|
+
return special
|
|
668
|
+
|
|
669
|
+
# STAGE 2: Strip Latin prefix
|
|
670
|
+
stage2 = strip_latin_prefix(stage1)
|
|
671
|
+
|
|
672
|
+
# Check special mapping on stripped version
|
|
673
|
+
special = check_special_mapping(stage2)
|
|
674
|
+
if special:
|
|
675
|
+
return special
|
|
676
|
+
|
|
677
|
+
# STAGE 3: Apply phonetic rules (with smart vowel handling)
|
|
678
|
+
stage3 = apply_phonetic_rules(stage2)
|
|
679
|
+
|
|
680
|
+
# STAGE 4: Extract consonant skeleton
|
|
681
|
+
phonetic_root = extract_consonant_skeleton(stage3)
|
|
682
|
+
|
|
683
|
+
# (STAGE 5 post-normalization corrections removed in v3.7.1: pure rules only.)
|
|
684
|
+
|
|
685
|
+
# STAGE 6 (v3.6.0): root-initial vowel classing, A/E -> A, O/U -> O,
|
|
686
|
+
# I/Y -> I. Promoted from adjudications after a corpus-wide dry-run
|
|
687
|
+
# (42 merges, 0 splits) and per-merge review on 2026-07-11: Alfassi =
|
|
688
|
+
# Elfassy, Assayag = Esayag, Oziel = Uziel, Ifrah = Yfrah, etc.
|
|
689
|
+
# Special-mapping outputs bypass this stage (they are authoritative).
|
|
690
|
+
# Evidence: docs/mjpn/mapping_analysis_2026_07_11.txt
|
|
691
|
+
if phonetic_root and phonetic_root[0] in _INITIAL_VOWEL_CLASS:
|
|
692
|
+
phonetic_root = _INITIAL_VOWEL_CLASS[phonetic_root[0]] + phonetic_root[1:]
|
|
693
|
+
|
|
694
|
+
return phonetic_root
|
|
695
|
+
|
|
696
|
+
def vowel_signature(text):
|
|
697
|
+
"""Vowel-class signature of `text`: map A/E->A, O/U->O, I/Y->I over the
|
|
698
|
+
form with trailing vowels trimmed, then collapse adjacent duplicates."""
|
|
699
|
+
if not text:
|
|
700
|
+
return ''
|
|
701
|
+
trimmed = text.rstrip('AEIOUY')
|
|
702
|
+
sig = []
|
|
703
|
+
for ch in trimmed:
|
|
704
|
+
cls = _VOWEL_CLASS.get(ch)
|
|
705
|
+
if cls and (not sig or sig[-1] != cls):
|
|
706
|
+
sig.append(cls)
|
|
707
|
+
return ''.join(sig)
|
|
708
|
+
|
|
709
|
+
def _post_stage2_form(name):
|
|
710
|
+
"""The name after Stage 1 (start patterns) and Stage 2 (prefix strip),
|
|
711
|
+
which is where the distinguishing vowels still live (BEN not yet gone)."""
|
|
712
|
+
if not name:
|
|
713
|
+
return ''
|
|
714
|
+
cleaned = normalize_accents(name.strip().upper())
|
|
715
|
+
if not cleaned or cleaned in INVALID_SURNAMES:
|
|
716
|
+
return cleaned
|
|
717
|
+
return strip_latin_prefix(normalize_start_patterns(cleaned))
|
|
718
|
+
|
|
719
|
+
def normalize_to_phonetic_v4(name):
|
|
720
|
+
"""Return (root, subroot). root is exactly normalize_to_phonetic(name);
|
|
721
|
+
subroot = root + '~' + vowel_signature(post-Stage-2 form). Empty root -> ('','').
|
|
722
|
+
|
|
723
|
+
Subroot definition (I12, the single authority): vowel classes A/E->A, O/U->O,
|
|
724
|
+
I/Y->I, adjacent duplicates collapsed, computed on the post-prefix-strip form
|
|
725
|
+
with trailing vowels INCLUDING Y trimmed. An EMPTY signature renders as
|
|
726
|
+
`root~-` (never the bare root), so every attested subroot carries a `~`. The
|
|
727
|
+
special-mapping path computes the signature from the input form (via
|
|
728
|
+
_post_stage2_form) exactly like every other path."""
|
|
729
|
+
root = normalize_to_phonetic(name)
|
|
730
|
+
if not root:
|
|
731
|
+
return ('', '')
|
|
732
|
+
sig = vowel_signature(_post_stage2_form(name))
|
|
733
|
+
return (root, f"{root}~{sig}" if sig else f"{root}~-")
|
|
734
|
+
|
|
735
|
+
def _is_non_name(nom):
|
|
736
|
+
"""Whole-field marker test, before any tokenizing: 'Sans inscription'
|
|
737
|
+
must not be split into two tokens and given a root."""
|
|
738
|
+
return normalize_accents((nom or '').strip().upper()) in NON_NAME_MARKERS
|
|
739
|
+
|
|
740
|
+
def _compound_tokens(nom):
|
|
741
|
+
"""Preprocess a Nom into the final surname tokens (parens dropped,
|
|
742
|
+
separators normalised, particle+name rejoined). Shared by
|
|
743
|
+
compute_compound_indices and compute_compound_indices_v4 so they cannot drift."""
|
|
744
|
+
if not nom or not nom.strip():
|
|
745
|
+
return []
|
|
746
|
+
s = re.sub(r'\s*\([^)]*\)\s*', ' ', nom.strip()) # drop parenthesised comments
|
|
747
|
+
s = re.sub(r'\s*-\s*', ' ', s) # normalise separators
|
|
748
|
+
s = re.sub(r'\s+', ' ', s).strip().upper()
|
|
749
|
+
if not s:
|
|
750
|
+
return []
|
|
751
|
+
tokens = [t for t in s.split(' ') if t]
|
|
752
|
+
merged, i = [], 0
|
|
753
|
+
while i < len(tokens):
|
|
754
|
+
t = tokens[i]
|
|
755
|
+
if t in _COMPOUND_PARTICLES and i + 1 < len(tokens) and len(tokens[i + 1]) >= 3:
|
|
756
|
+
merged.append(t + tokens[i + 1])
|
|
757
|
+
i += 2
|
|
758
|
+
else:
|
|
759
|
+
merged.append(t)
|
|
760
|
+
i += 1
|
|
761
|
+
return merged
|
|
762
|
+
|
|
763
|
+
def compute_compound_indices(nom):
|
|
764
|
+
"""
|
|
765
|
+
Compute (primary_root, secondary_root) for a Nom value.
|
|
766
|
+
|
|
767
|
+
For single-word Nom: returns (root, None).
|
|
768
|
+
For multi-word Nom representing a real compound (e.g., "levy bencheton",
|
|
769
|
+
"bittan serfaty"): returns the MJPN root of each surname part in order.
|
|
770
|
+
Tokens that look like noise (FAMILLE, articles, parentheticals) or are
|
|
771
|
+
not normalizable are skipped.
|
|
772
|
+
|
|
773
|
+
Returns:
|
|
774
|
+
tuple (str|None, str|None) — both may be None if nothing normalizes
|
|
775
|
+
(e.g., "eleves yeshiva d'agadir").
|
|
776
|
+
|
|
777
|
+
Examples:
|
|
778
|
+
>>> compute_compound_indices('cohen')
|
|
779
|
+
('KHN', None)
|
|
780
|
+
>>> compute_compound_indices('levy bencheton')
|
|
781
|
+
('LB', 'SHTN')
|
|
782
|
+
>>> compute_compound_indices('bendavid ohayon')
|
|
783
|
+
('DBD', 'HYN')
|
|
784
|
+
>>> compute_compound_indices('leb lugasi')
|
|
785
|
+
('LB', 'LGS')
|
|
786
|
+
"""
|
|
787
|
+
if _is_non_name(nom):
|
|
788
|
+
return (None, None)
|
|
789
|
+
tokens = _compound_tokens(nom)
|
|
790
|
+
if not tokens:
|
|
791
|
+
return (None, None)
|
|
792
|
+
|
|
793
|
+
# Single token → just normalize
|
|
794
|
+
if len(tokens) == 1:
|
|
795
|
+
root = normalize_to_phonetic(tokens[0])
|
|
796
|
+
return (root or None, None)
|
|
797
|
+
|
|
798
|
+
# Multi token — normalize each, drop empties / noise duplicates
|
|
799
|
+
roots = []
|
|
800
|
+
for tok in tokens:
|
|
801
|
+
if tok in _COMPOUND_NOISE_WORDS:
|
|
802
|
+
continue
|
|
803
|
+
r = normalize_to_phonetic(tok)
|
|
804
|
+
if r and r not in roots:
|
|
805
|
+
roots.append(r)
|
|
806
|
+
if len(roots) >= 2:
|
|
807
|
+
break
|
|
808
|
+
|
|
809
|
+
if not roots:
|
|
810
|
+
return (None, None)
|
|
811
|
+
if len(roots) == 1:
|
|
812
|
+
return (roots[0], None)
|
|
813
|
+
return (roots[0], roots[1])
|
|
814
|
+
|
|
815
|
+
def compute_compound_indices_v4(nom):
|
|
816
|
+
"""(root, root2, subroot, subroot2) — v4 dual index plus per-root subroots.
|
|
817
|
+
|
|
818
|
+
Same token selection as compute_compound_indices; each selected token also
|
|
819
|
+
yields its subroot via normalize_to_phonetic_v4.
|
|
820
|
+
|
|
821
|
+
Examples:
|
|
822
|
+
>>> compute_compound_indices_v4('cohen')
|
|
823
|
+
('KHN', None, 'KHN~OA', None)
|
|
824
|
+
>>> compute_compound_indices_v4('levy bencheton')[:2]
|
|
825
|
+
('LB', 'SHTN')
|
|
826
|
+
"""
|
|
827
|
+
if _is_non_name(nom):
|
|
828
|
+
return (None, None, None, None)
|
|
829
|
+
tokens = _compound_tokens(nom)
|
|
830
|
+
if not tokens:
|
|
831
|
+
return (None, None, None, None)
|
|
832
|
+
if len(tokens) == 1:
|
|
833
|
+
root, subroot = normalize_to_phonetic_v4(tokens[0])
|
|
834
|
+
return (root or None, None, subroot or None, None)
|
|
835
|
+
|
|
836
|
+
pairs, seen = [], []
|
|
837
|
+
for tok in tokens:
|
|
838
|
+
if tok in _COMPOUND_NOISE_WORDS:
|
|
839
|
+
continue
|
|
840
|
+
root, subroot = normalize_to_phonetic_v4(tok)
|
|
841
|
+
if root and root not in seen:
|
|
842
|
+
seen.append(root)
|
|
843
|
+
pairs.append((root, subroot))
|
|
844
|
+
if len(pairs) >= 2:
|
|
845
|
+
break
|
|
846
|
+
if not pairs:
|
|
847
|
+
return (None, None, None, None)
|
|
848
|
+
r1, s1 = pairs[0]
|
|
849
|
+
r2, s2 = pairs[1] if len(pairs) > 1 else (None, None)
|
|
850
|
+
return (r1, r2, s1, s2)
|
|
851
|
+
|
|
852
|
+
|
|
853
|
+
# =============================================================================
|
|
854
|
+
# XLSX RUNNER + STATS (tool layer — the engine above is generated verbatim)
|
|
855
|
+
# =============================================================================
|
|
856
|
+
|
|
857
|
+
def process_name(name):
|
|
858
|
+
"""Run one raw Name through the full pipeline.
|
|
859
|
+
|
|
860
|
+
Returns dict with root/root2/subroot/subroot2, whether a special mapping
|
|
861
|
+
fired, and a status: ok | compound | invalid | empty.
|
|
862
|
+
"""
|
|
863
|
+
if name is None or not str(name).strip():
|
|
864
|
+
return {"name": "" if name is None else str(name),
|
|
865
|
+
"root": None, "root2": None, "subroot": None, "subroot2": None,
|
|
866
|
+
"special": False, "status": "empty"}
|
|
867
|
+
raw = str(name).strip()
|
|
868
|
+
root, root2, subroot, subroot2 = compute_compound_indices_v4(raw)
|
|
869
|
+
special = any(check_special_mapping(normalize_accents(t))
|
|
870
|
+
for t in _compound_tokens(raw))
|
|
871
|
+
if root is None:
|
|
872
|
+
status = "invalid"
|
|
873
|
+
elif root2:
|
|
874
|
+
status = "compound"
|
|
875
|
+
else:
|
|
876
|
+
status = "ok"
|
|
877
|
+
return {"name": raw, "root": root, "root2": root2,
|
|
878
|
+
"subroot": subroot, "subroot2": subroot2,
|
|
879
|
+
"special": special, "status": status}
|
|
880
|
+
|
|
881
|
+
|
|
882
|
+
def compute_stats(rows):
|
|
883
|
+
"""Aggregate stats over process_name() rows."""
|
|
884
|
+
total = len(rows)
|
|
885
|
+
by_status = Counter(r["status"] for r in rows)
|
|
886
|
+
special_hits = sum(1 for r in rows if r["special"])
|
|
887
|
+
families = defaultdict(list) # root -> [names]
|
|
888
|
+
for r in rows:
|
|
889
|
+
for root in (r["root"], r["root2"]):
|
|
890
|
+
if root:
|
|
891
|
+
families[root].append(r["name"])
|
|
892
|
+
sizes = Counter(len(v) for v in families.values())
|
|
893
|
+
dist = {
|
|
894
|
+
"singleton (1 name)": sum(c for s, c in sizes.items() if s == 1),
|
|
895
|
+
"small (2-5)": sum(c for s, c in sizes.items() if 2 <= s <= 5),
|
|
896
|
+
"medium (6-20)": sum(c for s, c in sizes.items() if 6 <= s <= 20),
|
|
897
|
+
"large (>20)": sum(c for s, c in sizes.items() if s > 20),
|
|
898
|
+
}
|
|
899
|
+
top = sorted(families.items(), key=lambda kv: -len(kv[1]))[:20]
|
|
900
|
+
subroots_per_root = defaultdict(set)
|
|
901
|
+
for r in rows:
|
|
902
|
+
if r["root"] and r["subroot"]:
|
|
903
|
+
subroots_per_root[r["root"]].add(r["subroot"])
|
|
904
|
+
split_roots = {k: sorted(v) for k, v in subroots_per_root.items() if len(v) > 1}
|
|
905
|
+
return {
|
|
906
|
+
"total names": total,
|
|
907
|
+
"normalized": by_status.get("ok", 0) + by_status.get("compound", 0),
|
|
908
|
+
"compound names": by_status.get("compound", 0),
|
|
909
|
+
"invalid / not normalizable": by_status.get("invalid", 0),
|
|
910
|
+
"empty cells": by_status.get("empty", 0),
|
|
911
|
+
"special-mapping hits": special_hits,
|
|
912
|
+
"unique roots": len(families),
|
|
913
|
+
"family size distribution": dist,
|
|
914
|
+
"top families": [(root, len(names), ", ".join(sorted(set(names))[:5]))
|
|
915
|
+
for root, names in top],
|
|
916
|
+
"roots split by subroot": {k: v for k, v in sorted(split_roots.items())},
|
|
917
|
+
}
|
|
918
|
+
|
|
919
|
+
|
|
920
|
+
def read_names_xlsx(path):
|
|
921
|
+
"""Read the 'Names' column (case-insensitive header) from an xlsx."""
|
|
922
|
+
from openpyxl import load_workbook
|
|
923
|
+
wb = load_workbook(path, read_only=True, data_only=True)
|
|
924
|
+
ws = wb.active
|
|
925
|
+
rows = ws.iter_rows(values_only=True)
|
|
926
|
+
header = next(rows, None)
|
|
927
|
+
if header is None:
|
|
928
|
+
raise SystemExit(f"{path}: empty sheet")
|
|
929
|
+
idx = None
|
|
930
|
+
for i, h in enumerate(header):
|
|
931
|
+
if h is not None and str(h).strip().lower() == "names":
|
|
932
|
+
idx = i
|
|
933
|
+
break
|
|
934
|
+
if idx is None:
|
|
935
|
+
raise SystemExit(f'{path}: no "Names" column (headers: {list(header)})')
|
|
936
|
+
names = [row[idx] for row in rows if row is not None and len(row) > idx]
|
|
937
|
+
wb.close()
|
|
938
|
+
return names
|
|
939
|
+
|
|
940
|
+
|
|
941
|
+
def write_output_xlsx(path, rows, stats):
|
|
942
|
+
from openpyxl import Workbook
|
|
943
|
+
wb = Workbook()
|
|
944
|
+
ws = wb.active
|
|
945
|
+
ws.title = "MJPN"
|
|
946
|
+
ws.append(["name", "root", "root2", "subroot", "subroot2",
|
|
947
|
+
"special_mapping", "status"])
|
|
948
|
+
for r in rows:
|
|
949
|
+
ws.append([r["name"], r["root"], r["root2"], r["subroot"],
|
|
950
|
+
r["subroot2"], "yes" if r["special"] else "", r["status"]])
|
|
951
|
+
st = wb.create_sheet("Stats")
|
|
952
|
+
for key in ("total names", "normalized", "compound names",
|
|
953
|
+
"invalid / not normalizable", "empty cells",
|
|
954
|
+
"special-mapping hits", "unique roots"):
|
|
955
|
+
st.append([key, stats[key]])
|
|
956
|
+
st.append([])
|
|
957
|
+
st.append(["family size distribution"])
|
|
958
|
+
for k, v in stats["family size distribution"].items():
|
|
959
|
+
st.append([k, v])
|
|
960
|
+
st.append([])
|
|
961
|
+
st.append(["top families", "count", "example names"])
|
|
962
|
+
for root, count, examples in stats["top families"]:
|
|
963
|
+
st.append([root, count, examples])
|
|
964
|
+
if stats["roots split by subroot"]:
|
|
965
|
+
st.append([])
|
|
966
|
+
st.append(["roots split by subroot (v4)", "subroots"])
|
|
967
|
+
for root, subs in stats["roots split by subroot"].items():
|
|
968
|
+
st.append([root, ", ".join(subs)])
|
|
969
|
+
wb.save(path)
|
|
970
|
+
|
|
971
|
+
|
|
972
|
+
def print_stats(stats):
|
|
973
|
+
for key in ("total names", "normalized", "compound names",
|
|
974
|
+
"invalid / not normalizable", "empty cells",
|
|
975
|
+
"special-mapping hits", "unique roots"):
|
|
976
|
+
print(f" {key}: {stats[key]}")
|
|
977
|
+
print(" family size distribution:")
|
|
978
|
+
for k, v in stats["family size distribution"].items():
|
|
979
|
+
print(f" {k}: {v}")
|
|
980
|
+
print(" top families:")
|
|
981
|
+
for root, count, examples in stats["top families"][:10]:
|
|
982
|
+
print(f" {root:<8} {count:>4} ({examples})")
|
|
983
|
+
|
|
984
|
+
|
|
985
|
+
def find_default_xlsx():
|
|
986
|
+
# Current directory first (the natural place when installed as a
|
|
987
|
+
# package), then the script's own folder (one-file standalone usage).
|
|
988
|
+
for folder in (Path.cwd(), Path(__file__).resolve().parent):
|
|
989
|
+
candidates = [p for p in sorted(folder.glob("*.xlsx"))
|
|
990
|
+
if not p.stem.endswith("_mjpn") and not p.name.startswith("~$")]
|
|
991
|
+
if candidates:
|
|
992
|
+
return candidates[0]
|
|
993
|
+
raise SystemExit("no .xlsx found in the current directory or next to "
|
|
994
|
+
f"{Path(__file__).name}; pass a path explicitly")
|
|
995
|
+
|
|
996
|
+
|
|
997
|
+
def main():
|
|
998
|
+
ap = argparse.ArgumentParser(description="MJPN root computation for an "
|
|
999
|
+
'xlsx list of names (column "Names")')
|
|
1000
|
+
ap.add_argument("xlsx", nargs="?", help="input .xlsx (default: first one "
|
|
1001
|
+
"in this script's folder)")
|
|
1002
|
+
ap.add_argument("--name", nargs="+", help="ad-hoc names instead of a file")
|
|
1003
|
+
ap.add_argument("--selftest", action="store_true")
|
|
1004
|
+
args = ap.parse_args()
|
|
1005
|
+
|
|
1006
|
+
if args.selftest:
|
|
1007
|
+
checks = [("BENHAIM", "HM"), ("MARCIANO", "MRSN"), ("OUAKNINE", "AKNN"),
|
|
1008
|
+
("SASSON", "SSN"), ("COHEN", "KHN"), ("BENARROCH", "ARSH")]
|
|
1009
|
+
ok = True
|
|
1010
|
+
for name, expected in checks:
|
|
1011
|
+
got = normalize_to_phonetic(name)
|
|
1012
|
+
print(f" {'OK ' if got == expected else 'XX '}{name} -> {got}")
|
|
1013
|
+
ok = ok and got == expected
|
|
1014
|
+
sys.exit(0 if ok else 1)
|
|
1015
|
+
|
|
1016
|
+
if args.name:
|
|
1017
|
+
for n in args.name:
|
|
1018
|
+
r = process_name(n)
|
|
1019
|
+
print(f"{n} -> root={r['root']} root2={r['root2']} "
|
|
1020
|
+
f"subroot={r['subroot']} subroot2={r['subroot2']}")
|
|
1021
|
+
return
|
|
1022
|
+
|
|
1023
|
+
src = Path(args.xlsx) if args.xlsx else find_default_xlsx()
|
|
1024
|
+
print(f"input: {src}")
|
|
1025
|
+
names = read_names_xlsx(src)
|
|
1026
|
+
rows = [process_name(n) for n in names]
|
|
1027
|
+
stats = compute_stats(rows)
|
|
1028
|
+
out = src.with_name(src.stem + "_mjpn.xlsx")
|
|
1029
|
+
write_output_xlsx(out, rows, stats)
|
|
1030
|
+
print(f"output: {out}")
|
|
1031
|
+
print_stats(stats)
|
|
1032
|
+
|
|
1033
|
+
|
|
1034
|
+
if __name__ == "__main__":
|
|
1035
|
+
main()
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: mjpn
|
|
3
|
+
Version: 6.0.0
|
|
4
|
+
Summary: Moroccan Jewish Phonetic Names (MJPN): phonetic root computation for Moroccan Jewish surnames, the engine behind yahasra.org
|
|
5
|
+
Author-email: Dr Yohanan Ouaknine <contact@yahasra.org>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://yahasra.org
|
|
8
|
+
Project-URL: Documentation, https://yahasra.org/about
|
|
9
|
+
Keywords: onomastics,phonetic,surnames,genealogy,jewish,morocco,sephardic
|
|
10
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Topic :: Sociology :: Genealogy
|
|
15
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
16
|
+
Requires-Python: >=3.8
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: openpyxl>=3.0
|
|
20
|
+
Dynamic: license-file
|
|
21
|
+
|
|
22
|
+
# MJPN — Moroccan Jewish Phonetic Names
|
|
23
|
+
|
|
24
|
+
Phonetic root computation for Moroccan Jewish surnames — the exact engine
|
|
25
|
+
behind [yahasra.org](https://yahasra.org)'s search across 32,000+ burial
|
|
26
|
+
records from Jewish cemeteries in Morocco.
|
|
27
|
+
|
|
28
|
+
Moroccan Jewish surnames were written down by French, Spanish and Hebrew
|
|
29
|
+
scribes, each with their own orthography: the same family appears as
|
|
30
|
+
*Ouaknine, Vaknin, Aknin, Boaknin, Oiknine…* MJPN reduces every variant to a
|
|
31
|
+
stable phonetic root (`AKNN`), so records that belong together can be found
|
|
32
|
+
together.
|
|
33
|
+
|
|
34
|
+
## Install
|
|
35
|
+
|
|
36
|
+
```
|
|
37
|
+
pip install mjpn
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Command line
|
|
41
|
+
|
|
42
|
+
```
|
|
43
|
+
mjpn myfile.xlsx # xlsx with a "Names" column (case-insensitive)
|
|
44
|
+
mjpn # first .xlsx found in the current script folder
|
|
45
|
+
mjpn --name COHEN OUAKNINE "LEVY BENCHETON"
|
|
46
|
+
mjpn --selftest
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
The xlsx run writes `<input>_mjpn.xlsx` next to the input, with:
|
|
50
|
+
|
|
51
|
+
- sheet **MJPN** — `name, root, root2, subroot, subroot2, special_mapping, status`
|
|
52
|
+
(`status` is `ok` / `compound` / `invalid` / `empty`)
|
|
53
|
+
- sheet **Stats** — totals, family-size distribution, top families,
|
|
54
|
+
roots split by v4 subroot
|
|
55
|
+
|
|
56
|
+
and prints the stats summary to stdout.
|
|
57
|
+
|
|
58
|
+
## Python API
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
from mjpn import normalize_to_phonetic, compute_compound_indices_v4
|
|
62
|
+
|
|
63
|
+
normalize_to_phonetic("OUAKNINE") # 'AKNN'
|
|
64
|
+
normalize_to_phonetic("BENARROCH") # 'HRSH'
|
|
65
|
+
compute_compound_indices_v4("levy bencheton")
|
|
66
|
+
# ('LB', 'SHTN', 'LB~A', 'SHTN~AO') — compound dual index + v4 subroots
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
## How it works
|
|
70
|
+
|
|
71
|
+
Four normalization stages (engine v3.5.2, v4 subroot chain):
|
|
72
|
+
|
|
73
|
+
1. **Start patterns** — Berber orthographic variants (OI→A) before anything else.
|
|
74
|
+
2. **Prefix stripping** — patronymic prefixes (BEN, IBN, OU, HA, EL…) with
|
|
75
|
+
structural-name exception lists so *Hamou* or *Elfassy* keep their prefix.
|
|
76
|
+
3. **Phonetic rules** — digraphs (CH→SH…), context-dependent C, V/W→B,
|
|
77
|
+
Spanish plural -IAS→-IA, smart vowel-pair handling (based on Beider's
|
|
78
|
+
*A Dictionary of Jewish Surnames from Maghreb*, 2017).
|
|
79
|
+
4. **Consonant skeleton** — the root; a vowel-class signature on top yields
|
|
80
|
+
the v4 *subroot* that keeps genuinely different families apart.
|
|
81
|
+
|
|
82
|
+
Plus 149 exact and 2 regex special mappings for families whose variants defy
|
|
83
|
+
the rules (Abuhatzeira, Azoulay, Zrihen…), curated from the yahasra.org
|
|
84
|
+
adjudication process.
|
|
85
|
+
|
|
86
|
+
`mjpn/mjpn_tool.py` is **generated** from the production engine and verified
|
|
87
|
+
against it: identical output on all 3,933 distinct surnames of the yahasra.org
|
|
88
|
+
burials database (0 mismatches).
|
|
89
|
+
|
|
90
|
+
## Citation
|
|
91
|
+
|
|
92
|
+
If you use MJPN in academic work, please cite:
|
|
93
|
+
|
|
94
|
+
> Ouaknine, Y. (2026). MJPN: Moroccan Jewish Phonetic Names — the phonetic
|
|
95
|
+
> indexing engine of yahasra.org, the Moroccan Jewish cemeteries database.
|
|
96
|
+
> https://yahasra.org
|
|
97
|
+
|
|
98
|
+
## License
|
|
99
|
+
|
|
100
|
+
MIT © Dr Yohanan Ouaknine
|
|
101
|
+
|
|
102
|
+
## Provenance and AI disclosure
|
|
103
|
+
|
|
104
|
+
The engine (`app/phonetic_normalizer.py` in the Yahasra codebase) encodes
|
|
105
|
+
orthographic rules authored and adjudicated by Dr Yohanan Ouaknine, tuned
|
|
106
|
+
against Beider (2017) with page references recorded per rule. This package is
|
|
107
|
+
generated mechanically from that engine by `scripts/build_mjpn_tool.py` — no
|
|
108
|
+
hand-maintained copy exists — and each release is parity-checked against the
|
|
109
|
+
full live corpus (5,314 distinct surnames, zero mismatches required).
|
|
110
|
+
|
|
111
|
+
Portions of the engine's implementation code and this package's build
|
|
112
|
+
tooling were produced with assistive AI (Anthropic Claude: Sonnet 4.x and
|
|
113
|
+
Opus 4.x/5, 2025-06 through 2026-08) under human direction and review; all
|
|
114
|
+
linguistic rules, special mappings and adjudications are human decisions.
|
|
115
|
+
This disclosure follows the AI policy of the Journal of Cultural Analytics.
|
|
116
|
+
|
|
117
|
+
## Version
|
|
118
|
+
|
|
119
|
+
Package 6.0.0 wraps engine 6.0.0 (v4 chain: subroots + compound dual
|
|
120
|
+
indexing), generated 2026-11-16 with 98 exact + 1 regex special mappings.
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
mjpn/__init__.py,sha256=IDHRQ3Z-iU2nSyO9aSZrDiAoceZ9AmVRiANVVZnzAx0,965
|
|
2
|
+
mjpn/mjpn_tool.py,sha256=gNpBSke6psHURz8aU3XxJ1HapvM1lc_WAXWixivJ6Pg,42559
|
|
3
|
+
mjpn-6.0.0.dist-info/licenses/LICENSE,sha256=q1mDwNHs4ZwsDeHe39N2-lC0BWyTLi_MK5H5wozSzLE,1076
|
|
4
|
+
mjpn-6.0.0.dist-info/METADATA,sha256=R4UstybdoCr9bHW1pgNtBVxWgr9f_wZXEKbBboUykRs,4684
|
|
5
|
+
mjpn-6.0.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
6
|
+
mjpn-6.0.0.dist-info/entry_points.txt,sha256=H01WCt3GBTU7R4Dfx09pElqbu-KW_o26FIVeZ9C_Mzg,45
|
|
7
|
+
mjpn-6.0.0.dist-info/top_level.txt,sha256=6q558vgYhSThTq81kf9HGDPuQWYxpXPGBn_X7HSADkY,5
|
|
8
|
+
mjpn-6.0.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Dr Yohanan Ouaknine
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
mjpn
|