nameparser 2.4.0__tar.gz → 2.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nameparser-2.4.0/nameparser.egg-info → nameparser-2.4.1}/PKG-INFO +1 -1
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_facade.py +2 -2
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/_classify.py +3 -1
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/_comma.py +11 -6
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/_extract.py +13 -7
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/_group.py +5 -2
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/_script_segment.py +2 -2
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/_tokenize.py +10 -4
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/_vocab.py +75 -13
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_version.py +4 -4
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/_invariants.py +2 -3
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/regexes.py +10 -1
- {nameparser-2.4.0 → nameparser-2.4.1/nameparser.egg-info}/PKG-INFO +1 -1
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser.egg-info/SOURCES.txt +5 -1
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_initials.py +3 -16
- nameparser-2.4.1/tests/v2/_differential_fixtures.py +55 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/cases.py +108 -109
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/conftest.py +14 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/test_group.py +1 -1
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/test_tokenize.py +4 -2
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/test_vocab.py +14 -1
- nameparser-2.4.1/tests/v2/test_cldr_agreement.py +208 -0
- nameparser-2.4.1/tests/v2/test_cldr_corpus.py +118 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_config_aliases.py +1 -2
- nameparser-2.4.1/tests/v2/test_corpora.py +190 -0
- nameparser-2.4.1/tests/v2/test_differential.py +454 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_doc_citations.py +12 -16
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_lexicon.py +23 -0
- nameparser-2.4.1/tests/v2/test_name_usage_tool.py +303 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_parser.py +111 -1
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_regex_sync.py +5 -5
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_render.py +1 -2
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_rules_doc.py +11 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_rules_doc_grammar.py +2 -6
- nameparser-2.4.1/tests/v2/test_snapshot.py +48 -0
- nameparser-2.4.0/tests/v2/_differential_fixtures.py +0 -120
- nameparser-2.4.0/tests/v2/test_differential.py +0 -4792
- nameparser-2.4.0/tests/v2/test_ledger_guards.py +0 -8902
- {nameparser-2.4.0 → nameparser-2.4.1}/AUTHORS +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/LICENSE +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/MANIFEST.in +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/README.rst +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/__init__.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/__main__.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_config_shim.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_lexicon.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_locale.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_parser.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/__init__.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/_assemble.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/_assign.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/_pieces.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/_post_rules.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/_segment.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/_state.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_policy.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_render.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_types.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/__init__.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/_deprecated.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/bound_first_names.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/bound_given_names.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/capitalization.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/conjunctions.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/maiden_markers.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/particles.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/prefixes.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/suffixes.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/surnames.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/titles.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/locales/__init__.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/locales/ja.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/locales/ru.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/locales/tr_az.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/locales/zh.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/parser.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/py.typed +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/util.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser.egg-info/dependency_links.txt +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser.egg-info/requires.txt +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/nameparser.egg-info/top_level.txt +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/pyproject.toml +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/setup.cfg +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/__init__.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/base.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/conftest.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_bound_given_names.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_brute_force.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_capitalization.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_comma_variants.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_conjunctions.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_constants.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_east_slavic_patronymic_order.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_first_name.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_middle_name_as_last.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_nicknames.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_output_format.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_particles.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_python_api.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_suffixes.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_titles.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_turkic_patronymic_order.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_variations.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/__init__.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/__init__.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/test_assemble.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/test_assign.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/test_classify.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/test_comma.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/test_extract.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/test_pieces.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/test_post_rules.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/test_script_segment.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/test_segment.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/test_state.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/rules_doc.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_benchmark.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_cases.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_cli.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_config_shim.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_contracts.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_doc_spellings.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_facade.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_facade_cases.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_layering.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_locale.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_locales.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_order_correspondence.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_policy.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_properties.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_reprs.py +0 -0
- {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_types.py +0 -0
|
@@ -709,8 +709,8 @@ class HumanName:
|
|
|
709
709
|
# which compare the seven role fields only (Case carries
|
|
710
710
|
# no initials column); the covering test is
|
|
711
711
|
# tests/test_initials.py::test_initials_middle_name_all_prefixes,
|
|
712
|
-
# and
|
|
713
|
-
# both surfaces for
|
|
712
|
+
# and tools/differential/snapshot.jsonl records initials()
|
|
713
|
+
# on both surfaces for every corpus name. _split_last
|
|
714
714
|
# already applies the same guard to the base, which is why
|
|
715
715
|
# last_base was never empty here.
|
|
716
716
|
return [_render._first_letter(w) for w in words]
|
|
@@ -52,6 +52,7 @@ is what lets the fork be reported without being taken.
|
|
|
52
52
|
"""
|
|
53
53
|
from __future__ import annotations
|
|
54
54
|
|
|
55
|
+
import unicodedata
|
|
55
56
|
|
|
56
57
|
from nameparser._lexicon import _normalize, _spells_an_initial
|
|
57
58
|
from nameparser._policy import CapsSuffixes
|
|
@@ -229,7 +230,8 @@ def _tags_for(token: WorkToken, n: str, state: ParseState,
|
|
|
229
230
|
# C: a listed member ('John Smith, MA'), and a word that
|
|
230
231
|
# is not alphabetic capitals ('John Smith, Ph. D.')
|
|
231
232
|
and n not in lex.suffix_acronyms_ambiguous
|
|
232
|
-
and token.text.isalpha() and token.text.isupper()
|
|
233
|
+
and (token.text.isalpha() and token.text.isupper()
|
|
234
|
+
or not unicodedata.is_normalized("NFC", token.text))
|
|
233
235
|
and caps_shape_candidate(token.text, lex, state.policy,
|
|
234
236
|
one_case)):
|
|
235
237
|
# #516's all-caps half: an unlisted word written in capitals
|
|
@@ -27,6 +27,7 @@ Policy.unlisted_caps_suffixes.
|
|
|
27
27
|
"""
|
|
28
28
|
from __future__ import annotations
|
|
29
29
|
|
|
30
|
+
import unicodedata
|
|
30
31
|
from collections.abc import Sequence
|
|
31
32
|
|
|
32
33
|
from nameparser._lexicon import _normalize
|
|
@@ -40,8 +41,8 @@ from nameparser._pipeline._state import (
|
|
|
40
41
|
)
|
|
41
42
|
from nameparser._pipeline._vocab import (
|
|
42
43
|
D, PH, caps_shape_candidate, claimed_as_non_name, delimiter_cores,
|
|
43
|
-
is_paired_initials, is_single_letter_numeral,
|
|
44
|
-
name_word_count, surname_unit_facts, unit_ends, written_as_a_name,
|
|
44
|
+
composed_len, is_caps_word, is_paired_initials, is_single_letter_numeral,
|
|
45
|
+
is_wholly_suffix, name_word_count, surname_unit_facts, unit_ends, written_as_a_name,
|
|
45
46
|
)
|
|
46
47
|
from nameparser._policy import CapsSuffixes
|
|
47
48
|
from nameparser._types import AmbiguityKind, Role
|
|
@@ -217,8 +218,11 @@ def decide(state: ParseState) -> ParseState:
|
|
|
217
218
|
maybe = [p[0] for p in p1 if len(p) == 1
|
|
218
219
|
and (AMBIGUOUS_ACRONYM_TAG in (tags := tokens[p[0]].tags)
|
|
219
220
|
or ("vocab:suffix" not in tags
|
|
220
|
-
and tokens[p[0]].text.isalpha()
|
|
221
|
-
|
|
221
|
+
and (tokens[p[0]].text.isalpha()
|
|
222
|
+
and tokens[p[0]].text.isupper()
|
|
223
|
+
or not unicodedata.is_normalized(
|
|
224
|
+
"NFC", tokens[p[0]].text)
|
|
225
|
+
and is_caps_word(tokens[p[0]].text))))]
|
|
222
226
|
# The part is read ONCE: here where it holds no word of the class,
|
|
223
227
|
# after the count below where it does, with the words the count
|
|
224
228
|
# licensed (a read here and a second after licensing threw the first
|
|
@@ -307,7 +311,7 @@ def decide(state: ParseState) -> ParseState:
|
|
|
307
311
|
continue
|
|
308
312
|
text = tokens[i].text
|
|
309
313
|
if not (is_paired_initials(text)
|
|
310
|
-
or (i in caps and
|
|
314
|
+
or (i in caps and composed_len(text) == 2)):
|
|
311
315
|
continue
|
|
312
316
|
if speaker < 0:
|
|
313
317
|
speaker = next(
|
|
@@ -375,7 +379,8 @@ def decide(state: ParseState) -> ParseState:
|
|
|
375
379
|
member_set = frozenset(members)
|
|
376
380
|
pairs = [k for k, p in enumerate(p1) if p[0] in member_set
|
|
377
381
|
and (is_paired_initials(tokens[p[0]].text)
|
|
378
|
-
or (p[0] in caps
|
|
382
|
+
or (p[0] in caps
|
|
383
|
+
and composed_len(tokens[p[0]].text) == 2))]
|
|
379
384
|
# spoken for by each other only: every class word a pair, and
|
|
380
385
|
# only titles among the other words in front of the first
|
|
381
386
|
pair_only = (len(pairs) >= 2 and len(pairs) == len(members)
|
|
@@ -38,7 +38,9 @@ from nameparser._lexicon import Lexicon, _normalize
|
|
|
38
38
|
from nameparser._pipeline._state import (
|
|
39
39
|
COMMA_CHARS, ParseState, PendingAmbiguity, copy_with,
|
|
40
40
|
)
|
|
41
|
-
from nameparser._pipeline._vocab import
|
|
41
|
+
from nameparser._pipeline._vocab import (
|
|
42
|
+
comma_follows_marker, maiden_marker_run,
|
|
43
|
+
)
|
|
42
44
|
from nameparser._types import AmbiguityKind, Role, Span
|
|
43
45
|
|
|
44
46
|
|
|
@@ -76,9 +78,12 @@ def _maiden_marked(content: str, lexicon: Lexicon) -> bool:
|
|
|
76
78
|
itself rather than reading the tags classify will set. The two
|
|
77
79
|
questions are deliberately not the same one: this splits on
|
|
78
80
|
WHITESPACE, so a marker the writer glued to punctuation is not one
|
|
79
|
-
here ('née,')
|
|
80
|
-
|
|
81
|
-
|
|
81
|
+
here ('née,') -- nor to classify, since #653 keeps a clause's comma
|
|
82
|
+
on its word. A nakaguro still divides the two: the tokenizer splits
|
|
83
|
+
'旧姓・田中' at it (rules.md#T2) and classify tags '旧姓', which is
|
|
84
|
+
what keeps _group's Role.MAIDEN filter reachable (cases.py's
|
|
85
|
+
marker_divided_by_a_nakaguro_keeps_the_clause_a_nickname). Sharing
|
|
86
|
+
the PREDICATE does not merge the two
|
|
82
87
|
questions -- each hands it a different sequence of words."""
|
|
83
88
|
words = content.split()
|
|
84
89
|
# A one-word clause cannot satisfy the condition whatever the
|
|
@@ -88,7 +93,8 @@ def _maiden_marked(content: str, lexicon: Lexicon) -> bool:
|
|
|
88
93
|
if len(words) < 2:
|
|
89
94
|
return False
|
|
90
95
|
run = maiden_marker_run(words, lexicon.maiden_markers)
|
|
91
|
-
return run > 0 and len(words) > run
|
|
96
|
+
return (run > 0 and len(words) > run
|
|
97
|
+
and not comma_follows_marker(words[run]))
|
|
92
98
|
|
|
93
99
|
|
|
94
100
|
# rules.md#N2: "a quote whose open and close are the same character
|
|
@@ -164,8 +170,8 @@ def _unmatched(open_: str, offset: int) -> tuple[int, PendingAmbiguity]:
|
|
|
164
170
|
|
|
165
171
|
|
|
166
172
|
# rules.md#N1: "a clause enclosed by a configured nickname delimiter
|
|
167
|
-
# pair reads as the nickname and is lifted out of the name
|
|
168
|
-
#
|
|
173
|
+
# pair reads as the nickname and is lifted out of the name" (its
|
|
174
|
+
# punctuation is tokenize's half of the rule)
|
|
169
175
|
# rules.md#M1: "with a delimiter pair configured for maiden names, its
|
|
170
176
|
# enclosed clause reads as the maiden name" (history: decisions.md#M1)
|
|
171
177
|
def extract_delimited(state: ParseState) -> ParseState:
|
|
@@ -78,7 +78,7 @@ from nameparser._pipeline._state import (
|
|
|
78
78
|
import nameparser._pipeline._comma as _comma
|
|
79
79
|
from nameparser._pipeline._vocab import D, PH
|
|
80
80
|
from nameparser._pipeline._vocab import (
|
|
81
|
-
delimiter_cores,
|
|
81
|
+
comma_follows_marker, delimiter_cores,
|
|
82
82
|
)
|
|
83
83
|
from nameparser._types import AmbiguityKind, Role
|
|
84
84
|
|
|
@@ -1523,8 +1523,11 @@ def group(state: ParseState) -> ParseState:
|
|
|
1523
1523
|
# test, since the tokens inside a clause are contiguous in
|
|
1524
1524
|
# index order -- a clause holding nothing but its marker
|
|
1525
1525
|
# keeps its words, exactly as a one-token "(Nee)" does.
|
|
1526
|
+
# rules.md#M3: "a marker followed by a comma is no marker"
|
|
1527
|
+
# -- asked of the same word extract's M3 test asks it of
|
|
1526
1528
|
if (first + run < len(tokens)
|
|
1527
|
-
and tokens[first + run].span.end <= clause.end
|
|
1529
|
+
and tokens[first + run].span.end <= clause.end
|
|
1530
|
+
and not comma_follows_marker(tokens[first + run].text)):
|
|
1528
1531
|
dropped.extend(range(first, first + run))
|
|
1529
1532
|
return copy_with(
|
|
1530
1533
|
state, tokens=tuple(tokens), pieces=tuple(all_pieces),
|
|
@@ -255,8 +255,8 @@ def _is_post_nominal(state: ParseState, i: int) -> bool:
|
|
|
255
255
|
# classification rather than cut into 박사 + 님, which is what the
|
|
256
256
|
# same input gives when the step is removed. '선생님' is post-nominal
|
|
257
257
|
# entire with no name word behind it, so the scan yields no site at
|
|
258
|
-
# all and the token stays whole. Both are
|
|
259
|
-
#
|
|
258
|
+
# all and the token stays whole. Both are corpus names and
|
|
259
|
+
# rules.md#W2 example lines; decisions.md#cjk-comma-demotion
|
|
260
260
|
# carries the forced-predicate measurement behind them.
|
|
261
261
|
def _peel_site(state: ParseState, flat: Sequence[int],
|
|
262
262
|
tails: frozenset[str]) -> tuple[int, int] | None:
|
|
@@ -3,7 +3,8 @@
|
|
|
3
3
|
Consumes: original, masked (regions to skip), extracted (regions that
|
|
4
4
|
tokenize with a pre-set role).
|
|
5
5
|
Produces: tokens (span-sorted WorkTokens; text always == original
|
|
6
|
-
slice), comma_offsets (segmentation points;
|
|
6
|
+
slice), comma_offsets (segmentation points of the main stream; a comma
|
|
7
|
+
there is never a token, one inside an extracted region is word text),
|
|
7
8
|
interpunct_offsets (间隔号 transcription markers, #298; never tokens).
|
|
8
9
|
Reads: Policy.strip_emoji, Policy.strip_bidi.
|
|
9
10
|
|
|
@@ -110,7 +111,12 @@ def _tokenize_region(state: ParseState, start: int, end: int,
|
|
|
110
111
|
tok_start: int | None = None
|
|
111
112
|
for i in range(start, end):
|
|
112
113
|
ch = text[i]
|
|
113
|
-
|
|
114
|
+
# rules.md#N1: "a comma inside it kept as written — the comma
|
|
115
|
+
# divides nothing there" -- a comma separates only where it
|
|
116
|
+
# records a segmentation offset, the main stream; inside an
|
|
117
|
+
# extracted region it is the clause's text, like ';' (#653)
|
|
118
|
+
is_separator = ((record_offsets and ch in COMMA_CHARS)
|
|
119
|
+
or _ignorable(ch, state))
|
|
114
120
|
if ch == _INTERPUNCT:
|
|
115
121
|
# Region-local flanks: a B7 at a region edge stays token
|
|
116
122
|
# text, and the bound also stops a custom delimiter's
|
|
@@ -155,8 +161,8 @@ def tokenize(state: ParseState) -> ParseState:
|
|
|
155
161
|
for start, end in zip(boundaries[::2], boundaries[1::2]):
|
|
156
162
|
_tokenize_region(state, start, end, None, True, tokens, commas,
|
|
157
163
|
interpuncts)
|
|
158
|
-
# extracted regions: pre-set role
|
|
159
|
-
#
|
|
164
|
+
# extracted regions: pre-set role; a comma is word text there and an
|
|
165
|
+
# interpunct a mere separator, neither recording an offset
|
|
160
166
|
for role, inner in state.extracted:
|
|
161
167
|
_tokenize_region(state, inner.start, inner.end, role, False,
|
|
162
168
|
tokens, commas, interpuncts)
|
|
@@ -51,7 +51,7 @@ from nameparser._lexicon import (
|
|
|
51
51
|
)
|
|
52
52
|
from nameparser._policy import (CapsSuffixes, Policy, Script, _JA_SCRIPTS, _NO_INITIALS,
|
|
53
53
|
_SCRIPT_RANGES, _script_matcher)
|
|
54
|
-
from nameparser._pipeline._state import WorkToken, comma_bucket
|
|
54
|
+
from nameparser._pipeline._state import COMMA_CHARS, WorkToken, comma_bucket
|
|
55
55
|
|
|
56
56
|
# Ported verbatim from v1 (nameparser/config/regexes.py "initial") minus
|
|
57
57
|
# its empty-string alternative -- WorkToken text is never empty. Kept in
|
|
@@ -159,12 +159,25 @@ in_initialless_script = _script_matcher(*_NO_INITIALS, whole=False)
|
|
|
159
159
|
# the comma structure to a credential run
|
|
160
160
|
# (mechanisms.md#ONE-PREDICATE-PER-QUESTION).
|
|
161
161
|
#
|
|
162
|
-
#
|
|
163
|
-
#
|
|
164
|
-
#
|
|
165
|
-
#
|
|
166
|
-
#
|
|
167
|
-
|
|
162
|
+
# The shape shared with config's "period_abbreviation" (layering
|
|
163
|
+
# forbids the config import; keep in sync by hand -- the pattern TEXT
|
|
164
|
+
# is what tests/v2/test_regex_sync.py compares, reaching this object
|
|
165
|
+
# through `_pieces._PERIOD_ABBREV`, an IMPORT of this one, not a second
|
|
166
|
+
# definition -- the sync test's target name did not move). It is
|
|
167
|
+
# composed from `_LETTER` below, so the text is v1's plus the marks.
|
|
168
|
+
|
|
169
|
+
# `_LETTER` is the letter every shape regex in this module spells
|
|
170
|
+
# (`_PERIOD_ABBREV`, `_DOTTED`, `_CHUNKED`, `_PAIRED_INITIALS`): one
|
|
171
|
+
# letter (a word character that is not a digit or underscore) plus the
|
|
172
|
+
# combining marks written after it (#652). `inž.` typed as i, n, z,
|
|
173
|
+
# U+030C, '.' is the same word as the composed one, and `\w` alone ends
|
|
174
|
+
# it at the z. `_MARKS` is the general combining blocks -- what NFD of
|
|
175
|
+
# a Latin, Greek or Cyrillic letter produces -- and not the marks that
|
|
176
|
+
# only some scripts write (Indic vowel signs, Arabic and Hebrew
|
|
177
|
+
# points); rules.md#H2's Accepted clause owns that limit.
|
|
178
|
+
_MARKS = '[\u0300-\u036f\u1ab0-\u1aff\u1dc0-\u1dff\u20d0-\u20ff\ufe20-\ufe2f]'
|
|
179
|
+
_LETTER = r'[^\W\d_]' + _MARKS + '*'
|
|
180
|
+
_PERIOD_ABBREV = re.compile(r'^(?:' + _LETTER + r'){2,}\.$')
|
|
168
181
|
|
|
169
182
|
|
|
170
183
|
# rules.md#H2: "an abbreviation opening the part of the name that
|
|
@@ -363,13 +376,13 @@ def ambiguous_lean(text: str, one_case: bool) -> Lean | None:
|
|
|
363
376
|
return "name"
|
|
364
377
|
|
|
365
378
|
|
|
366
|
-
_DOTTED = re.compile(r"(?:
|
|
379
|
+
_DOTTED = re.compile(r"(?:" + _LETTER + r"\.)+")
|
|
367
380
|
# #544: the CHUNKED spelling -- two or more runs of letters, each
|
|
368
381
|
# closed by a period ('M.Eng.', 'L.Ac.'). Both callers compare the
|
|
369
382
|
# period-free letters against the listed member, so the chunks must
|
|
370
383
|
# spell it; one chunk ('Ma.', 'Ed.') is the single trailing period and
|
|
371
384
|
# stays outside the gate.
|
|
372
|
-
_CHUNKED = re.compile(r"(?:
|
|
385
|
+
_CHUNKED = re.compile(r"(?:(?:" + _LETTER + r")+\.){2,}")
|
|
373
386
|
|
|
374
387
|
|
|
375
388
|
def _dotted(text: str) -> bool:
|
|
@@ -389,7 +402,7 @@ def _dotted(text: str) -> bool:
|
|
|
389
402
|
# a given and a middle initial are written run together ('M.J.',
|
|
390
403
|
# 'G.J.'). The trailing period is optional, as `period_joined_vocab`'s
|
|
391
404
|
# shape verdict lets it be.
|
|
392
|
-
_PAIRED_INITIALS = re.compile(r"
|
|
405
|
+
_PAIRED_INITIALS = re.compile(_LETTER + r"\." + _LETTER + r"\.?")
|
|
393
406
|
|
|
394
407
|
|
|
395
408
|
def is_paired_initials(text: str) -> bool:
|
|
@@ -651,13 +664,35 @@ def caps_shape_candidate(text: str, lexicon: Lexicon, policy: Policy,
|
|
|
651
664
|
AMBIGUOUS_ACRONYM_TAG test), so a listed member never pays for
|
|
652
665
|
the call.
|
|
653
666
|
"""
|
|
654
|
-
if
|
|
655
|
-
|
|
656
|
-
|
|
667
|
+
if (policy.unlisted_caps_suffixes is CapsSuffixes.OFF
|
|
668
|
+
or one_case is not False):
|
|
669
|
+
return False
|
|
670
|
+
if not text.isascii():
|
|
671
|
+
text = unicodedata.normalize("NFC", text)
|
|
672
|
+
if not (len(text) >= 2 and text.isalpha() and text.isupper()):
|
|
657
673
|
return False
|
|
658
674
|
return not in_any_wordlist(_normalize(text), lexicon)
|
|
659
675
|
|
|
660
676
|
|
|
677
|
+
def is_caps_word(text: str) -> bool:
|
|
678
|
+
"""Whether TEXT is alphabetic capitals once NFC-composed: `isalpha()`
|
|
679
|
+
is False for a combining mark, so the decomposed spelling of
|
|
680
|
+
'ŽŠČ' would otherwise not be a caps word (#652). Callers test the
|
|
681
|
+
text as written inline first, in C, and call this only for text
|
|
682
|
+
that is not NFC (`unicodedata.is_normalized`, also C), so composed
|
|
683
|
+
text pays no frame."""
|
|
684
|
+
composed = unicodedata.normalize("NFC", text)
|
|
685
|
+
return composed.isalpha() and composed.isupper()
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
def composed_len(text: str) -> int:
|
|
689
|
+
"""The length of TEXT in letters as the writer sees them: code
|
|
690
|
+
points once NFC-composed. A caps word admitted decomposed (#652)
|
|
691
|
+
is longer than its composed twin, and the two-letter pair test
|
|
692
|
+
(`John Smith, MÁ`) counts letters."""
|
|
693
|
+
return len(text if text.isascii() else unicodedata.normalize("NFC", text))
|
|
694
|
+
|
|
695
|
+
|
|
661
696
|
def in_any_wordlist(n: str, lexicon: Lexicon) -> bool:
|
|
662
697
|
"""Whether the folded word `n` is in ANY of the lexicon's wordlists
|
|
663
698
|
(`_lexicon._VOCAB_FIELDS`, the whole roster): the caps shape's
|
|
@@ -1167,6 +1202,33 @@ def maiden_marker_run(words: Sequence[str], markers: frozenset[str]) -> int:
|
|
|
1167
1202
|
return 0
|
|
1168
1203
|
|
|
1169
1204
|
|
|
1205
|
+
|
|
1206
|
+
# rules.md#M3: "a marker followed by a comma is no marker"
|
|
1207
|
+
def comma_follows_marker(word: str) -> bool:
|
|
1208
|
+
"""Whether `word`, the one standing after a marker run inside a
|
|
1209
|
+
delimited clause, opens with a comma -- standing alone (' , ') or
|
|
1210
|
+
glued to the name after it (' ,Jones'). A comma glued to the marker
|
|
1211
|
+
itself ('née,') needs no test: that word is no marker to begin with,
|
|
1212
|
+
the vocabulary storing no comma. The one predicate for extract's M3
|
|
1213
|
+
test over whitespace words and group's M1 drop over tokens, which
|
|
1214
|
+
since #653 hold the same comma
|
|
1215
|
+
(mechanisms.md#ONE-PREDICATE-PER-QUESTION).
|
|
1216
|
+
|
|
1217
|
+
The comma need not be the word's first character, only the first
|
|
1218
|
+
that is a comma or a letter or digit. A whitespace word can open
|
|
1219
|
+
with what the tokenizer drops or divides at -- a bidi mark, an
|
|
1220
|
+
emoji, a nakaguro -- so '\u200f, Jones' is a comma-led WORD only
|
|
1221
|
+
past its mark, while the token group asks about is ',' already;
|
|
1222
|
+
reading past the rest makes the two sites agree by construction
|
|
1223
|
+
(#653's second review)."""
|
|
1224
|
+
for ch in word:
|
|
1225
|
+
if ch in COMMA_CHARS:
|
|
1226
|
+
return True
|
|
1227
|
+
if ch.isalnum():
|
|
1228
|
+
return False
|
|
1229
|
+
return False
|
|
1230
|
+
|
|
1231
|
+
|
|
1170
1232
|
def tag_marker_runs(tokens: Sequence[WorkToken], comma_offsets: Sequence[int],
|
|
1171
1233
|
markers: frozenset[str],
|
|
1172
1234
|
folded: Sequence[str] | None = None) -> dict[int, str]:
|
|
@@ -4,16 +4,16 @@
|
|
|
4
4
|
#:
|
|
5
5
|
#: Bumped when a cycle OPENS, not when it ships, so the tree says what
|
|
6
6
|
#: it is building. Note the cost of the tuple being purely numeric: it
|
|
7
|
-
#: cannot carry the dev marker, so `VERSION >= (2,
|
|
8
|
-
#: true here while 2.
|
|
7
|
+
#: cannot carry the dev marker, so `VERSION >= (2, 5, 0)` is already
|
|
8
|
+
#: true here while 2.5.0 is unreleased. Compare `__version__` instead
|
|
9
9
|
#: where that distinction matters.
|
|
10
|
-
VERSION = (2, 4,
|
|
10
|
+
VERSION = (2, 4, 1)
|
|
11
11
|
#: PEP 440 pre-release/dev segment appended to the numeric version, or
|
|
12
12
|
#: "" for a final release. Joined WITHOUT a dot ("2.0.0rc1", not
|
|
13
13
|
#: "2.0.0.rc1"); setuptools reads __version__ as the package version.
|
|
14
14
|
#:
|
|
15
15
|
#: "dev" through the cycle, cleared at release. It normalizes to
|
|
16
|
-
#: 2.
|
|
16
|
+
#: 2.5.0.dev0, which sorts above 2.4.0 and BELOW 2.5.0, so an install
|
|
17
17
|
#: from master can never masquerade as the release it precedes.
|
|
18
18
|
PRE_RELEASE = ""
|
|
19
19
|
__version__ = ".".join(map(str, VERSION)) + PRE_RELEASE
|
|
@@ -30,9 +30,8 @@ asserting things with the parser's own fold makes the constant's
|
|
|
30
30
|
hygiene depend on the parser, and the question worth asking is not
|
|
31
31
|
"is this entry pre-folded" but "does ``Lexicon`` store it unchanged".
|
|
32
32
|
That is one assertion over ``_PHRASE_FIELDS`` in
|
|
33
|
-
``tests/v2/
|
|
34
|
-
|
|
35
|
-
written ``'z. domu'`` is caught.
|
|
33
|
+
``tests/v2/test_lexicon.py``, and it is where an entry written
|
|
34
|
+
``'z. domu'`` is caught.
|
|
36
35
|
"""
|
|
37
36
|
from __future__ import annotations
|
|
38
37
|
|
|
@@ -15,6 +15,14 @@ re_emoji = re.compile(
|
|
|
15
15
|
# as nothing but stick to name parts and break equality/lookups.
|
|
16
16
|
re_bidi = re.compile('[\u061C\u200E\u200F\u202A-\u202E\u2066-\u2069]+')
|
|
17
17
|
|
|
18
|
+
# Combining marks (Unicode Mn) are not \w, so a letter written as a base
|
|
19
|
+
# plus a mark (NFD `inž.`) must carry them explicitly or the shape
|
|
20
|
+
# rules read a decomposed name differently from its composed twin
|
|
21
|
+
# (#652). Same class as _pipeline/_vocab.py's _MARKS; test_regex_sync
|
|
22
|
+
# compares the two class strings and pins `period_abbreviation`
|
|
23
|
+
# against `_vocab`'s copy.
|
|
24
|
+
_MARKS = '[\u0300-\u036f\u1ab0-\u1aff\u1dc0-\u1dff\u20d0-\u20ff\ufe20-\ufe2f]'
|
|
25
|
+
|
|
18
26
|
EMPTY_REGEX = re.compile('')
|
|
19
27
|
|
|
20
28
|
REGEXES = {
|
|
@@ -52,7 +60,8 @@ REGEXES = {
|
|
|
52
60
|
r'^(оглу|оглы|оғлу|ўғли|угли|кызы|гызы|қызы|қизи|улы|ұлы|уулу)$',
|
|
53
61
|
re.I,
|
|
54
62
|
),
|
|
55
|
-
"period_abbreviation": re.compile(
|
|
63
|
+
"period_abbreviation": re.compile(
|
|
64
|
+
r'^(?:[^\W\d_]' + _MARKS + r'*){2,}\.$'),
|
|
56
65
|
}
|
|
57
66
|
"""
|
|
58
67
|
All regular expressions used by the parser are precompiled and stored in the config.
|
|
@@ -83,20 +83,23 @@ tests/v2/conftest.py
|
|
|
83
83
|
tests/v2/rules_doc.py
|
|
84
84
|
tests/v2/test_benchmark.py
|
|
85
85
|
tests/v2/test_cases.py
|
|
86
|
+
tests/v2/test_cldr_agreement.py
|
|
87
|
+
tests/v2/test_cldr_corpus.py
|
|
86
88
|
tests/v2/test_cli.py
|
|
87
89
|
tests/v2/test_config_aliases.py
|
|
88
90
|
tests/v2/test_config_shim.py
|
|
89
91
|
tests/v2/test_contracts.py
|
|
92
|
+
tests/v2/test_corpora.py
|
|
90
93
|
tests/v2/test_differential.py
|
|
91
94
|
tests/v2/test_doc_citations.py
|
|
92
95
|
tests/v2/test_doc_spellings.py
|
|
93
96
|
tests/v2/test_facade.py
|
|
94
97
|
tests/v2/test_facade_cases.py
|
|
95
98
|
tests/v2/test_layering.py
|
|
96
|
-
tests/v2/test_ledger_guards.py
|
|
97
99
|
tests/v2/test_lexicon.py
|
|
98
100
|
tests/v2/test_locale.py
|
|
99
101
|
tests/v2/test_locales.py
|
|
102
|
+
tests/v2/test_name_usage_tool.py
|
|
100
103
|
tests/v2/test_order_correspondence.py
|
|
101
104
|
tests/v2/test_parser.py
|
|
102
105
|
tests/v2/test_policy.py
|
|
@@ -106,6 +109,7 @@ tests/v2/test_render.py
|
|
|
106
109
|
tests/v2/test_reprs.py
|
|
107
110
|
tests/v2/test_rules_doc.py
|
|
108
111
|
tests/v2/test_rules_doc_grammar.py
|
|
112
|
+
tests/v2/test_snapshot.py
|
|
109
113
|
tests/v2/test_types.py
|
|
110
114
|
tests/v2/pipeline/__init__.py
|
|
111
115
|
tests/v2/pipeline/test_assemble.py
|
|
@@ -292,22 +292,9 @@ class InitialsTestCase(HumanNameTestBase):
|
|
|
292
292
|
# 'y' outside the marked set is still the connective.
|
|
293
293
|
hn = HumanName("maria y lopez")
|
|
294
294
|
self.m(hn.initials(), "m. y. l.", hn)
|
|
295
|
-
# These two are among #528's four movers,
|
|
296
|
-
#
|
|
297
|
-
#
|
|
298
|
-
# names by itself -- but only "john e jones" also has a row in
|
|
299
|
-
# _CROSS_RULE_WINNERS (tests/v2/test_ledger_guards.py) and
|
|
300
|
-
# _RECORDED_DIFFS (tools/differential/compare.py): those two
|
|
301
|
-
# Python dicts, keyed by the LEDGER FILENAME rather than
|
|
302
|
-
# anything stored in the TOML, record that
|
|
303
|
-
# "fix(initials-per-word) a connective run initials each word"
|
|
304
|
-
# also reaches "john e jones" through its lowercase ' e ' and
|
|
305
|
-
# contests the name with #528's rule. "jones, john e" has no
|
|
306
|
-
# row in either: that connective-run rule's operative half is a
|
|
307
|
-
# `(?=\s)` lookahead requiring whitespace AFTER the letter, and
|
|
308
|
-
# a letter at the end of the string can never satisfy it -- so
|
|
309
|
-
# the comma form is a distinct shape from the space-written
|
|
310
|
-
# form even though both give the same initials() answer.
|
|
295
|
+
# These two are among #528's four movers, written once with
|
|
296
|
+
# the letter mid-name and once at the end after a comma: both
|
|
297
|
+
# give the same initials() answer by different routes.
|
|
311
298
|
hn = HumanName("john e jones")
|
|
312
299
|
self.m(hn.initials(), "j. e. j.", hn)
|
|
313
300
|
hn = HumanName("jones, john e")
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""Shared handles on tools/differential/: its corpora and its modules.
|
|
2
|
+
|
|
3
|
+
Neither is importable the ordinary way: `tools/` is outside
|
|
4
|
+
`testpaths` and is not a package, so its modules are loaded by path,
|
|
5
|
+
and the corpora are data files.
|
|
6
|
+
|
|
7
|
+
Not a conftest: these are constants and plain helpers, not fixtures in
|
|
8
|
+
pytest's sense, and several modules want them by name.
|
|
9
|
+
"""
|
|
10
|
+
import importlib.util
|
|
11
|
+
import json
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from types import ModuleType
|
|
14
|
+
|
|
15
|
+
_TOOLS = Path(__file__).parents[2] / "tools" / "differential"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _entry_name(raw: object) -> str:
|
|
19
|
+
"""A corpus line's name, whichever format the line uses: a bare
|
|
20
|
+
JSON string or an object carrying "name"."""
|
|
21
|
+
if isinstance(raw, str):
|
|
22
|
+
return raw
|
|
23
|
+
assert isinstance(raw, dict) and isinstance(raw.get("name"), str), raw
|
|
24
|
+
return raw["name"]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
#: Corpus files _CORPUS_NAMES leaves out. corpus_cldr.jsonl
|
|
28
|
+
#: (2026-10-12): about 1,900 fictional CLDR renderings whose ground
|
|
29
|
+
#: truth the agreement ratchet in test_cldr_agreement.py enforces. The
|
|
30
|
+
#: tests reading _CORPUS_NAMES carry counts measured over the names the
|
|
31
|
+
#: corpora held before it arrived.
|
|
32
|
+
_UNMEASURED_CORPORA = frozenset({"corpus_cldr.jsonl"})
|
|
33
|
+
|
|
34
|
+
#: Every corpus name, deduplicated, except the files in
|
|
35
|
+
#: _UNMEASURED_CORPORA.
|
|
36
|
+
_CORPUS_NAMES = sorted({
|
|
37
|
+
_entry_name(json.loads(line))
|
|
38
|
+
for path in sorted(_TOOLS.glob("corpus*.jsonl"))
|
|
39
|
+
if path.name not in _UNMEASURED_CORPORA
|
|
40
|
+
for line in path.read_text(encoding="utf-8").splitlines() if line.strip()})
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def load_tool(stem: str) -> ModuleType:
|
|
44
|
+
"""A module from tools/differential/, loaded by path.
|
|
45
|
+
|
|
46
|
+
`tools/` is outside testpaths, and adding it would run
|
|
47
|
+
--doctest-modules over the corpus builders, so these are imported
|
|
48
|
+
this way rather than made importable. None has import-time side
|
|
49
|
+
effects: each main() is behind a __name__ guard."""
|
|
50
|
+
spec = importlib.util.spec_from_file_location(
|
|
51
|
+
f"differential_{stem}", _TOOLS / f"{stem}.py")
|
|
52
|
+
assert spec is not None and spec.loader is not None, stem
|
|
53
|
+
module = importlib.util.module_from_spec(spec)
|
|
54
|
+
spec.loader.exec_module(module)
|
|
55
|
+
return module
|