nameparser 2.4.0__tar.gz → 2.4.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. {nameparser-2.4.0/nameparser.egg-info → nameparser-2.4.1}/PKG-INFO +1 -1
  2. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_facade.py +2 -2
  3. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/_classify.py +3 -1
  4. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/_comma.py +11 -6
  5. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/_extract.py +13 -7
  6. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/_group.py +5 -2
  7. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/_script_segment.py +2 -2
  8. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/_tokenize.py +10 -4
  9. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/_vocab.py +75 -13
  10. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_version.py +4 -4
  11. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/_invariants.py +2 -3
  12. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/regexes.py +10 -1
  13. {nameparser-2.4.0 → nameparser-2.4.1/nameparser.egg-info}/PKG-INFO +1 -1
  14. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser.egg-info/SOURCES.txt +5 -1
  15. {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_initials.py +3 -16
  16. nameparser-2.4.1/tests/v2/_differential_fixtures.py +55 -0
  17. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/cases.py +108 -109
  18. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/conftest.py +14 -0
  19. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/test_group.py +1 -1
  20. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/test_tokenize.py +4 -2
  21. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/test_vocab.py +14 -1
  22. nameparser-2.4.1/tests/v2/test_cldr_agreement.py +208 -0
  23. nameparser-2.4.1/tests/v2/test_cldr_corpus.py +118 -0
  24. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_config_aliases.py +1 -2
  25. nameparser-2.4.1/tests/v2/test_corpora.py +190 -0
  26. nameparser-2.4.1/tests/v2/test_differential.py +454 -0
  27. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_doc_citations.py +12 -16
  28. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_lexicon.py +23 -0
  29. nameparser-2.4.1/tests/v2/test_name_usage_tool.py +303 -0
  30. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_parser.py +111 -1
  31. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_regex_sync.py +5 -5
  32. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_render.py +1 -2
  33. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_rules_doc.py +11 -0
  34. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_rules_doc_grammar.py +2 -6
  35. nameparser-2.4.1/tests/v2/test_snapshot.py +48 -0
  36. nameparser-2.4.0/tests/v2/_differential_fixtures.py +0 -120
  37. nameparser-2.4.0/tests/v2/test_differential.py +0 -4792
  38. nameparser-2.4.0/tests/v2/test_ledger_guards.py +0 -8902
  39. {nameparser-2.4.0 → nameparser-2.4.1}/AUTHORS +0 -0
  40. {nameparser-2.4.0 → nameparser-2.4.1}/LICENSE +0 -0
  41. {nameparser-2.4.0 → nameparser-2.4.1}/MANIFEST.in +0 -0
  42. {nameparser-2.4.0 → nameparser-2.4.1}/README.rst +0 -0
  43. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/__init__.py +0 -0
  44. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/__main__.py +0 -0
  45. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_config_shim.py +0 -0
  46. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_lexicon.py +0 -0
  47. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_locale.py +0 -0
  48. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_parser.py +0 -0
  49. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/__init__.py +0 -0
  50. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/_assemble.py +0 -0
  51. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/_assign.py +0 -0
  52. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/_pieces.py +0 -0
  53. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/_post_rules.py +0 -0
  54. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/_segment.py +0 -0
  55. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_pipeline/_state.py +0 -0
  56. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_policy.py +0 -0
  57. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_render.py +0 -0
  58. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/_types.py +0 -0
  59. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/__init__.py +0 -0
  60. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/_deprecated.py +0 -0
  61. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/bound_first_names.py +0 -0
  62. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/bound_given_names.py +0 -0
  63. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/capitalization.py +0 -0
  64. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/conjunctions.py +0 -0
  65. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/maiden_markers.py +0 -0
  66. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/particles.py +0 -0
  67. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/prefixes.py +0 -0
  68. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/suffixes.py +0 -0
  69. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/surnames.py +0 -0
  70. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/config/titles.py +0 -0
  71. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/locales/__init__.py +0 -0
  72. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/locales/ja.py +0 -0
  73. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/locales/ru.py +0 -0
  74. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/locales/tr_az.py +0 -0
  75. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/locales/zh.py +0 -0
  76. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/parser.py +0 -0
  77. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/py.typed +0 -0
  78. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser/util.py +0 -0
  79. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser.egg-info/dependency_links.txt +0 -0
  80. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser.egg-info/requires.txt +0 -0
  81. {nameparser-2.4.0 → nameparser-2.4.1}/nameparser.egg-info/top_level.txt +0 -0
  82. {nameparser-2.4.0 → nameparser-2.4.1}/pyproject.toml +0 -0
  83. {nameparser-2.4.0 → nameparser-2.4.1}/setup.cfg +0 -0
  84. {nameparser-2.4.0 → nameparser-2.4.1}/tests/__init__.py +0 -0
  85. {nameparser-2.4.0 → nameparser-2.4.1}/tests/base.py +0 -0
  86. {nameparser-2.4.0 → nameparser-2.4.1}/tests/conftest.py +0 -0
  87. {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_bound_given_names.py +0 -0
  88. {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_brute_force.py +0 -0
  89. {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_capitalization.py +0 -0
  90. {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_comma_variants.py +0 -0
  91. {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_conjunctions.py +0 -0
  92. {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_constants.py +0 -0
  93. {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_east_slavic_patronymic_order.py +0 -0
  94. {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_first_name.py +0 -0
  95. {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_middle_name_as_last.py +0 -0
  96. {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_nicknames.py +0 -0
  97. {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_output_format.py +0 -0
  98. {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_particles.py +0 -0
  99. {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_python_api.py +0 -0
  100. {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_suffixes.py +0 -0
  101. {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_titles.py +0 -0
  102. {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_turkic_patronymic_order.py +0 -0
  103. {nameparser-2.4.0 → nameparser-2.4.1}/tests/test_variations.py +0 -0
  104. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/__init__.py +0 -0
  105. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/__init__.py +0 -0
  106. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/test_assemble.py +0 -0
  107. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/test_assign.py +0 -0
  108. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/test_classify.py +0 -0
  109. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/test_comma.py +0 -0
  110. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/test_extract.py +0 -0
  111. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/test_pieces.py +0 -0
  112. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/test_post_rules.py +0 -0
  113. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/test_script_segment.py +0 -0
  114. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/test_segment.py +0 -0
  115. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/pipeline/test_state.py +0 -0
  116. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/rules_doc.py +0 -0
  117. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_benchmark.py +0 -0
  118. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_cases.py +0 -0
  119. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_cli.py +0 -0
  120. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_config_shim.py +0 -0
  121. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_contracts.py +0 -0
  122. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_doc_spellings.py +0 -0
  123. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_facade.py +0 -0
  124. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_facade_cases.py +0 -0
  125. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_layering.py +0 -0
  126. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_locale.py +0 -0
  127. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_locales.py +0 -0
  128. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_order_correspondence.py +0 -0
  129. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_policy.py +0 -0
  130. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_properties.py +0 -0
  131. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_reprs.py +0 -0
  132. {nameparser-2.4.0 → nameparser-2.4.1}/tests/v2/test_types.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: nameparser
3
- Version: 2.4.0
3
+ Version: 2.4.1
4
4
  Summary: A simple Python module for parsing human names into their individual components.
5
5
  Author-email: Derek Gulbranson <derek73@gmail.com>
6
6
  License: LGPL
@@ -709,8 +709,8 @@ class HumanName:
709
709
  # which compare the seven role fields only (Case carries
710
710
  # no initials column); the covering test is
711
711
  # tests/test_initials.py::test_initials_middle_name_all_prefixes,
712
- # and since #484 the differential compares initials() on
713
- # both surfaces for names whose roles agree. _split_last
712
+ # and tools/differential/snapshot.jsonl records initials()
713
+ # on both surfaces for every corpus name. _split_last
714
714
  # already applies the same guard to the base, which is why
715
715
  # last_base was never empty here.
716
716
  return [_render._first_letter(w) for w in words]
@@ -52,6 +52,7 @@ is what lets the fork be reported without being taken.
52
52
  """
53
53
  from __future__ import annotations
54
54
 
55
+ import unicodedata
55
56
 
56
57
  from nameparser._lexicon import _normalize, _spells_an_initial
57
58
  from nameparser._policy import CapsSuffixes
@@ -229,7 +230,8 @@ def _tags_for(token: WorkToken, n: str, state: ParseState,
229
230
  # C: a listed member ('John Smith, MA'), and a word that
230
231
  # is not alphabetic capitals ('John Smith, Ph. D.')
231
232
  and n not in lex.suffix_acronyms_ambiguous
232
- and token.text.isalpha() and token.text.isupper()
233
+ and (token.text.isalpha() and token.text.isupper()
234
+ or not unicodedata.is_normalized("NFC", token.text))
233
235
  and caps_shape_candidate(token.text, lex, state.policy,
234
236
  one_case)):
235
237
  # #516's all-caps half: an unlisted word written in capitals
@@ -27,6 +27,7 @@ Policy.unlisted_caps_suffixes.
27
27
  """
28
28
  from __future__ import annotations
29
29
 
30
+ import unicodedata
30
31
  from collections.abc import Sequence
31
32
 
32
33
  from nameparser._lexicon import _normalize
@@ -40,8 +41,8 @@ from nameparser._pipeline._state import (
40
41
  )
41
42
  from nameparser._pipeline._vocab import (
42
43
  D, PH, caps_shape_candidate, claimed_as_non_name, delimiter_cores,
43
- is_paired_initials, is_single_letter_numeral, is_wholly_suffix,
44
- name_word_count, surname_unit_facts, unit_ends, written_as_a_name,
44
+ composed_len, is_caps_word, is_paired_initials, is_single_letter_numeral,
45
+ is_wholly_suffix, name_word_count, surname_unit_facts, unit_ends, written_as_a_name,
45
46
  )
46
47
  from nameparser._policy import CapsSuffixes
47
48
  from nameparser._types import AmbiguityKind, Role
@@ -217,8 +218,11 @@ def decide(state: ParseState) -> ParseState:
217
218
  maybe = [p[0] for p in p1 if len(p) == 1
218
219
  and (AMBIGUOUS_ACRONYM_TAG in (tags := tokens[p[0]].tags)
219
220
  or ("vocab:suffix" not in tags
220
- and tokens[p[0]].text.isalpha()
221
- and tokens[p[0]].text.isupper()))]
221
+ and (tokens[p[0]].text.isalpha()
222
+ and tokens[p[0]].text.isupper()
223
+ or not unicodedata.is_normalized(
224
+ "NFC", tokens[p[0]].text)
225
+ and is_caps_word(tokens[p[0]].text))))]
222
226
  # The part is read ONCE: here where it holds no word of the class,
223
227
  # after the count below where it does, with the words the count
224
228
  # licensed (a read here and a second after licensing threw the first
@@ -307,7 +311,7 @@ def decide(state: ParseState) -> ParseState:
307
311
  continue
308
312
  text = tokens[i].text
309
313
  if not (is_paired_initials(text)
310
- or (i in caps and len(text) == 2)):
314
+ or (i in caps and composed_len(text) == 2)):
311
315
  continue
312
316
  if speaker < 0:
313
317
  speaker = next(
@@ -375,7 +379,8 @@ def decide(state: ParseState) -> ParseState:
375
379
  member_set = frozenset(members)
376
380
  pairs = [k for k, p in enumerate(p1) if p[0] in member_set
377
381
  and (is_paired_initials(tokens[p[0]].text)
378
- or (p[0] in caps and len(tokens[p[0]].text) == 2))]
382
+ or (p[0] in caps
383
+ and composed_len(tokens[p[0]].text) == 2))]
379
384
  # spoken for by each other only: every class word a pair, and
380
385
  # only titles among the other words in front of the first
381
386
  pair_only = (len(pairs) >= 2 and len(pairs) == len(members)
@@ -38,7 +38,9 @@ from nameparser._lexicon import Lexicon, _normalize
38
38
  from nameparser._pipeline._state import (
39
39
  COMMA_CHARS, ParseState, PendingAmbiguity, copy_with,
40
40
  )
41
- from nameparser._pipeline._vocab import maiden_marker_run
41
+ from nameparser._pipeline._vocab import (
42
+ comma_follows_marker, maiden_marker_run,
43
+ )
42
44
  from nameparser._types import AmbiguityKind, Role, Span
43
45
 
44
46
 
@@ -76,9 +78,12 @@ def _maiden_marked(content: str, lexicon: Lexicon) -> bool:
76
78
  itself rather than reading the tags classify will set. The two
77
79
  questions are deliberately not the same one: this splits on
78
80
  WHITESPACE, so a marker the writer glued to punctuation is not one
79
- here ('née,'); the tokenizer splits that comma off and classify
80
- still tags the token, which is what keeps _group's Role.MAIDEN
81
- filter reachable. Sharing the PREDICATE does not merge the two
81
+ here ('née,') -- nor to classify, since #653 keeps a clause's comma
82
+ on its word. A nakaguro still divides the two: the tokenizer splits
83
+ '旧姓・田中' at it (rules.md#T2) and classify tags '旧姓', which is
84
+ what keeps _group's Role.MAIDEN filter reachable (cases.py's
85
+ marker_divided_by_a_nakaguro_keeps_the_clause_a_nickname). Sharing
86
+ the PREDICATE does not merge the two
82
87
  questions -- each hands it a different sequence of words."""
83
88
  words = content.split()
84
89
  # A one-word clause cannot satisfy the condition whatever the
@@ -88,7 +93,8 @@ def _maiden_marked(content: str, lexicon: Lexicon) -> bool:
88
93
  if len(words) < 2:
89
94
  return False
90
95
  run = maiden_marker_run(words, lexicon.maiden_markers)
91
- return run > 0 and len(words) > run
96
+ return (run > 0 and len(words) > run
97
+ and not comma_follows_marker(words[run]))
92
98
 
93
99
 
94
100
  # rules.md#N2: "a quote whose open and close are the same character
@@ -164,8 +170,8 @@ def _unmatched(open_: str, offset: int) -> tuple[int, PendingAmbiguity]:
164
170
 
165
171
 
166
172
  # rules.md#N1: "a clause enclosed by a configured nickname delimiter
167
- # pair reads as the nickname and is lifted out of the name; an empty
168
- # enclosure is simply dropped"
173
+ # pair reads as the nickname and is lifted out of the name" (its
174
+ # punctuation is tokenize's half of the rule)
169
175
  # rules.md#M1: "with a delimiter pair configured for maiden names, its
170
176
  # enclosed clause reads as the maiden name" (history: decisions.md#M1)
171
177
  def extract_delimited(state: ParseState) -> ParseState:
@@ -78,7 +78,7 @@ from nameparser._pipeline._state import (
78
78
  import nameparser._pipeline._comma as _comma
79
79
  from nameparser._pipeline._vocab import D, PH
80
80
  from nameparser._pipeline._vocab import (
81
- delimiter_cores,
81
+ comma_follows_marker, delimiter_cores,
82
82
  )
83
83
  from nameparser._types import AmbiguityKind, Role
84
84
 
@@ -1523,8 +1523,11 @@ def group(state: ParseState) -> ParseState:
1523
1523
  # test, since the tokens inside a clause are contiguous in
1524
1524
  # index order -- a clause holding nothing but its marker
1525
1525
  # keeps its words, exactly as a one-token "(Nee)" does.
1526
+ # rules.md#M3: "a marker followed by a comma is no marker"
1527
+ # -- asked of the same word extract's M3 test asks it of
1526
1528
  if (first + run < len(tokens)
1527
- and tokens[first + run].span.end <= clause.end):
1529
+ and tokens[first + run].span.end <= clause.end
1530
+ and not comma_follows_marker(tokens[first + run].text)):
1528
1531
  dropped.extend(range(first, first + run))
1529
1532
  return copy_with(
1530
1533
  state, tokens=tuple(tokens), pieces=tuple(all_pieces),
@@ -255,8 +255,8 @@ def _is_post_nominal(state: ParseState, i: int) -> bool:
255
255
  # classification rather than cut into 박사 + 님, which is what the
256
256
  # same input gives when the step is removed. '선생님' is post-nominal
257
257
  # entire with no name word behind it, so the scan yields no site at
258
- # all and the token stays whole. Both are contract-tier corpus names
259
- # and rules.md#W2 example lines; decisions.md#cjk-comma-demotion
258
+ # all and the token stays whole. Both are corpus names and
259
+ # rules.md#W2 example lines; decisions.md#cjk-comma-demotion
260
260
  # carries the forced-predicate measurement behind them.
261
261
  def _peel_site(state: ParseState, flat: Sequence[int],
262
262
  tails: frozenset[str]) -> tuple[int, int] | None:
@@ -3,7 +3,8 @@
3
3
  Consumes: original, masked (regions to skip), extracted (regions that
4
4
  tokenize with a pre-set role).
5
5
  Produces: tokens (span-sorted WorkTokens; text always == original
6
- slice), comma_offsets (segmentation points; never tokens),
6
+ slice), comma_offsets (segmentation points of the main stream; a comma
7
+ there is never a token, one inside an extracted region is word text),
7
8
  interpunct_offsets (间隔号 transcription markers, #298; never tokens).
8
9
  Reads: Policy.strip_emoji, Policy.strip_bidi.
9
10
 
@@ -110,7 +111,12 @@ def _tokenize_region(state: ParseState, start: int, end: int,
110
111
  tok_start: int | None = None
111
112
  for i in range(start, end):
112
113
  ch = text[i]
113
- is_separator = ch in COMMA_CHARS or _ignorable(ch, state)
114
+ # rules.md#N1: "a comma inside it kept as written — the comma
115
+ # divides nothing there" -- a comma separates only where it
116
+ # records a segmentation offset, the main stream; inside an
117
+ # extracted region it is the clause's text, like ';' (#653)
118
+ is_separator = ((record_offsets and ch in COMMA_CHARS)
119
+ or _ignorable(ch, state))
114
120
  if ch == _INTERPUNCT:
115
121
  # Region-local flanks: a B7 at a region edge stays token
116
122
  # text, and the bound also stops a custom delimiter's
@@ -155,8 +161,8 @@ def tokenize(state: ParseState) -> ParseState:
155
161
  for start, end in zip(boundaries[::2], boundaries[1::2]):
156
162
  _tokenize_region(state, start, end, None, True, tokens, commas,
157
163
  interpuncts)
158
- # extracted regions: pre-set role, commas and interpuncts are mere
159
- # separators
164
+ # extracted regions: pre-set role; a comma is word text there and an
165
+ # interpunct a mere separator, neither recording an offset
160
166
  for role, inner in state.extracted:
161
167
  _tokenize_region(state, inner.start, inner.end, role, False,
162
168
  tokens, commas, interpuncts)
@@ -51,7 +51,7 @@ from nameparser._lexicon import (
51
51
  )
52
52
  from nameparser._policy import (CapsSuffixes, Policy, Script, _JA_SCRIPTS, _NO_INITIALS,
53
53
  _SCRIPT_RANGES, _script_matcher)
54
- from nameparser._pipeline._state import WorkToken, comma_bucket
54
+ from nameparser._pipeline._state import COMMA_CHARS, WorkToken, comma_bucket
55
55
 
56
56
  # Ported verbatim from v1 (nameparser/config/regexes.py "initial") minus
57
57
  # its empty-string alternative -- WorkToken text is never empty. Kept in
@@ -159,12 +159,25 @@ in_initialless_script = _script_matcher(*_NO_INITIALS, whole=False)
159
159
  # the comma structure to a credential run
160
160
  # (mechanisms.md#ONE-PREDICATE-PER-QUESTION).
161
161
  #
162
- # Ported verbatim from v1 (nameparser/config/regexes.py
163
- # "period_abbreviation") -- layering forbids the config import; keep
164
- # in sync by hand (tests/v2/test_regex_sync.py, which reaches this
165
- # object through `_pieces._PERIOD_ABBREV`, an IMPORT of this one, not
166
- # a second definition -- the sync test's target name did not move).
167
- _PERIOD_ABBREV = re.compile(r'^[^\W\d_]{2,}\.$')
162
+ # The shape shared with config's "period_abbreviation" (layering
163
+ # forbids the config import; keep in sync by hand -- the pattern TEXT
164
+ # is what tests/v2/test_regex_sync.py compares, reaching this object
165
+ # through `_pieces._PERIOD_ABBREV`, an IMPORT of this one, not a second
166
+ # definition -- the sync test's target name did not move). It is
167
+ # composed from `_LETTER` below, so the text is v1's plus the marks.
168
+
169
+ # `_LETTER` is the letter every shape regex in this module spells
170
+ # (`_PERIOD_ABBREV`, `_DOTTED`, `_CHUNKED`, `_PAIRED_INITIALS`): one
171
+ # letter (a word character that is not a digit or underscore) plus the
172
+ # combining marks written after it (#652). `inž.` typed as i, n, z,
173
+ # U+030C, '.' is the same word as the composed one, and `\w` alone ends
174
+ # it at the z. `_MARKS` is the general combining blocks -- what NFD of
175
+ # a Latin, Greek or Cyrillic letter produces -- and not the marks that
176
+ # only some scripts write (Indic vowel signs, Arabic and Hebrew
177
+ # points); rules.md#H2's Accepted clause owns that limit.
178
+ _MARKS = '[\u0300-\u036f\u1ab0-\u1aff\u1dc0-\u1dff\u20d0-\u20ff\ufe20-\ufe2f]'
179
+ _LETTER = r'[^\W\d_]' + _MARKS + '*'
180
+ _PERIOD_ABBREV = re.compile(r'^(?:' + _LETTER + r'){2,}\.$')
168
181
 
169
182
 
170
183
  # rules.md#H2: "an abbreviation opening the part of the name that
@@ -363,13 +376,13 @@ def ambiguous_lean(text: str, one_case: bool) -> Lean | None:
363
376
  return "name"
364
377
 
365
378
 
366
- _DOTTED = re.compile(r"(?:[^\W\d_]\.)+")
379
+ _DOTTED = re.compile(r"(?:" + _LETTER + r"\.)+")
367
380
  # #544: the CHUNKED spelling -- two or more runs of letters, each
368
381
  # closed by a period ('M.Eng.', 'L.Ac.'). Both callers compare the
369
382
  # period-free letters against the listed member, so the chunks must
370
383
  # spell it; one chunk ('Ma.', 'Ed.') is the single trailing period and
371
384
  # stays outside the gate.
372
- _CHUNKED = re.compile(r"(?:[^\W\d_]+\.){2,}")
385
+ _CHUNKED = re.compile(r"(?:(?:" + _LETTER + r")+\.){2,}")
373
386
 
374
387
 
375
388
  def _dotted(text: str) -> bool:
@@ -389,7 +402,7 @@ def _dotted(text: str) -> bool:
389
402
  # a given and a middle initial are written run together ('M.J.',
390
403
  # 'G.J.'). The trailing period is optional, as `period_joined_vocab`'s
391
404
  # shape verdict lets it be.
392
- _PAIRED_INITIALS = re.compile(r"[^\W\d_]\.[^\W\d_]\.?")
405
+ _PAIRED_INITIALS = re.compile(_LETTER + r"\." + _LETTER + r"\.?")
393
406
 
394
407
 
395
408
  def is_paired_initials(text: str) -> bool:
@@ -651,13 +664,35 @@ def caps_shape_candidate(text: str, lexicon: Lexicon, policy: Policy,
651
664
  AMBIGUOUS_ACRONYM_TAG test), so a listed member never pays for
652
665
  the call.
653
666
  """
654
- if not (policy.unlisted_caps_suffixes is not CapsSuffixes.OFF
655
- and one_case is False
656
- and len(text) >= 2 and text.isalpha() and text.isupper()):
667
+ if (policy.unlisted_caps_suffixes is CapsSuffixes.OFF
668
+ or one_case is not False):
669
+ return False
670
+ if not text.isascii():
671
+ text = unicodedata.normalize("NFC", text)
672
+ if not (len(text) >= 2 and text.isalpha() and text.isupper()):
657
673
  return False
658
674
  return not in_any_wordlist(_normalize(text), lexicon)
659
675
 
660
676
 
677
+ def is_caps_word(text: str) -> bool:
678
+ """Whether TEXT is alphabetic capitals once NFC-composed: `isalpha()`
679
+ is False for a combining mark, so the decomposed spelling of
680
+ 'ŽŠČ' would otherwise not be a caps word (#652). Callers test the
681
+ text as written inline first, in C, and call this only for text
682
+ that is not NFC (`unicodedata.is_normalized`, also C), so composed
683
+ text pays no frame."""
684
+ composed = unicodedata.normalize("NFC", text)
685
+ return composed.isalpha() and composed.isupper()
686
+
687
+
688
+ def composed_len(text: str) -> int:
689
+ """The length of TEXT in letters as the writer sees them: code
690
+ points once NFC-composed. A caps word admitted decomposed (#652)
691
+ is longer than its composed twin, and the two-letter pair test
692
+ (`John Smith, MÁ`) counts letters."""
693
+ return len(text if text.isascii() else unicodedata.normalize("NFC", text))
694
+
695
+
661
696
  def in_any_wordlist(n: str, lexicon: Lexicon) -> bool:
662
697
  """Whether the folded word `n` is in ANY of the lexicon's wordlists
663
698
  (`_lexicon._VOCAB_FIELDS`, the whole roster): the caps shape's
@@ -1167,6 +1202,33 @@ def maiden_marker_run(words: Sequence[str], markers: frozenset[str]) -> int:
1167
1202
  return 0
1168
1203
 
1169
1204
 
1205
+
1206
+ # rules.md#M3: "a marker followed by a comma is no marker"
1207
+ def comma_follows_marker(word: str) -> bool:
1208
+ """Whether `word`, the one standing after a marker run inside a
1209
+ delimited clause, opens with a comma -- standing alone (' , ') or
1210
+ glued to the name after it (' ,Jones'). A comma glued to the marker
1211
+ itself ('née,') needs no test: that word is no marker to begin with,
1212
+ the vocabulary storing no comma. The one predicate for extract's M3
1213
+ test over whitespace words and group's M1 drop over tokens, which
1214
+ since #653 hold the same comma
1215
+ (mechanisms.md#ONE-PREDICATE-PER-QUESTION).
1216
+
1217
+ The comma need not be the word's first character, only the first
1218
+ that is a comma or a letter or digit. A whitespace word can open
1219
+ with what the tokenizer drops or divides at -- a bidi mark, an
1220
+ emoji, a nakaguro -- so '\u200f, Jones' is a comma-led WORD only
1221
+ past its mark, while the token group asks about is ',' already;
1222
+ reading past the rest makes the two sites agree by construction
1223
+ (#653's second review)."""
1224
+ for ch in word:
1225
+ if ch in COMMA_CHARS:
1226
+ return True
1227
+ if ch.isalnum():
1228
+ return False
1229
+ return False
1230
+
1231
+
1170
1232
  def tag_marker_runs(tokens: Sequence[WorkToken], comma_offsets: Sequence[int],
1171
1233
  markers: frozenset[str],
1172
1234
  folded: Sequence[str] | None = None) -> dict[int, str]:
@@ -4,16 +4,16 @@
4
4
  #:
5
5
  #: Bumped when a cycle OPENS, not when it ships, so the tree says what
6
6
  #: it is building. Note the cost of the tuple being purely numeric: it
7
- #: cannot carry the dev marker, so `VERSION >= (2, 4, 0)` is already
8
- #: true here while 2.4.0 is unreleased. Compare `__version__` instead
7
+ #: cannot carry the dev marker, so `VERSION >= (2, 5, 0)` is already
8
+ #: true here while 2.5.0 is unreleased. Compare `__version__` instead
9
9
  #: where that distinction matters.
10
- VERSION = (2, 4, 0)
10
+ VERSION = (2, 4, 1)
11
11
  #: PEP 440 pre-release/dev segment appended to the numeric version, or
12
12
  #: "" for a final release. Joined WITHOUT a dot ("2.0.0rc1", not
13
13
  #: "2.0.0.rc1"); setuptools reads __version__ as the package version.
14
14
  #:
15
15
  #: "dev" through the cycle, cleared at release. It normalizes to
16
- #: 2.4.0.dev0, which sorts above 2.3.0 and BELOW 2.4.0, so an install
16
+ #: 2.5.0.dev0, which sorts above 2.4.0 and BELOW 2.5.0, so an install
17
17
  #: from master can never masquerade as the release it precedes.
18
18
  PRE_RELEASE = ""
19
19
  __version__ = ".".join(map(str, VERSION)) + PRE_RELEASE
@@ -30,9 +30,8 @@ asserting things with the parser's own fold makes the constant's
30
30
  hygiene depend on the parser, and the question worth asking is not
31
31
  "is this entry pre-folded" but "does ``Lexicon`` store it unchanged".
32
32
  That is one assertion over ``_PHRASE_FIELDS`` in
33
- ``tests/v2/test_ledger_guards.py``, the suite that already imports the
34
- parser's fold for exactly this purpose, and it is where an entry
35
- written ``'z. domu'`` is caught.
33
+ ``tests/v2/test_lexicon.py``, and it is where an entry written
34
+ ``'z. domu'`` is caught.
36
35
  """
37
36
  from __future__ import annotations
38
37
 
@@ -15,6 +15,14 @@ re_emoji = re.compile(
15
15
  # as nothing but stick to name parts and break equality/lookups.
16
16
  re_bidi = re.compile('[\u061C\u200E\u200F\u202A-\u202E\u2066-\u2069]+')
17
17
 
18
+ # Combining marks (Unicode Mn) are not \w, so a letter written as a base
19
+ # plus a mark (NFD `inž.`) must carry them explicitly or the shape
20
+ # rules read a decomposed name differently from its composed twin
21
+ # (#652). Same class as _pipeline/_vocab.py's _MARKS; test_regex_sync
22
+ # compares the two class strings and pins `period_abbreviation`
23
+ # against `_vocab`'s copy.
24
+ _MARKS = '[\u0300-\u036f\u1ab0-\u1aff\u1dc0-\u1dff\u20d0-\u20ff\ufe20-\ufe2f]'
25
+
18
26
  EMPTY_REGEX = re.compile('')
19
27
 
20
28
  REGEXES = {
@@ -52,7 +60,8 @@ REGEXES = {
52
60
  r'^(оглу|оглы|оғлу|ўғли|угли|кызы|гызы|қызы|қизи|улы|ұлы|уулу)$',
53
61
  re.I,
54
62
  ),
55
- "period_abbreviation": re.compile(r'^[^\W\d_]{2,}\.$'),
63
+ "period_abbreviation": re.compile(
64
+ r'^(?:[^\W\d_]' + _MARKS + r'*){2,}\.$'),
56
65
  }
57
66
  """
58
67
  All regular expressions used by the parser are precompiled and stored in the config.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: nameparser
3
- Version: 2.4.0
3
+ Version: 2.4.1
4
4
  Summary: A simple Python module for parsing human names into their individual components.
5
5
  Author-email: Derek Gulbranson <derek73@gmail.com>
6
6
  License: LGPL
@@ -83,20 +83,23 @@ tests/v2/conftest.py
83
83
  tests/v2/rules_doc.py
84
84
  tests/v2/test_benchmark.py
85
85
  tests/v2/test_cases.py
86
+ tests/v2/test_cldr_agreement.py
87
+ tests/v2/test_cldr_corpus.py
86
88
  tests/v2/test_cli.py
87
89
  tests/v2/test_config_aliases.py
88
90
  tests/v2/test_config_shim.py
89
91
  tests/v2/test_contracts.py
92
+ tests/v2/test_corpora.py
90
93
  tests/v2/test_differential.py
91
94
  tests/v2/test_doc_citations.py
92
95
  tests/v2/test_doc_spellings.py
93
96
  tests/v2/test_facade.py
94
97
  tests/v2/test_facade_cases.py
95
98
  tests/v2/test_layering.py
96
- tests/v2/test_ledger_guards.py
97
99
  tests/v2/test_lexicon.py
98
100
  tests/v2/test_locale.py
99
101
  tests/v2/test_locales.py
102
+ tests/v2/test_name_usage_tool.py
100
103
  tests/v2/test_order_correspondence.py
101
104
  tests/v2/test_parser.py
102
105
  tests/v2/test_policy.py
@@ -106,6 +109,7 @@ tests/v2/test_render.py
106
109
  tests/v2/test_reprs.py
107
110
  tests/v2/test_rules_doc.py
108
111
  tests/v2/test_rules_doc_grammar.py
112
+ tests/v2/test_snapshot.py
109
113
  tests/v2/test_types.py
110
114
  tests/v2/pipeline/__init__.py
111
115
  tests/v2/pipeline/test_assemble.py
@@ -292,22 +292,9 @@ class InitialsTestCase(HumanNameTestBase):
292
292
  # 'y' outside the marked set is still the connective.
293
293
  hn = HumanName("maria y lopez")
294
294
  self.m(hn.initials(), "m. y. l.", hn)
295
- # These two are among #528's four movers, and the #528 ledger
296
- # rule's own name_regex (a literal alternation, in
297
- # tools/differential/expected_since_1.4.0.toml) covers all four
298
- # names by itself -- but only "john e jones" also has a row in
299
- # _CROSS_RULE_WINNERS (tests/v2/test_ledger_guards.py) and
300
- # _RECORDED_DIFFS (tools/differential/compare.py): those two
301
- # Python dicts, keyed by the LEDGER FILENAME rather than
302
- # anything stored in the TOML, record that
303
- # "fix(initials-per-word) a connective run initials each word"
304
- # also reaches "john e jones" through its lowercase ' e ' and
305
- # contests the name with #528's rule. "jones, john e" has no
306
- # row in either: that connective-run rule's operative half is a
307
- # `(?=\s)` lookahead requiring whitespace AFTER the letter, and
308
- # a letter at the end of the string can never satisfy it -- so
309
- # the comma form is a distinct shape from the space-written
310
- # form even though both give the same initials() answer.
295
+ # These two are among #528's four movers, written once with
296
+ # the letter mid-name and once at the end after a comma: both
297
+ # give the same initials() answer by different routes.
311
298
  hn = HumanName("john e jones")
312
299
  self.m(hn.initials(), "j. e. j.", hn)
313
300
  hn = HumanName("jones, john e")
@@ -0,0 +1,55 @@
1
+ """Shared handles on tools/differential/: its corpora and its modules.
2
+
3
+ Neither is importable the ordinary way: `tools/` is outside
4
+ `testpaths` and is not a package, so its modules are loaded by path,
5
+ and the corpora are data files.
6
+
7
+ Not a conftest: these are constants and plain helpers, not fixtures in
8
+ pytest's sense, and several modules want them by name.
9
+ """
10
+ import importlib.util
11
+ import json
12
+ from pathlib import Path
13
+ from types import ModuleType
14
+
15
+ _TOOLS = Path(__file__).parents[2] / "tools" / "differential"
16
+
17
+
18
+ def _entry_name(raw: object) -> str:
19
+ """A corpus line's name, whichever format the line uses: a bare
20
+ JSON string or an object carrying "name"."""
21
+ if isinstance(raw, str):
22
+ return raw
23
+ assert isinstance(raw, dict) and isinstance(raw.get("name"), str), raw
24
+ return raw["name"]
25
+
26
+
27
+ #: Corpus files _CORPUS_NAMES leaves out. corpus_cldr.jsonl
28
+ #: (2026-10-12): about 1,900 fictional CLDR renderings whose ground
29
+ #: truth the agreement ratchet in test_cldr_agreement.py enforces. The
30
+ #: tests reading _CORPUS_NAMES carry counts measured over the names the
31
+ #: corpora held before it arrived.
32
+ _UNMEASURED_CORPORA = frozenset({"corpus_cldr.jsonl"})
33
+
34
+ #: Every corpus name, deduplicated, except the files in
35
+ #: _UNMEASURED_CORPORA.
36
+ _CORPUS_NAMES = sorted({
37
+ _entry_name(json.loads(line))
38
+ for path in sorted(_TOOLS.glob("corpus*.jsonl"))
39
+ if path.name not in _UNMEASURED_CORPORA
40
+ for line in path.read_text(encoding="utf-8").splitlines() if line.strip()})
41
+
42
+
43
+ def load_tool(stem: str) -> ModuleType:
44
+ """A module from tools/differential/, loaded by path.
45
+
46
+ `tools/` is outside testpaths, and adding it would run
47
+ --doctest-modules over the corpus builders, so these are imported
48
+ this way rather than made importable. None has import-time side
49
+ effects: each main() is behind a __name__ guard."""
50
+ spec = importlib.util.spec_from_file_location(
51
+ f"differential_{stem}", _TOOLS / f"{stem}.py")
52
+ assert spec is not None and spec.loader is not None, stem
53
+ module = importlib.util.module_from_spec(spec)
54
+ spec.loader.exec_module(module)
55
+ return module