nameparser 2.0.0rc2__tar.gz → 2.1.0__tar.gz

{nameparser-2.0.0rc2/nameparser.egg-info → nameparser-2.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nameparser
-Version: 2.0.0rc2
+Version: 2.1.0
 Summary: A simple Python module for parsing human names into their individual components.
 Author-email: Derek Gulbranson <derek73@gmail.com>
 License: LGPL
@@ -22,6 +22,8 @@ Requires-Python: >=3.11
 Description-Content-Type: text/x-rst
 License-File: LICENSE
 License-File: AUTHORS
+Provides-Extra: ja
+Requires-Dist: namedivider-python>=0.4; extra == "ja"
 Dynamic: license-file
 
 Name Parser
@@ -33,6 +35,19 @@ nameparser parses human names into seven fields — title, given, middle,
 family, suffix, nickname, maiden. Results are immutable, configuration is
 composable, and locale packs are opt-in.
 
+📣 **nameparser 2.0 is out.** Existing ``HumanName`` code keeps working
+through 2.x, and most 1.x code needs no changes. The `migration guide
+<https://nameparser.readthedocs.io/en/latest/migrate.html>`__ has the
+field-by-field map. Please `open an issue
+<https://github.com/derek73/python-nameparser/issues>`__ for anything that
+parses wrong.
+
+**2.1 adds East Asian name support.** Chinese, Japanese and Korean names
+written in their own scripts are read family-first, unspaced Korean names
+are split against the census surname list, and CJK honorifics are
+recognized. See `East Asian names
+<https://nameparser.readthedocs.io/en/latest/usage.html#east-asian-names>`__.
+
 Installation
 ------------
 
@@ -48,16 +63,41 @@ Quick Start Example
 .. code-block:: python
 
     >>> from nameparser import parse
-    >>> name = parse("Dr. Juan Q. Xavier de la Vega III")
-    >>> name.given, name.family
-    ('Juan', 'de la Vega')
+    >>> name = parse("Dr. Juan Q. Xavier de la Vega III (Doc Vega)")
+    >>> name
+    <ParsedName: [
+        title: 'Dr.'
+        given: 'Juan'
+        middle: 'Q. Xavier'
+        family: 'de la Vega'
+        suffix: 'III'
+        nickname: 'Doc Vega'
+    ]>
+    >>> name.family_base, name.family_particles
+    ('Vega', 'de la')
     >>> name.render("{family}, {given}")
     'de la Vega, Juan'
 
-Those seven fields are ``title``, ``given``, ``middle``, ``family``,
-``suffix``, ``nickname``, and ``maiden`` — plus aggregate views like
-``given_names``, ``surnames``, ``family_base``, and ``family_particles``
-for combining or splitting them further.
+    >>> parse("김민준").family                     # Korean: unspaced, split on the census list
+    '김'
+    >>> parse("高橋 みなみ").family                 # Japanese: kanji with kana, family first
+    '高橋'
+    >>> parse("김민준씨").suffix                   # an honorific written against the name
+    '씨'
+    >>> parse("г-н Иван Петров").title             # Cyrillic title
+    'г-н'
+    >>> parse("محمد بن سلمان").family              # Arabic: بن chains onto the family name
+    'بن سلمان'
+
+    >>> from nameparser import locales, parser_for
+    >>> chinese = parser_for(locales.ZH)           # Han text does not say which language
+    >>> chinese.parse("毛泽东").family              # so splitting it is opt-in
+    '毛'
+    >>> russian = parser_for(locales.RU)
+    >>> russian.parse("Сидоров Иван Петрович").family
+    'Сидоров'
+    >>> locales.available()
+    ('ja', 'ru', 'tr_az', 'zh')
 
 Learn more
 ----------

{nameparser-2.0.0rc2 → nameparser-2.1.0}/README.rst

@@ -7,6 +7,19 @@ nameparser parses human names into seven fields — title, given, middle,
 family, suffix, nickname, maiden. Results are immutable, configuration is
 composable, and locale packs are opt-in.
 
+📣 **nameparser 2.0 is out.** Existing ``HumanName`` code keeps working
+through 2.x, and most 1.x code needs no changes. The `migration guide
+<https://nameparser.readthedocs.io/en/latest/migrate.html>`__ has the
+field-by-field map. Please `open an issue
+<https://github.com/derek73/python-nameparser/issues>`__ for anything that
+parses wrong.
+
+**2.1 adds East Asian name support.** Chinese, Japanese and Korean names
+written in their own scripts are read family-first, unspaced Korean names
+are split against the census surname list, and CJK honorifics are
+recognized. See `East Asian names
+<https://nameparser.readthedocs.io/en/latest/usage.html#east-asian-names>`__.
+
 Installation
 ------------
 
@@ -22,16 +35,41 @@ Quick Start Example
 .. code-block:: python
 
     >>> from nameparser import parse
-    >>> name = parse("Dr. Juan Q. Xavier de la Vega III")
-    >>> name.given, name.family
-    ('Juan', 'de la Vega')
+    >>> name = parse("Dr. Juan Q. Xavier de la Vega III (Doc Vega)")
+    >>> name
+    <ParsedName: [
+        title: 'Dr.'
+        given: 'Juan'
+        middle: 'Q. Xavier'
+        family: 'de la Vega'
+        suffix: 'III'
+        nickname: 'Doc Vega'
+    ]>
+    >>> name.family_base, name.family_particles
+    ('Vega', 'de la')
     >>> name.render("{family}, {given}")
     'de la Vega, Juan'
 
-Those seven fields are ``title``, ``given``, ``middle``, ``family``,
-``suffix``, ``nickname``, and ``maiden`` — plus aggregate views like
-``given_names``, ``surnames``, ``family_base``, and ``family_particles``
-for combining or splitting them further.
+    >>> parse("김민준").family                     # Korean: unspaced, split on the census list
+    '김'
+    >>> parse("高橋 みなみ").family                 # Japanese: kanji with kana, family first
+    '高橋'
+    >>> parse("김민준씨").suffix                   # an honorific written against the name
+    '씨'
+    >>> parse("г-н Иван Петров").title             # Cyrillic title
+    'г-н'
+    >>> parse("محمد بن سلمان").family              # Arabic: بن chains onto the family name
+    'بن سلمان'
+
+    >>> from nameparser import locales, parser_for
+    >>> chinese = parser_for(locales.ZH)           # Han text does not say which language
+    >>> chinese.parse("毛泽东").family              # so splitting it is opt-in
+    '毛'
+    >>> russian = parser_for(locales.RU)
+    >>> russian.parse("Сидоров Иван Петрович").family
+    'Сидоров'
+    >>> locales.available()
+    ('ja', 'ru', 'tr_az', 'zh')
 
 Learn more
 ----------

{nameparser-2.0.0rc2 → nameparser-2.1.0}/nameparser/__init__.py

@@ -11,6 +11,7 @@ from nameparser._locale import Locale
 from nameparser._parser import Parser, parse, parser_for
 from nameparser._policy import (
     DEFAULT_NICKNAME_DELIMITERS,
+    DEFAULT_SCRIPT_ORDERS,
     FAMILY_FIRST,
     FAMILY_FIRST_GIVEN_LAST,
     GIVEN_FIRST,
@@ -18,6 +19,7 @@ from nameparser._policy import (
     PatronymicRule,
     Policy,
     PolicyPatch,
+    Script,
 )
 from nameparser._types import (
     STABLE_TAGS,
@@ -25,6 +27,8 @@ from nameparser._types import (
     AmbiguityKind,
     ParsedName,
     Role,
+    Segmentation,
+    Segmenter,
     Span,
     Token,
 )
@@ -34,9 +38,9 @@ __all__ = [
     "HumanName",
     # v2 core
     "Span", "Role", "Token", "Ambiguity", "AmbiguityKind", "ParsedName",
-    "STABLE_TAGS",
-    "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "UNSET",
+    "STABLE_TAGS", "Segmentation", "Segmenter",
+    "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "Script", "UNSET",
     "GIVEN_FIRST", "FAMILY_FIRST", "FAMILY_FIRST_GIVEN_LAST",
-    "DEFAULT_NICKNAME_DELIMITERS", "Locale",
+    "DEFAULT_NICKNAME_DELIMITERS", "DEFAULT_SCRIPT_ORDERS", "Locale",
     "Parser", "parse", "parser_for",
 ]

{nameparser-2.0.0rc2 → nameparser-2.1.0}/nameparser/_config_shim.py

@@ -986,10 +986,23 @@ class Constants:
         removal path.
         """
         from nameparser.config.maiden_markers import MAIDEN_MARKERS
+        from nameparser.config.suffixes import GLUED_HONORIFICS
+        from nameparser.config.surnames import KOREAN_SURNAMES
         acronyms = frozenset(self.suffix_acronyms)
         particles = frozenset(self.prefixes)
         bound = frozenset(self.bound_first_names)
         ambiguous_acronyms = frozenset(self.suffix_acronyms_ambiguous) & acronyms
+        # Drop any ambiguous acronym from the word set rather than the
+        # other way round. Lexicon forbids the overlap because the word
+        # branch bypasses the period gate, and adding an ambiguous
+        # acronym to suffix_not_acronyms is INERT in v1 anyway:
+        # is_suffix already accepts it via the acronym branch, and
+        # reserve_last keeps it as the surname. So ignoring the
+        # addition reproduces v1 ("Jack Ma" keeps last='Ma'), where
+        # dropping it from the AMBIGUOUS set instead ungated the word
+        # and lost the family name -- a silent misparse worse than the
+        # raise it avoided.
+        suffix_words = frozenset(self.suffix_not_acronyms) - ambiguous_acronyms
         # keep in sync with _lexicon._default_lexicon() (pinned by
         # tests/v2/test_config_shim.py::test_snapshot_field_translation)
         lexicon = Lexicon(
@@ -1016,18 +1029,7 @@ class Constants:
                     if e == " ".join(e.split())
                 ) if t),
             suffix_acronyms=acronyms,
-            # Drop any ambiguous acronym from the word set rather than
-            # the other way round. Lexicon forbids the overlap because
-            # the word branch bypasses the period gate, and adding an
-            # ambiguous acronym to suffix_not_acronyms is INERT in v1
-            # anyway: is_suffix already accepts it via the acronym
-            # branch, and reserve_last keeps it as the surname. So
-            # ignoring the addition reproduces v1 ("Jack Ma" keeps
-            # last='Ma'), where dropping it from the AMBIGUOUS set
-            # instead ungated the word and lost the family name --
-            # a silent misparse worse than the raise it avoided.
-            suffix_words=frozenset(
-                self.suffix_not_acronyms) - ambiguous_acronyms,
+            suffix_words=suffix_words,
             # Intersect with acronyms: Lexicon enforces ambiguous <=
             # acronyms; v1 behaves the same when an acronym is deleted
             # but its ambiguous entry lingers (the entry stops
@@ -1061,6 +1063,22 @@ class Constants:
             # v1 Constants has no manager for these (#274 is 2.0
             # behavior); the data module is the only source
             maiden_markers=frozenset(MAIDEN_MARKERS),
+            # likewise no v1 manager: the unspaced-name segmentation
+            # vocabulary is 2.0 behavior (#271), so it rides in the
+            # snapshot only -- v1's Constants surface stays frozen.
+            # Unwrapped where maiden_markers above is wrapped: this
+            # module is born frozen (#293), so no wrap
+            surnames=KOREAN_SURNAMES,
+            # likewise no v1 manager: the glued-honorific tail set is
+            # 2.1 behavior (#308), so it rides in the snapshot only.
+            # Wrapped, unlike surnames above: suffixes.py is still a
+            # mutable v1 module, not born-frozen like surnames.py
+            # (#293). Intersect with the word set: Lexicon enforces
+            # tails <= suffix_words, and v1 semantics are that deleting
+            # a suffix word turns the behavior off -- a lingering tail
+            # simply stops mattering, the same rule ambiguous_acronyms
+            # gets against suffix_acronyms above.
+            honorific_tails=frozenset(GLUED_HONORIFICS) & suffix_words,
             # TupleManager is dict[str, object] (v1 parity: values were
             # never statically str-typed); every real entry is a str,
             # same assumption _DelimiterManager's sentinel lookup makes

{nameparser-2.0.0rc2 → nameparser-2.1.0}/nameparser/_facade.py

@@ -623,6 +623,24 @@ class HumanName:
         # spelling the facade actually exposes as attributes.
         return getattr(self, _V1_SPELLING.get(key, key))
 
+    def __setattr__(self, name: str, value: object) -> None:
+        # "given"/"family" are the 2.0 spellings of first/last; the
+        # facade has no such attributes, so plain assignment creates a
+        # stray instance attribute while the parse (and .first/.last)
+        # keeps the old value -- a silently forked name. Warn but
+        # still set: ad-hoc attribute stashing is a legal v1 pattern,
+        # so any code that worked keeps working. Only these two names
+        # warn -- the other five 2.0 field names are real properties
+        # whose setters work, and Role members reach here as their
+        # string values (StrEnum).
+        if name in _V1_SPELLING:
+            warnings.warn(
+                f"assigning HumanName.{name} creates an inert attribute; "
+                f"the parse is unchanged -- use .{_V1_SPELLING[name]} "
+                f"(the v1 spelling) to update the name",
+                UserWarning, stacklevel=2)
+        super().__setattr__(name, value)
+
     def as_dict(self, include_empty: bool = True) -> dict[str, str]:
         """The seven v1-named components as a dict; include_empty=False
         drops empty fields."""

{nameparser-2.0.0rc2 → nameparser-2.1.0}/nameparser/_lexicon.py

@@ -23,20 +23,44 @@ from typing import cast
 _VOCAB_FIELDS = (
     "titles", "given_name_titles", "suffix_acronyms", "suffix_words",
     "suffix_acronyms_ambiguous", "particles", "particles_ambiguous",
-    "conjunctions", "bound_given_names", "maiden_markers",
+    "conjunctions", "bound_given_names", "maiden_markers", "surnames",
+    "honorific_tails",
 )
 
-#: (marker, base, why) triples. Each marker narrows how entries of its
-#: base vocabulary are read and carries no vocabulary of its own, so an
-#: entry outside the base is a configuration mistake -- but the mistake
-#: differs per pair, and the reason is recorded here rather than
-#: generalized, because an orphan is NOT simply inert:
+#: (marker, base, why) triples. Each marker QUALIFIES how entries of
+#: its base vocabulary are read and carries no vocabulary of its own,
+#: so an entry outside the base is a configuration mistake -- but the
+#: mistake differs per pair, and the reason is recorded here rather
+#: than generalized, because an orphan is NOT simply inert. Nor is the
+#: qualification one-directional: the first two NARROW their base (an
+#: entry is read as vocabulary in fewer places), while honorific_tails
+#: WIDENS it, granting a suffix word the glued position on top of the
+#: whole-token match every suffix word already gets.
 #:
 #: * particles_ambiguous: _assign keys on the tag alone, so an orphan
 #:   makes the parse emit a spurious particle-or-given ambiguity.
 #: * suffix_acronyms_ambiguous: _vocab returns True on the ambiguous
 #:   set before testing suffix_acronyms, so an orphan silently turns a
 #:   word into a period-gated suffix.
+#: * honorific_tails: script_segment peels the tail into its own token
+#:   before classify ever runs, so an orphan splits the name and leaves
+#:   the fragment stranded inside it -- worse than not peeling at all.
+#:   Its base is deliberately NARROWER than what actually claims the
+#:   peeled piece: suffix_as_written ORs suffix_words with the
+#:   non-ambiguous suffix_acronyms, so a tail listed only as an acronym
+#:   would classify fine yet is rejected here. Accepted, and a decision
+#:   rather than an oversight -- the three-term predicate is easy to
+#:   get wrong in the dangerous direction (an ambiguous acronym admitted
+#:   as a tail would peel a period-gated word off a real name), and
+#:   nothing needs the acronym half: the shipped tails are CJK
+#:   honorifics, which are words.
+#:   The same relation is asserted a second time in config/suffixes.py,
+#:   over the raw GLUED_HONORIFICS/SUFFIX_NOT_ACRONYMS constants at
+#:   import. The two are not redundant in the way they look: that one
+#:   is an `assert`, stripped under `python -O`, while the check here
+#:   raises unconditionally -- so under -O this is what still holds the
+#:   SHIPPED vocabulary to the invariant, as it is the only thing that
+#:   ever held a caller's own.
 #:
 #: given_name_titles is deliberately NOT here and has no check of its
 #: own -- see the note in __post_init__ for why every attempt at one
@@ -46,6 +70,8 @@ _SUBSET_FIELDS = (
      "an orphan emits a spurious particle-or-given ambiguity"),
     ("suffix_acronyms_ambiguous", "suffix_acronyms",
      "an orphan silently becomes a period-gated suffix"),
+    ("honorific_tails", "suffix_words",
+     "an orphan splits the name and leaves the tail inside it"),
 )
 
 
@@ -327,6 +353,31 @@ class Lexicon:
     #: field ("née", "geb.", "roz.", ...). Full default list:
     #: :data:`~nameparser.config.maiden_markers.MAIDEN_MARKERS`.
     maiden_markers: frozenset[str] = frozenset()
+    #: Family names for the unspaced-name segmentation stage (#271),
+    #: matched longest-first against the start of the FIRST token
+    #: written wholly in a script :attr:`Policy.segment_scripts
+    #: <nameparser.Policy.segment_scripts>` activates. The default
+    #: carries the Korean census list
+    #: (:data:`~nameparser.config.surnames.KOREAN_SURNAMES`); Chinese
+    #: surnames ship in locales.ZH because Han segmentation is opt-in.
+    surnames: frozenset[str] = frozenset()
+    #: Honorifics that may be peeled off the END of a name token
+    #: (#308), matched longest-first: 田中さん splits into 田中 and さん
+    #: before the tokens are classified. Every entry must also be a
+    #: :attr:`suffix_words` entry -- the peeled tail is claimed by
+    #: suffix classification like any other post-nominal. Deliberately
+    #: NOT gated on :attr:`Policy.segment_scripts
+    #: <nameparser.Policy.segment_scripts>` (unlike :attr:`surnames`
+    #: above): 田中さん peels under the default policy, where HAN is in
+    #: no activation set, because a tail entry carries its own license
+    #: to fire. Entries are matched against the RAW token text, and
+    #: only within a name containing a non-ASCII character, so an ASCII
+    #: or mixed-case entry is at best conditionally active -- a ``"Jr"``
+    #: entry is stored ``"jr"`` and matches only lowercase text. The
+    #: field is effectively CJK-scoped in 2.1, which is what the shipped
+    #: vocabulary is. Full default list:
+    #: :data:`~nameparser.config.suffixes.GLUED_HONORIFICS`.
+    honorific_tails: frozenset[str] = frozenset()
     #: Lowercase word -> exact-cased replacement used by capitalized()
     #: ("phd" -> "Ph.D."). Pair-valued: change it with
     #: dataclasses.replace(), not add()/remove(); read it as a mapping
@@ -569,8 +620,10 @@ def _default_lexicon() -> Lexicon:
     from nameparser.config.maiden_markers import MAIDEN_MARKERS
     from nameparser.config.prefixes import NON_FIRST_NAME_PREFIXES, PREFIXES
     from nameparser.config.suffixes import (
-        SUFFIX_ACRONYMS, SUFFIX_ACRONYMS_AMBIGUOUS, SUFFIX_NOT_ACRONYMS,
+        GLUED_HONORIFICS, SUFFIX_ACRONYMS, SUFFIX_ACRONYMS_AMBIGUOUS,
+        SUFFIX_NOT_ACRONYMS,
     )
+    from nameparser.config.surnames import KOREAN_SURNAMES
     from nameparser.config.titles import FIRST_NAME_TITLES, TITLES
 
     # v1 data modules export plain `set[str]`; wrap each at this call site
@@ -590,6 +643,10 @@ def _default_lexicon() -> Lexicon:
         conjunctions=frozenset(CONJUNCTIONS),
         bound_given_names=frozenset(BOUND_FIRST_NAMES),
         maiden_markers=frozenset(MAIDEN_MARKERS),
+        # surnames.py is born frozen (#293) -- no call-site wrap needed,
+        # unlike the v1 modules above (their wraps drop when #293 lands)
+        surnames=KOREAN_SURNAMES,
+        honorific_tails=frozenset(GLUED_HONORIFICS),
         # pass canonical pair-tuples so this strictly-typed call site never
         # feeds a Mapping to the tuple-annotated field; __post_init__
         # still tolerates a Mapping at runtime for interactive use

{nameparser-2.0.0rc2 → nameparser-2.1.0}/nameparser/_parser.py

@@ -11,17 +11,18 @@ from __future__ import annotations
 import dataclasses
 import functools
 import warnings
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 
 from nameparser._lexicon import Lexicon
 from nameparser._locale import Locale
 from nameparser._pipeline import run
 from nameparser._pipeline._assemble import assemble
 from nameparser._pipeline._state import ParseState
-from nameparser._policy import UNSET, Policy, PolicyPatch, apply_patch
+from nameparser._pipeline._vocab import _SCRIPT_MATCHERS
+from nameparser._policy import UNSET, Policy, PolicyPatch, _Unset, apply_patch
 from nameparser._types import (
-    FOLDED_TAG, ParsedName, Token, _guarded_getstate, _guarded_setstate,
-    _validated_field_strings,
+    FOLDED_TAG, ParsedName, Segmenter, Token, _guarded_getstate,
+    _guarded_setstate, _validated_field_strings,
 )
 
 
@@ -30,18 +31,37 @@ class Parser:
     """A configured name parser: a :class:`Lexicon` (vocabulary) plus
     a :class:`Policy` (behavior), both defaulted when omitted. Build
     one when you need non-default configuration, build it once, and
-    call :meth:`parse` many times -- it is immutable, thread-safe, and
-    picklable by construction: all validity checking happens at
+    call :meth:`parse` many times -- it is immutable and thread-safe.
+
+    An optional keyword-only ``segmenter`` (a :data:`~nameparser.Segmenter`)
+    plugs in outside knowledge of where an unspaced CJK token divides --
+    Japanese kanji names, which no bundled list can settle. It is
+    consulted only for a token the segmentation stage gates in and the
+    vocabulary DECLINES, so a locale pack's surnames always win where
+    they match; returning None declines in turn and the token stays
+    whole. Two promises narrow when one is supplied (locales spec §4):
+    parse-totality gains its one exception -- an exception raised by
+    the segmenter propagates, because a user-supplied callable's own
+    error is a user-code error, not a content error -- and this Parser
+    pickles only if its segmenter does (a module-level function
+    pickles; a lambda or closure does not). With no segmenter, both
+    promises hold unconditionally: all validity checking happens at
     construction, so a Parser that constructs successfully cannot fail
     at parse time on any str content.
 
     (The None field defaults resolve in __post_init__; after
-    construction both fields are always non-None -- the annotations
-    state the steady-state truth, hence the assignment ignores on the
-    defaults.)"""
+    construction lexicon and policy are always non-None -- the
+    annotations state the steady-state truth, hence the assignment
+    ignores on the defaults.)"""
 
     lexicon: Lexicon = None  # type: ignore[assignment]  # None -> default()
     policy: Policy = None  # type: ignore[assignment]    # None -> Policy()
+    #: An optional hook supplying outside knowledge of where an unspaced
+    #: token divides -- see the class docstring; None leaves such tokens
+    #: whole. Keyword-only, so the reserved growth stays additive
+    #: (locales spec §4): positional construction keeps its two-argument
+    #: shape.
+    segmenter: Segmenter | None = field(default=None, kw_only=True)
 
     # in the class body so @dataclass(slots=True) keeps them
     __getstate__ = _guarded_getstate
@@ -58,16 +78,73 @@ class Parser:
         elif not isinstance(self.policy, Policy):
             raise TypeError(
                 f"policy must be a Policy or None, got {self.policy!r}")
+        if self.segmenter is not None and not callable(self.segmenter):
+            raise TypeError(
+                f"segmenter must be callable or None, got {self.segmenter!r}")
+        # A configuration gap that used to be silent (#272's API, made
+        # loud before 2.1.0): segment_scripts can activate a script
+        # that neither the vocabulary nor a segmenter can ever divide
+        # -- the JA pack's whole shape, when its segmenter is
+        # forgotten. The parser then behaves identically to a working
+        # one minus the feature, which reads as "not working" with no
+        # signal why. Statically decidable here, so say it here; a
+        # warning rather than an error because the inert pack is a
+        # pinned, deliberate property (a JA registration must be safe
+        # without the extra), and warnings are filterable by the rare
+        # caller who wants exactly that.
+        if self.segmenter is None:
+            uncovered = sorted(
+                script.value
+                for script in self.policy.segment_scripts
+                if not any(_SCRIPT_MATCHERS[script](entry)
+                           for entry in self.lexicon.surnames))
+            if uncovered:
+                names = ", ".join(uncovered)
+                one = len(uncovered) == 1
+                # the ja hint only where a Japanese script is among the
+                # dead ones -- a hangul-only gap (a from-scratch
+                # lexicon under the default policy) has different
+                # remedies, and pointing it at ja_segmenter would be a
+                # non sequitur
+                ja_hint = (
+                    " For Japanese, pass "
+                    "segmenter=locales.ja_segmenter() (install with: "
+                    "pip install 'nameparser[ja]')."
+                    if {"han", "hiragana", "katakana"} & set(uncovered)
+                    else "")
+                warnings.warn(
+                    f"Policy.segment_scripts activates {names} but the "
+                    f"vocabulary has no surnames in "
+                    f"{'that script' if one else 'those scripts'} "
+                    f"and no segmenter is configured: unspaced names "
+                    f"written in {'it' if one else 'them'} will never "
+                    f"divide. Supply covering surnames, pass a "
+                    f"segmenter, or deactivate with "
+                    f"Policy(segment_scripts=frozenset()).{ja_hint}",
+                    UserWarning, stacklevel=3)
 
     def __repr__(self) -> str:
-        # composes the two bounded component reprs (spec §2 reprs)
-        return f"Parser({self.lexicon!r}, {self.policy!r})"
+        # composes the two bounded component reprs (spec §2 reprs); the
+        # segmenter shows by name, and only when one is set, so the
+        # default Parser's repr is unchanged
+        seg = ""
+        if self.segmenter is not None:
+            # never repr() the callable itself: a partial reprs its
+            # bound arguments and a callable instance its address, both
+            # unbounded -- the class name is the bounded fallback
+            name = (getattr(self.segmenter, "__qualname__", None)
+                    or type(self.segmenter).__name__)
+            seg = f", segmenter={name}"
+        return f"Parser({self.lexicon!r}, {self.policy!r}{seg})"
 
     def parse(self, text: str) -> ParsedName:
         """Parse one name string into a :class:`ParsedName`. Never
         raises on string content (unparseable input yields empty
         fields plus ambiguities); non-str raises TypeError eagerly,
-        with a decode hint for bytes (bytes support ended with 1.x)."""
+        with a decode hint for bytes (bytes support ended with 1.x).
+        The one exception to that totality is a configured
+        ``segmenter``, whose own exceptions propagate (see the class
+        docstring)."""
         if isinstance(text, bytes):
             raise TypeError(
                 "parse() takes str, not bytes -- decode first, e.g. "
@@ -75,7 +152,7 @@ class Parser:
         if not isinstance(text, str):
             raise TypeError(f"parse() takes str, got {text!r}")
         state = ParseState(original=text, lexicon=self.lexicon,
-                           policy=self.policy)
+                           policy=self.policy, segmenter=self.segmenter)
         return assemble(run(state))
 
     # -- editing ----------------------------------------------------------
@@ -92,7 +169,11 @@ class Parser:
         role choices and ambiguities are discarded -- every harvested
         token takes the named field's role -- and its structural
         behavior applies: delimiter characters do not become tokens,
-        and a mid-value maiden marker is consumed as in parsing.
+        and a maiden marker is consumed as in parsing -- mid-value
+        always, and leading a DELIMITED value under a policy routing
+        that pair to maiden, where "(née Jones)" revises to "Jones"
+        while the bare "née Jones" keeps its marker, a leading marker
+        in an undelimited value being no marker at all (#329).
         Tokens are synthetic (span=None); original is unchanged; a
         value with no name content (empty, whitespace, or punctuation
         only) clears the field; ambiguities referencing replaced
@@ -151,14 +232,27 @@ def parse(text: str) -> ParsedName:
     return _default_parser().parse(text)
 
 
-def parser_for(*locales: Locale, base: Parser | None = None) -> Parser:
+def parser_for(*locales: Locale, base: Parser | None = None,
+               segmenter: Segmenter | None | _Unset = UNSET) -> Parser:
     """Lexicon fragments unioned left-to-right onto base's; policy
     patches applied left-to-right (later wins; set-valued fields union
     per the patch metadata). Validation errors raised while applying a
     pack are wrapped with that pack's identity (spec §4 amendment) --
     PolicyPatch validates lazily, so with stacked packs the raw error
     would otherwise point at nothing. Two packs setting the same SCALAR
-    field is a declared conflict: UserWarning, later wins."""
+    field is a declared conflict: UserWarning, later wins.
+
+    A ``segmenter`` is passed straight through to the built Parser --
+    ``parser_for(locales.JA, segmenter=locales.ja_segmenter())`` is how
+    a pack and a segmenter combine, since packs are pure data and
+    cannot supply one. The argument has THREE states, the same
+    :data:`~nameparser.UNSET` spelling a PolicyPatch field uses, because
+    None is a meaningful value here and not an absence: omitted (UNSET)
+    carries base's segmenter through unchanged; a callable OVERRIDES
+    base's (later wins, the rule scalar policy fields follow); and an
+    explicit ``None`` CLEARS base's, which is how you derive an
+    unsegmented parser from a segmented one without rebuilding its
+    lexicon and policy by hand."""
     if base is not None and not isinstance(base, Parser):
         raise TypeError(f"base must be a Parser or None, got {base!r}")
     for loc in locales:
@@ -166,6 +260,15 @@ def parser_for(*locales: Locale, base: Parser | None = None) -> Parser:
             raise TypeError(f"parser_for() takes Locale packs, got {loc!r}")
     lexicon = base.lexicon if base is not None else Lexicon.default()
     policy = base.policy if base is not None else Policy()
+    # Resolved here rather than at the return because the return builds
+    # a FRESH Parser: any field not listed there silently takes its
+    # default, and a dropped segmenter would be invisible. UNSET, not
+    # None, is what "not given" means -- None is the CLEAR request, and
+    # collapsing the two would make an explicit
+    # parser_for(..., segmenter=None) silently inherit the very
+    # segmenter it was asked to drop.
+    if segmenter is UNSET:
+        segmenter = base.segmenter if base is not None else None
     scalar_setters: dict[str, str] = {}
     for loc in locales:
         for f in dataclasses.fields(PolicyPatch):
@@ -189,4 +292,15 @@ def parser_for(*locales: Locale, base: Parser | None = None) -> Parser:
             # a subclass with extra mandatory args would break this rewrap
             raise type(exc)(
                 f"while applying locale {loc.code!r}: {exc}") from exc
-    return Parser(lexicon=lexicon, policy=policy)
+    # Construction warnings (the segmenterless-activation check in
+    # Parser.__post_init__) re-emit from THIS frame: its stacklevel is
+    # sized for direct Parser(...) construction, and through this
+    # function's extra frame the default single-line rendering would
+    # point into the library instead of at the caller -- the exact
+    # call the message tells them to change.
+    with warnings.catch_warnings(record=True) as caught:
+        warnings.simplefilter("always")
+        built = Parser(lexicon=lexicon, policy=policy, segmenter=segmenter)
+    for w in caught:
+        warnings.warn(w.message, stacklevel=2)
+    return built

{nameparser-2.0.0rc2 → nameparser-2.1.0}/nameparser/_pipeline/__init__.py

@@ -16,14 +16,15 @@ from nameparser._pipeline._classify import classify
 from nameparser._pipeline._extract import extract_delimited
 from nameparser._pipeline._group import group
 from nameparser._pipeline._post_rules import post_rules
+from nameparser._pipeline._script_segment import script_segment
 from nameparser._pipeline._segment import segment
 from nameparser._pipeline._state import ParseState
 from nameparser._pipeline._tokenize import tokenize
 
-#: The full seven-stage fold.
+#: The full eight-stage fold.
 STAGES: tuple[Callable[[ParseState], ParseState], ...] = (
-    extract_delimited, tokenize, segment, classify, group, assign,
-    post_rules,
+    extract_delimited, tokenize, segment, script_segment, classify,
+    group, assign, post_rules,
 )