vn-text 0.1.0__py3-none-any.whl

vn_text/__init__.py

@@ -0,0 +1,57 @@
+"""vn-text — correct Unicode primitives for Vietnamese text.
+
+Standard library only. Mirrors the TypeScript port function-for-function, and
+both are validated against the shared conformance suite in
+``conformance/vn-text-1.0.0.json``.
+
+    >>> from vn_text import fold
+    >>> fold("Đặng Minh Anh")
+    'dang minh anh'
+"""
+
+from __future__ import annotations
+
+from ._unicode import (
+    COMBINING_MARKS,
+    D_WITH_STROKE_LOWER,
+    D_WITH_STROKE_UPPER,
+    ETH_LOWER,
+    ETH_UPPER,
+    TONE_MARKS,
+    VIETNAMESE_SPECIFIC,
+)
+from .core import (
+    decompose,
+    deaccent,
+    fold,
+    is_vietnamese,
+    isVietnamese,
+    normalize,
+    repair_mojibake,
+    repairMojibake,
+    strip_stroke,
+    stripStroke,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "__version__",
+    "normalize",
+    "decompose",
+    "deaccent",
+    "strip_stroke",
+    "repair_mojibake",
+    "fold",
+    "is_vietnamese",
+    "stripStroke",
+    "repairMojibake",
+    "isVietnamese",
+    "D_WITH_STROKE_UPPER",
+    "D_WITH_STROKE_LOWER",
+    "ETH_UPPER",
+    "ETH_LOWER",
+    "TONE_MARKS",
+    "COMBINING_MARKS",
+    "VIETNAMESE_SPECIFIC",
+]

vn_text/_unicode.py

@@ -0,0 +1,76 @@
+"""Unicode constants that matter for Vietnamese text handling.
+
+The trap
+--------
+
+Four characters look like a capital or lowercase D with a stroke::
+
+    U+0110  LATIN CAPITAL LETTER D WITH STROKE   -> Vietnamese
+    U+0111  LATIN SMALL LETTER D WITH STROKE     -> Vietnamese
+    U+00D0  LATIN CAPITAL LETTER ETH              -> Icelandic, Danish
+    U+00F0  LATIN SMALL LETTER ETH                -> Icelandic, Danish
+
+None has a canonical decomposition, and none has a compatibility decomposition
+either -- verified against both NFD and NFKD. Unicode treats each as a letter in
+its own right rather than a decorated ``D``.
+
+Why that breaks search
+----------------------
+
+The obvious implementation, normalise to NFD and strip the combining marks,
+silently fails to fold Vietnamese ``Đ``::
+
+    'Đặng Minh'  ->  'Đang Minh'
+
+The surviving ``Đ`` means the index key no longer matches anything a user can
+type, because nobody types ``Đang`` to find ``Đặng``. Nothing throws. Search
+just quietly returns fewer results than it should.
+
+The obvious repair is worse. A blanket "fold any stroked D" rule also turns ETH
+into ``D``, corrupting Icelandic and Danish names. Transliteration tables that
+treat ``Đ`` as decoration hit exactly that.
+
+So the two must be distinguished explicitly. That is what
+:func:`vn_text.strip_stroke` does: U+0110 and U+0111, nothing else.
+
+All code points below are written as :func:`chr` calls rather than as literal
+characters or ``\\uXXXX`` escapes. Combining marks are invisible in an editor
+and a stray reformat or copy-paste can silently corrupt a character class that
+happens to still compile. Keeping this module pure ASCII makes that class of
+accident impossible, and keeps diffs readable.
+"""
+
+from __future__ import annotations
+
+import re
+
+#: LATIN CAPITAL LETTER D WITH STROKE (Vietnamese).
+D_WITH_STROKE_UPPER = chr(0x0110)
+
+#: LATIN SMALL LETTER D WITH STROKE (Vietnamese).
+D_WITH_STROKE_LOWER = chr(0x0111)
+
+#: LATIN CAPITAL LETTER ETH (Icelandic, Danish). Never fold to D.
+ETH_UPPER = chr(0x00D0)
+
+#: LATIN SMALL LETTER ETH (Icelandic, Danish). Never fold to d.
+ETH_LOWER = chr(0x00F0)
+
+#: The five Vietnamese tone marks as combining marks, in NFD form.
+#: sắc, huyền, hỏi, ngã, nặng.
+TONE_MARKS: tuple[str, ...] = (
+    chr(0x0301),  # sắc    - COMBINING ACUTE ACCENT
+    chr(0x0300),  # huyền  - COMBINING GRAVE ACCENT
+    chr(0x0309),  # hỏi    - COMBINING HOOK ABOVE
+    chr(0x0303),  # ngã    - COMBINING TILDE
+    chr(0x0323),  # nặng   - COMBINING DOT BELOW
+)
+
+#: Full Combining Diacritical Marks block, U+0300 to U+036F.
+COMBINING_MARKS = re.compile(f"[{chr(0x0300)}-{chr(0x036F)}]")
+
+#: Vietnamese-specific letters: D WITH STROKE plus the precomposed block
+#: U+1EA0 to U+1EF9 (a-circumflex-dot-below through y-tilde-dot-below).
+VIETNAMESE_SPECIFIC = re.compile(
+    f"[{D_WITH_STROKE_UPPER}{D_WITH_STROKE_LOWER}{chr(0x1EA0)}-{chr(0x1EF9)}]"
+)

vn_text/core.py

@@ -0,0 +1,206 @@
+"""Core Vietnamese text transforms.
+
+Standard library only, on purpose. A text-normalisation library that drags in
+a dependency is a dependency every consumer has to audit, and this one has to
+stay auditable itself.
+
+Every function here has a counterpart in the TypeScript port, and both ports
+are validated against the same conformance file at
+``conformance/vn-text-1.0.0.json``.
+"""
+
+from __future__ import annotations
+
+import unicodedata
+
+from ._unicode import (
+    COMBINING_MARKS,
+    D_WITH_STROKE_LOWER,
+    D_WITH_STROKE_UPPER,
+    ETH_LOWER,
+    ETH_UPPER,
+    VIETNAMESE_SPECIFIC,
+)
+
+__all__ = [
+    "normalize",
+    "decompose",
+    "deaccent",
+    "strip_stroke",
+    "repair_mojibake",
+    "fold",
+    "is_vietnamese",
+    # camelCase aliases for API parity with the TypeScript port.
+    "stripStroke",
+    "repairMojibake",
+    "isVietnamese",
+]
+
+
+def normalize(text: str) -> str:
+    """Normalise to Unicode Normalization Form C (composed).
+
+    The form to use for storage, database keys, equality checks and
+    deduplication. Two strings that look identical to a user can differ at the
+    code-point level if one came from a Windows-1252 editor and the other from
+    a UTF-8 terminal; NFC collapses them.
+
+    >>> normalize(decompose("Cái gì thế này")) == "Cái gì thế này"
+    True
+    """
+    return unicodedata.normalize("NFC", text)
+
+
+def decompose(text: str) -> str:
+    """Normalise to Unicode Normalization Form D (decomposed).
+
+    Vietnamese letters such as ``ế`` become a base ``e`` plus the combining
+    marks U+0302 (circumflex) and U+0301 (acute). Exposed because downstream
+    code that walks combining marks needs the decomposed form.
+    """
+    return unicodedata.normalize("NFD", text)
+
+
+def deaccent(text: str) -> str:
+    """Remove every combining diacritical mark, leaving base letters intact.
+
+    Applies to all of Unicode, not just Vietnamese -- ``café`` becomes
+    ``cafe``, ``Ѐ`` becomes ``Е``.
+
+    What this function deliberately does NOT do:
+
+    It does not fold ``đ`` (U+0111) to ``d``. Vietnamese D-WITH-STROKE has no
+    canonical decomposition, so it survives NFD untouched and comes back out
+    as ``đ``. That is correct: stripping tone marks should not change which
+    letter you are looking at. Use :func:`strip_stroke` or :func:`fold` when
+    you need ``đ`` to become ``d``.
+
+    It also leaves ETH (U+00D0) alone, because ETH is a real letter in
+    Icelandic and Danish, not a decorated D.
+
+    >>> deaccent("Tiếng Việt")
+    'Tieng Viet'
+    >>> deaccent("Đặng Minh")
+    'Đang Minh'
+    >>> deaccent("Ðor")
+    'Ðor'
+    """
+    return unicodedata.normalize(
+        "NFC", COMBINING_MARKS.sub("", unicodedata.normalize("NFD", text))
+    )
+
+
+def strip_stroke(text: str) -> str:
+    """Replace Vietnamese D-WITH-STROKE with plain ``d`` / ``D``.
+
+    Nothing else is changed. In particular tone marks are left alone -- that is
+    :func:`deaccent`'s job -- and ETH (U+00D0, U+00F0) is preserved because it
+    is a different letter belonging to another script.
+
+    >>> strip_stroke("Đặng")
+    'Dặng'
+    >>> strip_stroke("Ðor")
+    'Ðor'
+    """
+    return text.replace(D_WITH_STROKE_UPPER, "D").replace(
+        D_WITH_STROKE_LOWER, "d"
+    )
+
+
+def repair_mojibake(text: str) -> str:
+    """Repair the most common corruption in stored Vietnamese text.
+
+    Maps ``Ð`` (U+00D0) to ``Đ`` (U+0110) and ``ð`` (U+00F0) to ``đ`` (U+0111).
+
+    This is a **data repair** operation, not a search-key operation. It answers
+    a different question from :func:`deaccent` and :func:`fold`:
+
+    - :func:`fold` produces a search key and never reinterprets which letter you
+      have. It leaves ``Ð`` alone, because in Icelandic and Danish it is a real
+      letter called eth.
+    - ``repair_mojibake`` assumes the text *is* Vietnamese and that ``Ð`` is
+      damage. In Vietnamese text produced by legacy systems, ``Ð`` is
+      essentially always a mangled ``Đ``, and leaving it in place means a user
+      can never find the record again.
+
+    Compose them when the corpus is Vietnamese and may be damaged::
+
+        fold(repair_mojibake('Ðảm baỏ'))  # 'dam bao' -> finds the record
+
+    What it deliberately does not do:
+
+    - **It is Vietnamese-biased by design.** ``repair_mojibake('Ðor')`` returns
+      ``'Đor'``, which is wrong for an Icelandic name. That is the trade, and it
+      is the right one for a Vietnamese library, but do not call it on text you
+      know contains Scandinavian or Icelandic content.
+    - **It does not recover byte-level mojibake.** Damage of the form ``Ä Ä¡``
+      comes from UTF-8 bytes decoded as Windows-1252, and undoing it needs the
+      original bytes, not a character mapping. There is a dedicated library for
+      that (``ftfy``); guessing at it from a Unicode string is not reliable
+      enough to ship.
+    - **It does not fix spelling or vowel composition.** ``lựơng`` stays
+      ``lựơng``. Those are separate problems, and ``VietnameseTextNormalizer``
+      and ``underthesea.text_normalize`` are the right tools for them.
+
+    >>> repair_mojibake('Ðảm baỏ')
+    'Đảm bảo'
+    """
+    return text.replace(ETH_UPPER, D_WITH_STROKE_UPPER).replace(
+        ETH_LOWER, D_WITH_STROKE_LOWER
+    )
+
+
+def fold(text: str) -> str:
+    """Produce a search key.
+
+    Accent-free, stroke-free, lower-cased, with every run of punctuation and
+    symbols collapsed to a single space. This is the canonical form to index
+    under for Vietnamese search: two strings a user would consider the same
+    hit produce the same key.
+
+    >>> fold("Cái gì thế này")
+    'cai gi the nay'
+    >>> fold("Đặng Minh Anh")
+    'dang minh anh'
+    >>> fold("Hà Nội, Việt Nam!")
+    'ha noi viet nam'
+
+    Letters from other scripts are preserved rather than dropped, so
+    ``fold("東京 Tokyo")`` gives ``"東京 tokyo"`` -- the ideographs stay
+    searchable even though they carry no diacritics.
+    """
+    folded = strip_stroke(deaccent(text)).lower()
+    # Lower-casing can reintroduce combining marks, e.g. U+0130 -> i + U+0307.
+    folded = COMBINING_MARKS.sub("", unicodedata.normalize("NFD", folded))
+    # str.isalnum() is the closest stdlib equivalent of JavaScript's
+    # \p{L}\p{N}, covering the L* and N* general categories.
+    spaced = "".join(ch if ch.isalnum() else " " for ch in folded)
+    return " ".join(spaced.split())
+
+
+def is_vietnamese(text: str) -> bool:
+    """True when the string contains at least one Vietnamese-specific character.
+
+    Useful for deciding whether a string needs Vietnamese-aware handling at
+    all.
+
+    This is a fast heuristic, not a language detector: the common Vietnamese
+    surnames ``Nguyen``, ``Tran`` and ``Le`` are pure ASCII and return False.
+
+    >>> is_vietnamese("Đặng")
+    True
+    >>> is_vietnamese("Nguyen")
+    False
+    """
+    return VIETNAMESE_SPECIFIC.search(text) is not None
+
+
+#: API-parity alias, so code can move between the Python and TypeScript ports
+#: without renaming every call site.
+stripStroke = strip_stroke
+
+#: API-parity alias, see :func:`stripStroke`.
+repairMojibake = repair_mojibake
+
+#: API-parity alias, see :func:`stripStroke`.
+isVietnamese = is_vietnamese

vn_text-0.1.0.dist-info/METADATA

@@ -0,0 +1,98 @@
+Metadata-Version: 2.5
+Name: vn-text
+Version: 0.1.0
+Summary: Correct Unicode primitives for Vietnamese text: deaccent, fold, normalize, strip_stroke. Standard library only. Ships a cross-language conformance suite shared with the TypeScript port.
+Project-URL: Homepage, https://github.com/leeloc1809/vn-toolkit
+Project-URL: Repository, https://github.com/leeloc1809/vn-toolkit
+Project-URL: Issues, https://github.com/leeloc1809/vn-toolkit/issues
+License: MIT
+License-File: LICENSE
+Keywords: diacritics,i18n,nfc,nfd,normalization,search,slugify,unicode,vietnam,vietnamese
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Natural Language :: Vietnamese
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# vn-text
+
+Correct Unicode primitives for Vietnamese text. **No runtime dependencies** —
+standard library only.
+
+```bash
+pip install vn-text
+```
+
+```python
+from vn_text import fold, deaccent, strip_stroke, is_vietnamese
+
+fold('Đặng Minh Anh')     # 'dang minh anh'
+fold('Hà Nội, Việt Nam!')  # 'ha noi viet nam'
+deaccent('Tiếng Việt')    # 'Tieng Viet'
+strip_stroke('Đặng')      # 'Dặng'
+is_vietnamese('Nguyễn')   # True
+```
+
+## Why
+
+Vietnamese `Đ` (U+0110) has no canonical decomposition, so NFD cannot fold it.
+The usual "NFD then strip combining marks" recipe leaves `Đặng Minh` as
+`Đang Minh` — and nobody types `Đang` to find `Đặng`, so search quietly returns
+too few results. The tempting blanket fix also turns the Icelandic letter `Ð`
+(U+00D0) into `D` and corrupts those names.
+
+`strip_stroke` handles U+0110 and U+0111 explicitly, and nothing else.
+
+## Validation
+
+Every function is checked against the shared conformance suite at
+[`conformance/vn-text-1.0.0.json`](../../conformance/vn-text-1.0.0.json) — 156
+hand-authored cases that the TypeScript port also consumes, so the two
+implementations cannot drift apart.
+
+The test file runs with or without pytest, because a library whose correctness
+guarantee depends on a test runner is a library you cannot check on a machine
+that has no test runner:
+
+```bash
+python packages/vn-text-py/tests/test_vn_text.py   # stdlib only, nothing to install
+
+pytest packages/vn-text-py/tests -q                     # if you have it
+```
+
+## API parity
+
+`snake_case` is the Python convention, but `stripStroke` and `isVietnamese` are
+exported as aliases so a call site can move between the Python and TypeScript
+ports without renaming.
+
+## Known port risk
+
+`fold()` classifies characters using `str.isalnum()`, while the TypeScript port
+uses `\p{L}\p{N}`. These agree across all 127 distinct characters in the current
+corpus, measured by `scripts/verify-port-parity.mjs`. They are not guaranteed to
+agree for every code point; the conformance suite is how a divergence gets
+caught.
+
+## Scope
+
+Deliberately narrow. Tone-mark canonicalisation (`hóa` vs `hòa`) and fuzzy
+matching for unaccented-keyboard input are not here. Sorting is not here
+either -- it is [`vn-collate`](../vn-collate-py), which shares this
+repository's conformance infrastructure. See the
+[root README](../../README.md#what-this-is-not) for the reasoning.
+
+## Licence
+
+MIT

vn_text-0.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+vn_text/__init__.py,sha256=3Qg0FKvvCSb2ZONeigBog2dCkkltzexCip99AfTOUHI,1122
+vn_text/_unicode.py,sha256=TuP33WgIWgbZBoc0MT_xChl3m8jXm-d5qb6K-uAm55I,2922
+vn_text/core.py,sha256=TFvX66DTXF6T9unUbaL24YfJJsA6RdwcvIe3CzPdNHk,7225
+vn_text/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+vn_text-0.1.0.dist-info/METADATA,sha256=B41YqxdVqVW4au5t0WboJ1IKwBKsw6R_f-lZIKPUwc4,3683
+vn_text-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+vn_text-0.1.0.dist-info/licenses/LICENSE,sha256=jBDksFrsccC6mMMWfqpJtTCSLSe1WQXAJW0I1K99mB0,1080
+vn_text-0.1.0.dist-info/RECORD,,

vn_text-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

vn_text-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 vn-toolkit contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.