model2data 1.3.0__tar.gz → 1.3.1__tar.gz

{model2data-1.3.0/model2data.egg-info → model2data-1.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: model2data
-Version: 1.3.0
+Version: 1.3.1
 Summary: Generate analytics-ready datasets from DBML models
 Author: JB Analytica
 License-Expression: MIT

{model2data-1.3.0 → model2data-1.3.1}/model2data/generate/faker.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import random
 import re
+import unicodedata
 import uuid
 from dataclasses import dataclass
 from datetime import datetime, timedelta
@@ -147,9 +148,27 @@ _person_state: dict[str, list[_Person]] = {}
 _address_state: dict[str, list[_Address]] = {}
 
 
+# Letters that NFKD does not take apart, because they are their own letters
+# rather than a base plus an accent. Without these, "ø" and "ß" would simply
+# vanish along with the accents.
+_UNDECOMPOSED_LETTERS = str.maketrans(
+    {"ø": "o", "æ": "ae", "œ": "oe", "ß": "ss", "ł": "l", "đ": "d", "ð": "d", "þ": "th", "ı": "i"}
+)
+
+
 def _slug(value: str) -> str:
-    """Reduce a name to something that can sit inside an email or a username."""
-    return re.sub(r"[^a-z0-9]+", "", value.lower()) or "user"
+    """Reduce a name to something that can sit inside an email or a username.
+
+    Accents are folded, not dropped. Stripping them outright turned `Aimée` into
+    `aime` and `Müller` into `mller` -- not that person's name, and conspicuously
+    broken in exactly the European locales the locale option exists to serve.
+    NFKD splits most accented letters into a base letter plus a combining mark,
+    which encoding to ASCII then discards; the letters that do not decompose are
+    mapped first.
+    """
+    folded = value.lower().translate(_UNDECOMPOSED_LETTERS)
+    ascii_only = unicodedata.normalize("NFKD", folded).encode("ascii", "ignore").decode("ascii")
+    return re.sub(r"[^a-z0-9]+", "", ascii_only) or "user"
 
 
 # Resolved once per locale, not once per row. Both of these are constant for a

{model2data-1.3.0 → model2data-1.3.1/model2data.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: model2data
-Version: 1.3.0
+Version: 1.3.1
 Summary: Generate analytics-ready datasets from DBML models
 Author: JB Analytica
 License-Expression: MIT

{model2data-1.3.0 → model2data-1.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "model2data"
-version = "1.3.0"
+version = "1.3.1"
 description = "Generate analytics-ready datasets from DBML models"
 readme = "README_PYPI.md"
 requires-python = ">=3.10"

{model2data-1.3.0 → model2data-1.3.1}/tests/test_row_identity.py

@@ -370,3 +370,45 @@ class TestThinLocales:
         # picked once beats a different random country on every row.
         assert len(countries) == 1
         assert countries != {""}
+
+
+class TestNameFolding:
+    def test_accents_are_folded_not_dropped(self):
+        # Dropping them turned Aimée into "aime" and Müller into "mller" -- not
+        # that person's name, and most visible in exactly the European locales
+        # the locale option exists to serve.
+        assert faker_module._slug("Aimée") == "aimee"
+        assert faker_module._slug("Müller") == "muller"
+        assert faker_module._slug("Björn") == "bjorn"
+
+    def test_letters_that_do_not_decompose_are_mapped(self):
+        # NFKD leaves these intact because they are their own letters, not a
+        # base plus an accent, so they would vanish with the combining marks.
+        assert faker_module._slug("Søren") == "soren"
+        assert faker_module._slug("Weiß") == "weiss"
+        assert faker_module._slug("Łukasz") == "lukasz"
+        assert faker_module._slug("Æther") == "aether"
+
+    def test_punctuation_and_spacing_still_go(self):
+        assert faker_module._slug("O'Brien") == "obrien"
+        assert faker_module._slug("Van Der Berg") == "vanderberg"
+
+    def test_a_name_with_no_latin_letters_falls_back(self):
+        # Documented limitation rather than a target: deriving an address from a
+        # CJK name needs romanization, and Faker exposes no romanized first/last
+        # pair to build one from. Such a locale gets "user", disambiguated by the
+        # unique suffixing, which is meaningless but at least stable and unique.
+        assert faker_module._slug("日本") == "user"
+
+    def test_generated_emails_are_ascii_in_an_accented_locale(self):
+        df = generate_data_from_dbml(
+            tables={"customers": _person_table()},
+            refs=[],
+            base_rows=40,
+            seed=31,
+            locale="fr_FR",
+        )["customers"]
+        for _, row in df.iterrows():
+            _assert_row_is_one_person(row)
+            # An address has to be usable as an address, whatever the name says.
+            row["email"].encode("ascii")