sonatoki 0.5.2__tar.gz → 0.5.3__tar.gz

{sonatoki-0.5.2 → sonatoki-0.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sonatoki
-Version: 0.5.2
+Version: 0.5.3
 Summary: ilo li moku e toki li pana e sona ni: ni li toki ala toki pona?
 Author-Email: "jan Kekan San (@gregdan3)" <gregory.danielson3@gmail.com>
 License: AGPL-3.0-or-later

{sonatoki-0.5.2 → sonatoki-0.5.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sonatoki"
-version = "0.5.2"
+version = "0.5.3"
 description = "ilo li moku e toki li pana e sona ni: ni li toki ala toki pona?"
 authors = [
     { name = "jan Kekan San (@gregdan3)", email = "gregory.danielson3@gmail.com" },

{sonatoki-0.5.2 → sonatoki-0.5.3}/src/sonatoki/Configs.py

@@ -112,16 +112,30 @@ __corpus_tokens_dict: Set[str] = cast(
     ].tokens,  # pyright: ignore[reportAttributeAccessIssue]
 )
 __corpus_tokens_dict -= {
-    "an",
-    "i",
-    "me",
-    "ne",
-    "se",
-    "take",
-    "ten",
-    "to",
-    "u",
-    "we",
+    # Sandbox words are removed from the CorpusConfig if they appear more frequently in English than Toki Pona by a factor of at least 3.
+    # In this case, all of these appear more often in English by a factor of at least 10.
+    "aka",  # also known as
+    "an",  # article
+    "api",  # API
+    "i",  # 1st person
+    "kana",  # japanese script
+    "me",  # 1st person
+    "ne",  # "no" in several languages
+    "nu",  # "new", now in dutch
+    "se",  # spanish particle, "see"
+    "take",  # acquire, perhaps forcefully or without permission
+    "ten",  # 10
+    "to",  # to, too
+    "u",  # no u
+    "we",  # 1st person plural
+    "wi",  # wii and discussions of syllables
+    "sole",  # singular, of shoe
+    # unexplored candidates for removal
+    # "omen",  # ominous
+    # "papa",  # father
+    # "lo",  # "lo" and "loo"
+    # "ewe",  # sheep
+    # "pa",  # father- eh?
 }
 """Mimics the previous implementation of ilo pi toki pona taso."""
 LazyConfig: IloConfig = {

{sonatoki-0.5.2 → sonatoki-0.5.3}/tests/test_properties.py

@@ -1,6 +1,6 @@
 # PDM
 import hypothesis.strategies as st
-from hypothesis import given
+from hypothesis import given, assume
 
 # LOCAL
 from sonatoki.Filters import (
@@ -54,11 +54,13 @@ def test_ku_filters_non_overlap(s: str):
             | NIMI_LINKU_COMMON
             | NIMI_LINKU_UNCOMMON
             | NIMI_LINKU_OBSCURE
-            | NIMI_LINKU_SANDBOX - {"su"}
+            | NIMI_LINKU_SANDBOX
         )
     )
 )
 def test_linku_filters_non_overlap(s: str):
+    _ = assume(s != "su")
+
     s = Lowercase.clean(s)
     s = ConsecutiveDuplicates.clean(s)