sonatoki 0.5.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

sonatoki/Configs.py

@@ -1,6 +1,6 @@
 # STL
 from copy import deepcopy
-from typing import List, Type, TypedDict
+from typing import Set, List, Type, TypedDict, cast
 
 # PDM
 from typing_extensions import NotRequired
@@ -18,6 +18,7 @@ from sonatoki.Filters import (
     NimiKuLili,
     NimiKuSuli,
     ProperName,
+    Phonotactic,
     Punctuation,
     LongSyllabic,
     Miscellaneous,
@@ -102,6 +103,40 @@ CorpusConfig: IloConfig = {
     "scorer": SoftScaling,
     "passing_score": 0.8,
 }
+
+# TODO: create a mechanism to omit tokens from a filter with more granularity
+__corpus_tokens_dict: Set[str] = cast(
+    Set[str],
+    CorpusConfig["scoring_filters"][
+        0
+    ].tokens,  # pyright: ignore[reportAttributeAccessIssue]
+)
+__corpus_tokens_dict -= {
+    # Sandbox words are removed from the CorpusConfig if they appear more frequently in English than Toki Pona by a factor of at least 3.
+    # In this case, all of these appear more often in English by a factor of at least 10.
+    "aka",  # also known as
+    "an",  # article
+    "api",  # API
+    "i",  # 1st person
+    "kana",  # japanese script
+    "me",  # 1st person
+    "ne",  # "no" in several languages
+    "nu",  # "new", now in dutch
+    "se",  # spanish particle, "see"
+    "take",  # acquire, perhaps forcefully or without permission
+    "ten",  # 10
+    "to",  # to, too
+    "u",  # no u
+    "we",  # 1st person plural
+    "wi",  # wii and discussions of syllables
+    "sole",  # singular, of shoe
+    # unexplored candidates for removal
+    # "omen",  # ominous
+    # "papa",  # father
+    # "lo",  # "lo" and "loo"
+    # "ewe",  # sheep
+    # "pa",  # father- eh?
+}
 """Mimics the previous implementation of ilo pi toki pona taso."""
 LazyConfig: IloConfig = {
     "preprocessors": [Emoji, Backticks, URLs, AngleBracketObject, Reference],

sonatoki/Preprocessors.py

@@ -90,6 +90,12 @@ class DiscordEmotes(RegexPreprocessor):
     pattern = re.compile(r"<a?:[a-zA-Z0-9_]{2,}:[0-9]{2,}>")
 
 
+class ColonEmotes(RegexPreprocessor):
+    """Remove colon-marked emotes `:name:`"""
+
+    pattern = re.compile(r":[a-zA-Z0-9_]{2,}:")
+
+
 class DiscordMentions(RegexPreprocessor):
     pattern = re.compile(r"<@[\!\&]?[0-9]{2,}>")
 

sonatoki/constants.py

@@ -553,7 +553,7 @@ FALSE_POS_SYLLABIC = {
     "in",
     "no",
     "some",
-    # "papa",
+    # "papa",  # now in sandbox
     "on",
     "me",
     "ipa",
@@ -591,7 +591,7 @@ FALSE_POS_SYLLABIC = {
     "oposite",
     "anime",
     "potato",
-    # "japan",
+    "japan",
     "nose",
     "kilo",
     "alone",
@@ -629,17 +629,20 @@ FALSE_POS_SYLLABIC = {
     "awaken",
     "eliminate",
     "elite",
-    "misuse",
     "emanate",
     "iluminate",
     "imense",
     "imitate",
+    "injoke",
     "insane",
     "insolate",
     "insulate",
     "intense",
     "lemon",
     "manipulate",
+    "misuse",
+    "ne",  # "no" in many other languages
+    "wana",
 }
 
 FALSE_POS_ALPHABETIC: Set[str] = {