sonatoki 0.3.1__py3-none-any.whl → 0.3.2__py3-none-any.whl

sonatoki/Configs.py

@@ -5,17 +5,17 @@ from typing import List, Type, TypedDict
 # LOCAL
 from sonatoki.Filters import (
     Filter,
-    NimiPu,
     Numeric,
-    OrFilter,
     Syllabic,
     NimiUCSUR,
     Alphabetic,
     ProperName,
-    Phonotactic,
     Punctuation,
+    LongSyllabic,
+    Miscellaneous,
     NimiLinkuCore,
-    NimiPuSynonyms,
+    LongAlphabetic,
+    LongProperName,
     OrMemberFilter,
     NimiLinkuCommon,
     NimiLinkuObscure,
@@ -28,12 +28,9 @@ from sonatoki.Cleaners import Cleaner, ConsecutiveDuplicates
 from sonatoki.Tokenizers import Tokenizer, WordTokenizer
 from sonatoki.Preprocessors import (
     URLs,
+    Backticks,
     Reference,
     Preprocessor,
-    DiscordEmotes,
-    DiscordSpecial,
-    DiscordChannels,
-    DiscordMentions,
     AngleBracketObject,
 )
 
@@ -62,14 +59,14 @@ BaseConfig: IloConfig = {
 
 
 PrefConfig: IloConfig = {
-    "preprocessors": [URLs, Reference],
+    "preprocessors": [Backticks, URLs, Reference],
     "cleaners": [ConsecutiveDuplicates],
-    "ignoring_filters": [Numeric, Punctuation, EnglishIgnorables],
+    "ignoring_filters": [Numeric, Punctuation],
     "scoring_filters": [
-        OrMemberFilter(NimiLinkuCore, NimiLinkuCommon, NimiUCSUR),
-        Syllabic,
-        ProperName,
-        Alphabetic,
+        OrMemberFilter(NimiLinkuCore, NimiLinkuCommon, NimiUCSUR, Miscellaneous),
+        LongSyllabic,
+        LongProperName,
+        LongAlphabetic,
     ],
     "scorer": SoftScaling,
     "passing_score": 0.8,
@@ -77,9 +74,9 @@ PrefConfig: IloConfig = {
 }
 
 CorpusConfig: IloConfig = {
-    "preprocessors": [URLs, AngleBracketObject, Reference],
+    "preprocessors": [Backticks, URLs, AngleBracketObject, Reference],
     "cleaners": [ConsecutiveDuplicates],
-    "ignoring_filters": [Numeric, Punctuation, EnglishIgnorables],
+    "ignoring_filters": [Numeric, Punctuation],
     "scoring_filters": [
         OrMemberFilter(
             NimiLinkuCore,
@@ -88,10 +85,11 @@ CorpusConfig: IloConfig = {
             NimiLinkuObscure,
             NimiLinkuSandbox,
             NimiUCSUR,
+            Miscellaneous,
         ),
-        Syllabic,
-        ProperName,
-        Alphabetic,
+        LongSyllabic,
+        LongProperName,
+        LongAlphabetic,
     ],
     "scorer": SoftScaling,
     "passing_score": 0.8,
@@ -99,25 +97,28 @@ CorpusConfig: IloConfig = {
 }
 
 
+"""
+Mimics the previous implementation of ilo pi toki pona taso
+"""
 LazyConfig: IloConfig = {
-    "preprocessors": [URLs],
+    "preprocessors": [Backticks, URLs, AngleBracketObject, Reference],
     "cleaners": [ConsecutiveDuplicates],
     "ignoring_filters": [Numeric, Punctuation],
-    "scoring_filters": [Alphabetic, NimiUCSUR, ProperName],
+    "scoring_filters": [Alphabetic, NimiUCSUR, ProperName, Miscellaneous],
     "scorer": SoftPassFail,
     "passing_score": 0.8,
     "word_tokenizer": WordTokenizer,
 }
 
 DiscordConfig: IloConfig = {
-    "preprocessors": [URLs, AngleBracketObject, Reference],
+    "preprocessors": [Backticks, URLs, AngleBracketObject, Reference],
     "cleaners": [ConsecutiveDuplicates],
     "ignoring_filters": [Numeric, Punctuation, EnglishIgnorables],
     "scoring_filters": [
         OrMemberFilter(NimiLinkuCore, NimiLinkuCommon, NimiUCSUR),
-        Syllabic,
-        ProperName,
-        Alphabetic,
+        LongSyllabic,
+        LongProperName,
+        LongAlphabetic,
     ],
     "scorer": SoftScaling,
     "passing_score": 0.8,

sonatoki/Filters.py

@@ -42,6 +42,33 @@ class Filter(ABC):
         raise NotImplementedError
 
 
+class MinLen(Filter):
+    """
+    Meta filter meant to be inherited by another filter to add a length requirement.
+    Multiple-inherit with `MinLen` as the first argument so `super()` resolves correctly.
+    You may also construct any other filter with a minimum length filter like so:
+
+    ```
+    MinLen(Alphabetic, 3)
+    ```
+    """
+
+    length = 0
+
+    @classmethod
+    @cache(maxsize=None)
+    def filter(cls, token: str) -> bool:
+        if len(token) < cls.length:
+            return False
+        return super().filter(token)
+
+    def __new__(cls, filter: Type[Filter], length_: int) -> Type[Filter]:
+        class MinLenFilter(MinLen, Filter):
+            length = length_
+
+        return MinLenFilter
+
+
 class RegexFilter(Filter):
     pattern: "re.Pattern[str]"
 
@@ -83,11 +110,16 @@ class SubsetFilter(Filter):
 
 
 class Miscellaneous(MemberFilter):
-    tokens = set(ALLOWABLES)
+    tokens = prep_dictionary(ALLOWABLES)
 
 
 class EnglishIgnorables(MemberFilter):
-    tokens = set(IGNORABLES)
+    """NOTE: Not recommended for use.
+    It is better to use a Long* filter such as LongSyllabic than to use this filter.
+    This filter hides words from scoring rather than scoring them poorly,
+    which is more of a benefit than a loss for a word you would like to omit."""
+
+    tokens = prep_dictionary(IGNORABLES)
 
 
 class ProperName(Filter):
@@ -109,6 +141,10 @@ class ProperName(Filter):
         # this will errantly match.
 
 
+class LongProperName(MinLen, ProperName):
+    length = 2  # reject "names" of length 1
+
+
 class NimiPu(MemberFilter):
     tokens = prep_dictionary(NIMI_PU)
 
@@ -166,6 +202,10 @@ class Phonotactic(RegexFilter):
     )
 
 
+class LongPhonotactic(MinLen, Phonotactic):
+    length = 3
+
+
 class Syllabic(RegexFilter):
     """Determines if a given token is syllabically valid Toki Pona (or `n`).
     Words must have correctly ordered vowels and consonants, but the phonotactic
@@ -179,6 +219,10 @@ class Syllabic(RegexFilter):
     )
 
 
+class LongSyllabic(MinLen, Syllabic):
+    length = 3
+
+
 class Alphabetic(SubsetFilter):
     tokens = set(ALPHABET)
 
@@ -187,9 +231,8 @@ class AlphabeticRe(RegexFilter):
     pattern = re.compile(rf"[{ALPHABET}]+", flags=re.IGNORECASE)
 
 
-class TwoOrMoreAlphabetic(Filter):
-    # TODO: alphabetic implementation that ignores single characters
-    pass
+class LongAlphabetic(MinLen, Alphabetic):
+    length = 3
 
 
 class Numeric(Filter):
@@ -266,11 +309,9 @@ class OrFilter:
         if not len(filters) >= 2:
             raise ValueError("Provide at least two Filters to OrFilter.")
 
-        subset_filters = [f for f in filters if issubclass(f, MemberFilter)]
-        if len(subset_filters) >= 2:
-            raise Warning(
-                "Prefer OrMemberFilter for combining two or more MemberFilters."
-            )
+        member_filters = [f for f in filters if issubclass(f, MemberFilter)]
+        if len(member_filters) >= 2:
+            raise Warning("Use OrMemberFilter for combining two or more MemberFilters.")
 
         filter = cls.__generic_filter(*filters)
 
@@ -279,7 +320,7 @@ class OrFilter:
 
 class OrMemberFilter:
     @staticmethod
-    def __subset_filter(*filters: Type[MemberFilter]) -> Type[MemberFilter]:
+    def __member_filter(*filters: Type[MemberFilter]) -> Type[MemberFilter]:
         all_token_sets: List[Set[str]] = [f.tokens for f in filters]
         all_tokens: Set[str] = set().union(*all_token_sets)
 
@@ -291,7 +332,7 @@ class OrMemberFilter:
     def __new__(cls, *filters_: Type[MemberFilter]) -> Type[MemberFilter]:
         if not len(filters_) >= 2:
             raise ValueError("Provide two or more MemberFilters to OrMemberFilter.")
-        filter = cls.__subset_filter(*filters_)
+        filter = cls.__member_filter(*filters_)
         return filter
 
 
@@ -323,6 +364,11 @@ __all__ = [
     "Alphabetic",
     "AndFilter",
     "EnglishIgnorables",
+    "LongAlphabetic",
+    "LongPhonotactic",
+    "LongProperName",
+    "LongSyllabic",
+    "MinLen",
     "NimiLinkuCore",
     "NimiLinkuSandbox",
     "NimiPu",

sonatoki/constants.py

@@ -1,6 +1,6 @@
 # STL
 import json
-from typing import Dict, List
+from typing import Set, Dict, List
 from pathlib import Path
 
 # LOCAL
@@ -383,37 +383,62 @@ LANGUAGE = "english"  # for NLTK
 
 """Commonly occurring strings which are some kind of valid Toki Pona or external token"""
 ALLOWABLES = {
-    "cw",  # Content Warning
     "x",  # ala
     "y",  # anu
     "kxk",  # ken ala ken
     "wxw",  # wile ala wile
 }
 
-IGNORABLES = {
-    # o, e, n are not here bc they're not frequently problematic in english messages
-    "a",
-    "am",
-    "an",
-    "i",
-    "in",
-    "is",
-    "l",  # they'll
-    "m",  # i'm
-    "me",
-    "no",
-    "s",  # let's
-    "so",
-    "t",  # don't
-    "to",
-    "u",  # you
-    "we",
-    "un",  # un-
-    "use",
+PHONOMATCHES = {
+    # "a",  # ignore
+    # "an",  # against
+    # "i",  # against
+    # "in",  # against
     "some",
-    "like",
+    "like",  # against
+    # "me",  # against
+    # "no",  # against
+    # "on",  # against
+    # "se",  # against
+    # "so",  # against
+    # "some",  # against
+    "to",  # ignore
+    # "u",  # against
+    # "un",  # against
+    "use",  # against
+    # "we",  # against
 }
 
+ALPHABETIC_MATCHES = PHONOMATCHES | {
+    "a",
+    # "am",
+    # "as",
+    # "at",
+    # "aw",  # aww
+    # "ek",  # eek
+    # "ew",
+    # "ik",
+    # "il",  # ill
+    # "im",
+    # "im",
+    # "ip",
+    # "is",
+    # "it",
+    # "l",  # they'll
+    # "m",  # i'm
+    # "ok",
+    # "op",
+    # "ow",
+    # "s",  # let's
+    # "t",  # don't
+    # "up",
+    # "us",
+    # "ut",
+    # "uw",
+}
+
+IGNORABLES = PHONOMATCHES | ALPHABETIC_MATCHES
+
 UCSUR_RANGES = [
     "\\U000F1900-\\U000F1977",  # pu
     "\\U000F1978-\\U000F1988",  # ku suli
@@ -426,14 +451,14 @@ NIMI_UCSUR = find_unicode_chars(UCSUR_RANGES)
 # NIMI_PU_ALE_UCSUR_RANGES = NIMI_PU_UCSUR_RANGES + ["\\U000F1978-\\U000F197A"]
 
 
-def category_helper(data: Dict[str, Dict[str, str]], key: str, value: str) -> List[str]:
-    return [d["word"] for d in data.values() if d[key] == value]
+def category_helper(data: Dict[str, Dict[str, str]], key: str, value: str) -> Set[str]:
+    return {d["word"] for d in data.values() if d[key] == value}
 
 
 with open(LINKU) as f:
     linku: Dict[str, Dict[str, str]] = json.loads(f.read())
-    NIMI_PU: List[str] = category_helper(linku, "book", "pu")
-    NIMI_PU_SYNONYMS: List[str] = ["namako", "kin", "oko"]
+    NIMI_PU = category_helper(linku, "book", "pu")
+    NIMI_PU_SYNONYMS = {"namako", "kin", "oko"}
 
     NIMI_KU_SULI = category_helper(linku, "book", "ku suli")
     NIMI_KU_LILI = category_helper(linku, "book", "ku lili")
@@ -445,7 +470,7 @@ with open(LINKU) as f:
 
 with open(SANDBOX) as f:
     sandbox: Dict[str, Dict[str, str]] = json.loads(f.read())
-    NIMI_LINKU_SANDBOX: List[str] = [d["word"] for d in sandbox.values()]
+    NIMI_LINKU_SANDBOX = {d["word"] for d in sandbox.values()}
 
 del linku
 del sandbox

{sonatoki-0.3.1.dist-info → sonatoki-0.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sonatoki
-Version: 0.3.1
+Version: 0.3.2
 Summary: ilo li moku e toki li pana e sona ni: ni li toki ala toki pona?
 Author-Email: "jan Kekan San (@gregdan3)" <gregory.danielson3@gmail.com>
 License: AGPL-3.0-or-later

{sonatoki-0.3.1.dist-info → sonatoki-0.3.2.dist-info}/RECORD

@@ -1,18 +1,18 @@
-sonatoki-0.3.1.dist-info/METADATA,sha256=nWomuM-AeE98VwnWen7qffNclw8emxAf-oFtXwba8wI,6341
-sonatoki-0.3.1.dist-info/WHEEL,sha256=vnE8JVcI2Wz7GRKorsPArnBdnW2SWKWGow5gu5tHlRU,90
-sonatoki-0.3.1.dist-info/licenses/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
+sonatoki-0.3.2.dist-info/METADATA,sha256=9cnhaaYFLxN3uaubD0jfTAU_CC9wUGtzho4fs1UGLFc,6341
+sonatoki-0.3.2.dist-info/WHEEL,sha256=vnE8JVcI2Wz7GRKorsPArnBdnW2SWKWGow5gu5tHlRU,90
+sonatoki-0.3.2.dist-info/licenses/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
 sonatoki/Cleaners.py,sha256=m0j1a1vs9Mdqp724r9Xfh1Y_tyP6GYCkihv8rH8m7lA,1871
-sonatoki/Configs.py,sha256=NS1_esoDNna8LyH_9bPMkxbo2sMSilYhG1PwYLdq6L8,3402
-sonatoki/Filters.py,sha256=-j5xSZ8URjqalQVGMBabMvJ5ofZWazfN7YPfXkM_4uQ,9429
+sonatoki/Configs.py,sha256=o_uFp-Z6sbhbMi8drgQTkdu8S5LaTr0Xnns6Cg0cHSY,3548
+sonatoki/Filters.py,sha256=-7zIV_IBsbASR7pF5WuoABNtBW5a7L135Ev_Rrn35o4,10664
 sonatoki/Preprocessors.py,sha256=aMXXuFBDlJudvzvukvCa7BixuROXXEb62un7I-TGOGs,4441
 sonatoki/Scorers.py,sha256=W-1uYiqjsDejJzoe592ixs7wHazjJXPhuo-41zuJ26U,3643
 sonatoki/Tokenizers.py,sha256=So5_Tu6J98MD3yVcwB_X3lw2uMG0TN6XHcTbQjFCu5Q,4254
 sonatoki/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sonatoki/__main__.py,sha256=6xc-wIrrFo9wTyn4zRQNAmqwmJBtVvCMwV-CrM-hueA,82
-sonatoki/constants.py,sha256=XTFmEcnLBXwdYXjTq_EuW9e_TWLtnNLz2vFCf8m-sz0,12844
+sonatoki/constants.py,sha256=qq1_ZTsVKG_d7nqlJv3a-KS6ZvYwfUSHWA--e0BuyXc,13268
 sonatoki/ilo.py,sha256=yyLgNPI0Hmb4f1BzX6IRHr11FPChfL2xDR_9odlr8_8,3849
 sonatoki/linku.json,sha256=B5KNdhyM5UEfMciROgh1ECHr3i-ASBeMvwrkzNJX47c,271013
 sonatoki/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sonatoki/sandbox.json,sha256=hx6LRsfvmmTtqXcXIyCsfSaGK3DZ-GCdbM8xhZQBHoA,77650
 sonatoki/utils.py,sha256=OMaRyoNvKGKYQCBDjQyaCI58-wMpQ0wrrNjTJKsEZ9Y,3550
-sonatoki-0.3.1.dist-info/RECORD,,
+sonatoki-0.3.2.dist-info/RECORD,,