sonatoki 0.8.4__py3-none-any.whl → 0.9.0__py3-none-any.whl

sonatoki/Configs.py

@@ -9,14 +9,17 @@ from sonatoki.types import Number
 from sonatoki.Filters import (
     Or,
     And,
+    Len,
     Not,
     Filter,
     PuName,
     Numeric,
+    Syllabic,
     NimiUCSUR,
     Alphabetic,
     NimiKuLili,
     NimiKuSuli,
+    ProperName,
     Punctuation,
     LongSyllabic,
     Miscellaneous,
@@ -29,7 +32,7 @@ from sonatoki.Filters import (
     NimiLinkuUncommon,
     FalsePosAlphabetic,
 )
-from sonatoki.Scorers import Scorer, PassFail, SoftScaling, SoftPassFail
+from sonatoki.Scorers import Scorer, Soften, Voting, PassFail, SoftScaling, SoftPassFail
 from sonatoki.Cleaners import Cleaner, ConsecutiveDuplicates
 from sonatoki.Tokenizers import Tokenizer, WordTokenizerRe
 from sonatoki.Preprocessors import (
@@ -62,8 +65,8 @@ __DICT_PHONOMATCHES = {
     "we",  # 1st person plural, english
     "wi",  # wii and discussions of syllables
     "sole",  # singular, of shoe
+    "omen",  # ominous
     # unexplored candidates for removal
-    # "omen",  # ominous
     # "papa",  # father
     # "lo",  # "lo" and "loo"
     # "ewe",  # sheep
@@ -99,11 +102,11 @@ PrefConfig: IloConfig = {
     "cleaners": [ConsecutiveDuplicates],
     "ignoring_filters": [Numeric, Punctuation],
     "scoring_filters": [
-        Or(NimiLinkuByUsage(30), NimiUCSUR),
-        And(LongSyllabic, Not(FalsePosSyllabic)),
+        Len(Or(NimiLinkuByUsage(30), NimiUCSUR), max=15),
+        Len(And(Syllabic, Not(FalsePosSyllabic)), min=3, max=24),
         # NOTE: These are allowed to pass name and alphabetic below, because they *could* be wrong
-        LongProperName,
-        And(LongAlphabetic, Not(FalsePosAlphabetic)),
+        Len(ProperName, min=2, max=24),
+        Len(And(Alphabetic, Not(FalsePosAlphabetic)), min=3, max=24),
     ],
     "scorer": SoftScaling,
     "passing_score": 0.8,
@@ -114,15 +117,18 @@ CorpusConfig: IloConfig = {
     "cleaners": [ConsecutiveDuplicates],
     "ignoring_filters": [Numeric, Punctuation],
     "scoring_filters": [
-        Or(
-            # awkward but efficient syntax
-            NimiLinkuByUsage(0)(sub=__DICT_PHONOMATCHES),
-            NimiUCSUR,
-            Miscellaneous,
+        Len(
+            Or(
+                # awkward but efficient syntax
+                NimiLinkuByUsage(0)(sub=__DICT_PHONOMATCHES),
+                NimiUCSUR,
+                Miscellaneous,
+            ),
+            max=19,
         ),
-        And(LongSyllabic, Not(FalsePosSyllabic)),
-        LongProperName,
-        And(LongAlphabetic, Not(FalsePosAlphabetic)),
+        Len(And(Syllabic, Not(FalsePosSyllabic)), min=3, max=24),
+        Len(ProperName, min=2, max=24),
+        Len(And(Alphabetic, Not(FalsePosAlphabetic)), min=3, max=24),
     ],
     "scorer": SoftScaling,
     "passing_score": 0.8,

sonatoki/Filters.py

@@ -7,7 +7,7 @@ from functools import lru_cache as cache  # cache comes in 3.9
 
 # PDM
 import regex
-from typing_extensions import override
+from typing_extensions import override, deprecated
 
 # LOCAL
 from sonatoki.types import LinkuBooks, LinkuUsageDate, LinkuUsageCategory
@@ -41,6 +41,7 @@ class Filter(ABC):
         raise NotImplementedError
 
 
+@deprecated("Use sonatoki.Filters.Len instead")
 class MinLen(Filter):
     """
     Meta filter meant to be inherited by another filter to add a length requirement.
@@ -62,12 +63,54 @@ class MinLen(Filter):
         return super().filter(token)
 
     def __new__(cls, filter: Type[Filter], length_: int) -> Type[Filter]:
-        class MinLenFilter(MinLen, Filter):
+        class MinLenFilter(MinLen, filter):
             length = length_
 
         return MinLenFilter
 
 
+class Len(Filter):
+    """Meta filter to be inherited by another filter to add any length
+    requirement. A bound will only be considered if it is non-zero, so you may
+    omit a minimum length or a maximum length to bound only one of them.
+
+    If inherited when defining a class, `Len` must be the first argument so `super()` resolves correctly.
+
+    To add minimum or maximum length requirements when defining a class:
+    ```
+    class LongAlphabetic(Len, Alphabetic):
+        minlen = 3
+        maxlen = 20
+    ```
+
+    You may also construct any other filter with a minimum length filter like so:
+    ```
+    Len(Alphabetic, min=3, max=20)
+    ```
+    """
+
+    minlen = 0
+    maxlen = 0
+
+    @classmethod
+    @cache(maxsize=None)
+    def filter(cls, token: str) -> bool:
+        tokenlen = len(token)
+
+        if cls.minlen and tokenlen < cls.minlen:
+            return False
+        if cls.maxlen and tokenlen > cls.maxlen:
+            return False
+        return super().filter(token)
+
+    def __new__(cls, filter: Type[Filter], min: int = 0, max: int = 0) -> Type[Filter]:
+        class LenFilter(Len, filter):
+            minlen = min
+            maxlen = max
+
+        return LenFilter
+
+
 class RegexFilter(Filter):
     pattern: "re.Pattern[str]"
 
@@ -183,8 +226,8 @@ class PuName(Filter):
         # this will errantly match.
 
 
-class LongProperName(MinLen, ProperName):
-    length = 2  # reject "names" of length 1
+class LongProperName(Len, ProperName):
+    minlen = 2  # reject "names" of length 1
 
 
 class NimiLinkuByUsage:
@@ -252,8 +295,8 @@ class Phonotactic(RegexFilter):
     )
 
 
-class LongPhonotactic(MinLen, Phonotactic):
-    length = 3
+class LongPhonotactic(Len, Phonotactic):
+    minlen = 3
 
 
 class Syllabic(RegexFilter):
@@ -271,8 +314,8 @@ class Syllabic(RegexFilter):
     )
 
 
-class LongSyllabic(MinLen, Syllabic):
-    length = 3
+class LongSyllabic(Len, Syllabic):
+    minlen = 3
 
 
 class Alphabetic(SubsetFilter):
@@ -283,8 +326,8 @@ class AlphabeticRe(RegexFilter):
     pattern = re.compile(rf"[{ALPHABET}]+", flags=re.IGNORECASE)
 
 
-class LongAlphabetic(MinLen, Alphabetic):
-    length = 3
+class LongAlphabetic(Len, Alphabetic):
+    minlen = 3
 
 
 class Numeric(Filter):
@@ -448,15 +491,26 @@ class Not(Filter):
         return NotFilter
 
 
+class Pass(Filter):
+    @classmethod
+    @override
+    @cache(maxsize=None)
+    def filter(cls, token: str) -> bool:
+        return True
+
+
+class Fail(Not, Pass): ...
+
+
 __all__ = [
     "Alphabetic",
     "And",
     "FalsePosSyllabic",
+    "Len",
     "LongAlphabetic",
     "LongPhonotactic",
     "LongProperName",
     "LongSyllabic",
-    "MinLen",
     "NimiLinkuCore",
     "NimiLinkuSandbox",
     "NimiPu",

sonatoki/Scorers.py

@@ -8,7 +8,7 @@ from typing_extensions import override
 
 # LOCAL
 from sonatoki.types import Number, Scorecard
-from sonatoki.Filters import Filter
+from sonatoki.Filters import Pass, Filter
 
 
 class Scorer(ABC):
@@ -112,6 +112,67 @@ class Scaling(Scorer):
         return total_score / max_score if max_score else 0
 
 
+class Voting(Scaling):
+    """Derives from `Scaling` in assigning scores from 0 to 1 based on how soon
+    a filter matches, with the first filter scoring a 1. However, after all
+    scores are derived, each token scoring 0 is given a is given an opportunity
+    to score based on its nearest 3 neighbors.
+
+    If created with a Filter, tokens must also pass that filter to be
+    considered for voting.
+    """
+
+    prereq: Type[Filter] = Pass
+    threshold: int = 0
+
+    def __new__(cls, filter: Type[Filter], threshold_: int):
+        class AnonVoting(Voting):
+            prereq = filter
+            threshold = threshold_
+
+        return AnonVoting
+
+    @classmethod
+    @override
+    def score(cls, tokens: List[str], filters: List[Type[Filter]]) -> Number:
+        if not tokens:
+            return 1
+
+        if len(tokens) < 4:
+            return super().score(tokens, filters)
+
+        len_filters = len(filters)
+        max_score = len(tokens) * len_filters
+
+        # score_token only emits ints
+        # but the averaging emits floats
+        # it doesn't really matter as long as no score exceeds len_filters
+        scores: List[Number] = []
+        for token in tokens:
+            score = cls.score_token(token, filters, len_filters)
+            scores.append(score)
+
+        # only consider scores from before voting
+        copied_scores = scores[:]
+        for i, (token, score) in enumerate(zip(tokens, copied_scores)):
+            if score > cls.threshold:
+                continue
+            if not cls.prereq.filter(token):
+                continue
+
+            # TODO: this is kinda dumb.
+            # we want to get exactly 3 neighbors, favoring 2 before and 1 after
+            # the way i'm doing this is both bad and slow as hell
+            start = max(i - 2, 0)
+            end = min(i + 1, len(scores) - 1)
+            neighbors = copied_scores[start:i] + copied_scores[i + 1 : end + 1]
+            scores[i] = sum(neighbors) / len(neighbors)
+
+        total_score = sum(scores)
+
+        return total_score / max_score if max_score else 0
+
+
 class SoftPassFail(Soften, PassFail):
     """Same as `PassFail`, but shorter messages are subject to less harsh
     scoring."""
@@ -122,6 +183,11 @@ class SoftScaling(Soften, Scaling):
     scoring."""
 
 
+class SoftVoting(Soften, Voting):
+    """Same as `Voting`, but shorter messages are subject to less harsh
+    scoring."""
+
+
 class SentenceScorer(ABC):
     @classmethod
     @abstractmethod

sonatoki/__main__.py

@@ -60,11 +60,11 @@ def download_json(url: str) -> Dict[str, Any]:
 
 def regen_linku_data():
     data = download_json(LINKU_WORDS)
-    with open(os.path.join(HERE, "linku.json"), "w") as f:
+    with open(os.path.join(HERE, "linku.json"), "w", encoding="utf-8") as f:
         _ = f.write(json.dumps(data))
 
     data = download_json(LINKU_SANDBOX)
-    with open(os.path.join(HERE, "sandbox.json"), "w") as f:
+    with open(os.path.join(HERE, "sandbox.json"), "w", encoding="utf-8") as f:
         _ = f.write(json.dumps(data))
 
 
@@ -96,11 +96,11 @@ def regen_false_negatives():
             continue
 
     # TODO: include short matches or no?
-    with open(os.path.join(HERE, "syllabic.txt"), "w") as f:
+    with open(os.path.join(HERE, "syllabic.txt"), "w", encoding="utf-8") as f:
         syllabic_final = sorted([word + "\n" for word in syllabic_matches])
         f.writelines(syllabic_final)
 
-    with open(os.path.join(HERE, "alphabetic.txt"), "w") as f:
+    with open(os.path.join(HERE, "alphabetic.txt"), "w", encoding="utf-8") as f:
         alphabetic_final = sorted([word + "\n" for word in alphabetic_matches])
         f.writelines(alphabetic_final)
 

sonatoki/constants.py

@@ -648,6 +648,7 @@ FALSE_POS_SYLLABIC = {
     "insolate",
     "insulate",
     "intense",
+    "saluton",
     # "june",
     "lemon",
     "manipulate",
@@ -698,9 +699,9 @@ def linku_data() -> Dict[str, LinkuWord]:
     # NOTE: this does open+read+parse two files each time you construct a filter
     # but i expect users to construct filters only at the start of runtime
     # there is no reason to waste your RAM by leaving the linku data in it
-    with open(LINKU) as f:
+    with open(LINKU, "r", encoding="utf-8") as f:
         linku: Dict[str, LinkuWord] = json.loads(f.read())
-    with open(SANDBOX) as f:
+    with open(SANDBOX, "r", encoding="utf-8") as f:
         sandbox: Dict[str, LinkuWord] = json.loads(f.read())
 
     return {**linku, **sandbox}
@@ -731,10 +732,10 @@ def words_by_usage(
 NIMI_PU_SYNONYMS = {"namako", "kin", "oko"}
 
 
-# with open(SYLLABICS) as f:
+# with open(SYLLABICS, "r", encoding="utf-8") as f:
 #     FALSE_POS_SYLLABIC = {line.strip() for line in f}
 #
-# with open(ALPHABETICS) as f:
+# with open(ALPHABETICS, "r", encoding="utf-8") as f:
 #     FALSE_POS_ALPHABETIC = {line.strip() for line in f}
 
 __all__ = [

{sonatoki-0.8.4.dist-info → sonatoki-0.9.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sonatoki
-Version: 0.8.4
+Version: 0.9.0
 Summary: ilo li moku e toki li pana e sona ni: ni li toki ala toki pona?
 Author-Email: "jan Kekan San (@gregdan3)" <gregory.danielson3@gmail.com>
 License: AGPL-3.0-or-later

{sonatoki-0.8.4.dist-info → sonatoki-0.9.0.dist-info}/RECORD

@@ -1,16 +1,17 @@
-sonatoki-0.8.4.dist-info/METADATA,sha256=Nui-Em5-CwsiOt5mkyhF5bb6WM9VQ6sp9UlENnH5Udw,6893
-sonatoki-0.8.4.dist-info/WHEEL,sha256=rSwsxJWe3vzyR5HCwjWXQruDgschpei4h_giTm0dJVE,90
-sonatoki-0.8.4.dist-info/licenses/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
+sonatoki-0.9.0.dist-info/METADATA,sha256=fg-GTxWTWA71w87R_dRWY07pBioj_034nEGHgxE9_EU,6893
+sonatoki-0.9.0.dist-info/WHEEL,sha256=pM0IBB6ZwH3nkEPhtcp50KvKNX-07jYtnb1g1m6Z4Co,90
+sonatoki-0.9.0.dist-info/entry_points.txt,sha256=6OYgBcLyFCUgeqLgnvMyOJxPCWzgy7se4rLPKtNonMs,34
+sonatoki-0.9.0.dist-info/licenses/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
 sonatoki/Cleaners.py,sha256=x2dT3MpDUfbrHA0EP2D3n1sTiKFFi5jw9ha-1dX973o,1958
-sonatoki/Configs.py,sha256=-R-rTPUJfuSintpvC4UnOF1B9B93-Ooh_jmkZwhKvtk,4669
-sonatoki/Filters.py,sha256=rBEJrY_R6koFpoYl4yfo_9UR-i21HbvlUF0ORg1g0WE,13411
+sonatoki/Configs.py,sha256=dwWe1kmgXA9CmiF2KBKhkQfPYOs_TQHjXb5DW9wuMBA,4899
+sonatoki/Filters.py,sha256=8HAtR6_Rk6GPboaS_MHwSjZBJxYnAA8kYbRPI0eR6sM,14823
 sonatoki/Preprocessors.py,sha256=RmzkvPVo6Kdx1rZ5HeR9cTtx6oxpp2iLKrOMCUEqIrM,7107
-sonatoki/Scorers.py,sha256=aCU3p9rD4QOy-uu851FGGw-ARqUCG_l4V_z5rtRL420,5236
+sonatoki/Scorers.py,sha256=qoceRK14-D-xgi5DM290f-PJNOViEzOsRXBdsBbNIWM,7441
 sonatoki/Tokenizers.py,sha256=8lpC70bzXOpHyhVr5bmqpYKmdmQvJdf7X5-Icc9RRCw,5040
 sonatoki/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sonatoki/__main__.py,sha256=6n4kUF80APl6a0jV46h_ncHNuQbrLpZ_nAmiNAakiag,5673
+sonatoki/__main__.py,sha256=QIWRLYS1jb7OBUBK5s8kYoeiMv6MLBlt_I7H7tIVjpU,5745
 sonatoki/alphabetic.txt,sha256=duyqAKilD2vLIr75RShCIAnktNJcGeEoQIk18V6czmg,11702
-sonatoki/constants.py,sha256=Or9VEk19N8Bef14ZRgcP-r0F9qyVByv4CImqpoJ8I6Q,19324
+sonatoki/constants.py,sha256=Akqt8Kw3F9PPdpZ4UbcrTy2BGfIafNTf2YH0D35cO7k,19431
 sonatoki/ilo.py,sha256=Dsn0yagkwjqpAQoCj6mkZ6NqWeanRF2lxNDNoqjWGLo,5993
 sonatoki/linku.json,sha256=d72Dvht-a4gBmdqLLI8mElvo83zSpbxDmxJj05hOudM,295413
 sonatoki/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -18,4 +19,4 @@ sonatoki/sandbox.json,sha256=44csrQDaVtV-n8OyewabX1J9MmUFCsPct5C8E5Xuc58,140197
 sonatoki/syllabic.txt,sha256=HnqY4TrZ3tPcHah3TsvG9F9gjMrnAGdJ8hHJNHyyUPc,1712
 sonatoki/types.py,sha256=zoVJeaDLOPstREiHtoD9pv-AOCsJq2C4_GG3nTYd114,1267
 sonatoki/utils.py,sha256=sT5xLMEj0aLpy8GP92HKblJU1Wt1m8NUlMgCFWB32xQ,2265
-sonatoki-0.8.4.dist-info/RECORD,,
+sonatoki-0.9.0.dist-info/RECORD,,

{sonatoki-0.8.4.dist-info → sonatoki-0.9.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: pdm-backend (2.3.3)
+Generator: pdm-backend (2.4.2)
 Root-Is-Purelib: true
 Tag: py3-none-any

sonatoki-0.9.0.dist-info/entry_points.txt

@@ -0,0 +1,4 @@
+[console_scripts]
+
+[gui_scripts]
+