sonatoki 0.8.4__py3-none-any.whl → 0.9.1__py3-none-any.whl

sonatoki/Configs.py

@@ -9,14 +9,17 @@ from sonatoki.types import Number
 from sonatoki.Filters import (
     Or,
     And,
+    Len,
     Not,
     Filter,
     PuName,
     Numeric,
+    Syllabic,
     NimiUCSUR,
     Alphabetic,
     NimiKuLili,
     NimiKuSuli,
+    ProperName,
     Punctuation,
     LongSyllabic,
     Miscellaneous,
@@ -29,7 +32,7 @@ from sonatoki.Filters import (
     NimiLinkuUncommon,
     FalsePosAlphabetic,
 )
-from sonatoki.Scorers import Scorer, PassFail, SoftScaling, SoftPassFail
+from sonatoki.Scorers import Scorer, Soften, Voting, PassFail, SoftScaling, SoftPassFail
 from sonatoki.Cleaners import Cleaner, ConsecutiveDuplicates
 from sonatoki.Tokenizers import Tokenizer, WordTokenizerRe
 from sonatoki.Preprocessors import (
@@ -49,21 +52,21 @@ __DICT_PHONOMATCHES = {
     "an",  # article
     "api",  # API
     "i",  # 1st person
+    "je",  # 1st person pronoun, french
     "kana",  # japanese script
     "me",  # 1st person singular, english
     "ne",  # "no" in several languages
     "nu",  # "new" in english, "now" in dutch
+    "omen",  # ominous
     "se",  # spanish particle, english "see"
+    "sole",  # singular, of shoe
     "take",  # acquire, perhaps forcefully or without permission
     "ten",  # 10
     "to",  # to, too
-    "je",  # 1st person pronoun, french
     "u",  # no u
     "we",  # 1st person plural, english
     "wi",  # wii and discussions of syllables
-    "sole",  # singular, of shoe
     # unexplored candidates for removal
-    # "omen",  # ominous
     # "papa",  # father
     # "lo",  # "lo" and "loo"
     # "ewe",  # sheep
@@ -99,11 +102,11 @@ PrefConfig: IloConfig = {
     "cleaners": [ConsecutiveDuplicates],
     "ignoring_filters": [Numeric, Punctuation],
     "scoring_filters": [
-        Or(NimiLinkuByUsage(30), NimiUCSUR),
-        And(LongSyllabic, Not(FalsePosSyllabic)),
+        Len(Or(NimiLinkuByUsage(30), NimiUCSUR), max=15),
+        Len(And(Syllabic, Not(FalsePosSyllabic)), min=3, max=24),
         # NOTE: These are allowed to pass name and alphabetic below, because they *could* be wrong
-        LongProperName,
-        And(LongAlphabetic, Not(FalsePosAlphabetic)),
+        Len(ProperName, min=2, max=24),
+        Len(And(Alphabetic, Not(FalsePosAlphabetic)), min=3, max=24),
     ],
     "scorer": SoftScaling,
     "passing_score": 0.8,
@@ -114,15 +117,18 @@ CorpusConfig: IloConfig = {
     "cleaners": [ConsecutiveDuplicates],
     "ignoring_filters": [Numeric, Punctuation],
     "scoring_filters": [
-        Or(
-            # awkward but efficient syntax
-            NimiLinkuByUsage(0)(sub=__DICT_PHONOMATCHES),
-            NimiUCSUR,
-            Miscellaneous,
+        Len(
+            Or(
+                # awkward but efficient syntax
+                NimiLinkuByUsage(0)(sub=__DICT_PHONOMATCHES),
+                NimiUCSUR,
+                Miscellaneous,
+            ),
+            max=19,
         ),
-        And(LongSyllabic, Not(FalsePosSyllabic)),
-        LongProperName,
-        And(LongAlphabetic, Not(FalsePosAlphabetic)),
+        Len(And(Syllabic, Not(FalsePosSyllabic)), min=3, max=24),
+        Len(ProperName, min=2, max=24),
+        Len(And(Alphabetic, Not(FalsePosAlphabetic)), min=3, max=24),
     ],
     "scorer": SoftScaling,
     "passing_score": 0.8,

sonatoki/Filters.py

@@ -7,7 +7,7 @@ from functools import lru_cache as cache  # cache comes in 3.9
 
 # PDM
 import regex
-from typing_extensions import override
+from typing_extensions import override, deprecated
 
 # LOCAL
 from sonatoki.types import LinkuBooks, LinkuUsageDate, LinkuUsageCategory
@@ -41,6 +41,7 @@ class Filter(ABC):
         raise NotImplementedError
 
 
+@deprecated("Use sonatoki.Filters.Len instead")
 class MinLen(Filter):
     """
     Meta filter meant to be inherited by another filter to add a length requirement.
@@ -62,12 +63,54 @@ class MinLen(Filter):
         return super().filter(token)
 
     def __new__(cls, filter: Type[Filter], length_: int) -> Type[Filter]:
-        class MinLenFilter(MinLen, Filter):
+        class MinLenFilter(MinLen, filter):
             length = length_
 
         return MinLenFilter
 
 
+class Len(Filter):
+    """Meta filter to be inherited by another filter to add any length
+    requirement. A bound will only be considered if it is non-zero, so you may
+    omit a minimum length or a maximum length to bound only one of them.
+
+    If inherited when defining a class, `Len` must be the first argument so `super()` resolves correctly.
+
+    To add minimum or maximum length requirements when defining a class:
+    ```
+    class LongAlphabetic(Len, Alphabetic):
+        minlen = 3
+        maxlen = 20
+    ```
+
+    You may also construct any other filter with a minimum length filter like so:
+    ```
+    Len(Alphabetic, min=3, max=20)
+    ```
+    """
+
+    minlen = 0
+    maxlen = 0
+
+    @classmethod
+    @cache(maxsize=None)
+    def filter(cls, token: str) -> bool:
+        tokenlen = len(token)
+
+        if cls.minlen and tokenlen < cls.minlen:
+            return False
+        if cls.maxlen and tokenlen > cls.maxlen:
+            return False
+        return super().filter(token)
+
+    def __new__(cls, filter: Type[Filter], min: int = 0, max: int = 0) -> Type[Filter]:
+        class LenFilter(Len, filter):
+            minlen = min
+            maxlen = max
+
+        return LenFilter
+
+
 class RegexFilter(Filter):
     pattern: "re.Pattern[str]"
 
@@ -183,8 +226,8 @@ class PuName(Filter):
         # this will errantly match.
 
 
-class LongProperName(MinLen, ProperName):
-    length = 2  # reject "names" of length 1
+class LongProperName(Len, ProperName):
+    minlen = 2  # reject "names" of length 1
 
 
 class NimiLinkuByUsage:
@@ -252,8 +295,8 @@ class Phonotactic(RegexFilter):
     )
 
 
-class LongPhonotactic(MinLen, Phonotactic):
-    length = 3
+class LongPhonotactic(Len, Phonotactic):
+    minlen = 3
 
 
 class Syllabic(RegexFilter):
@@ -271,8 +314,8 @@ class Syllabic(RegexFilter):
     )
 
 
-class LongSyllabic(MinLen, Syllabic):
-    length = 3
+class LongSyllabic(Len, Syllabic):
+    minlen = 3
 
 
 class Alphabetic(SubsetFilter):
@@ -283,8 +326,8 @@ class AlphabeticRe(RegexFilter):
     pattern = re.compile(rf"[{ALPHABET}]+", flags=re.IGNORECASE)
 
 
-class LongAlphabetic(MinLen, Alphabetic):
-    length = 3
+class LongAlphabetic(Len, Alphabetic):
+    minlen = 3
 
 
 class Numeric(Filter):
@@ -448,15 +491,26 @@ class Not(Filter):
         return NotFilter
 
 
+class Pass(Filter):
+    @classmethod
+    @override
+    @cache(maxsize=None)
+    def filter(cls, token: str) -> bool:
+        return True
+
+
+class Fail(Not, Pass): ...
+
+
 __all__ = [
     "Alphabetic",
     "And",
     "FalsePosSyllabic",
+    "Len",
     "LongAlphabetic",
     "LongPhonotactic",
     "LongProperName",
     "LongSyllabic",
-    "MinLen",
     "NimiLinkuCore",
     "NimiLinkuSandbox",
     "NimiPu",

sonatoki/Scorers.py

@@ -8,7 +8,7 @@ from typing_extensions import override
 
 # LOCAL
 from sonatoki.types import Number, Scorecard
-from sonatoki.Filters import Filter
+from sonatoki.Filters import Pass, Filter
 
 
 class Scorer(ABC):
@@ -112,6 +112,72 @@ class Scaling(Scorer):
         return total_score / max_score if max_score else 0
 
 
+class Voting(Scaling):
+    """Derives from `Scaling` in assigning scores from 0 to 1 based on the
+    first matching filter out of the list of filters. However, after all scores
+    are derived, each token scoring less than the threshold is assigned the
+    average score of its nearest 3 neighbors. The default threshold is 0.
+
+    If there are 3 or fewer tokens, this scorer is identical to the
+    Scaling scorer.
+
+    If the Voting scorer is created with a Filter, tokens must also
+    match that filter to be considered for voting. For example, the
+    following Voting filter would only check words with a score of 0.3
+    or less that still match the Syllabic filter: `Voting(Syllabic, 0.3)`
+    """
+
+    prereq: Type[Filter] = Pass
+    threshold: int = 0
+
+    def __new__(cls, filter: Type[Filter], threshold_: int = 0) -> Type[Scorer]:
+        class AnonVoting(Voting):
+            prereq = filter
+            threshold = threshold_
+
+        return AnonVoting
+
+    @classmethod
+    @override
+    def score(cls, tokens: List[str], filters: List[Type[Filter]]) -> Number:
+        if not tokens:
+            return 1
+
+        if len(tokens) < 4:
+            return super().score(tokens, filters)
+
+        len_filters = len(filters)
+        max_score = len(tokens) * len_filters
+
+        # score_token only emits ints
+        # but the averaging emits floats
+        # it doesn't really matter as long as no score exceeds len_filters
+        scores: List[Number] = []
+        for token in tokens:
+            score = cls.score_token(token, filters, len_filters)
+            scores.append(score)
+
+        # only consider scores from before voting
+        copied_scores = scores[:]
+        for i, (token, score) in enumerate(zip(tokens, copied_scores)):
+            if score > cls.threshold:
+                continue
+            if not cls.prereq.filter(token):
+                continue
+
+            # TODO: this is kinda dumb.
+            # we want to get exactly 3 neighbors, favoring 2 before and 1 after
+            # the way i'm doing this is both bad and slow as hell
+            start = max(i - 2, 0)
+            end = min(i + 1, len(scores) - 1)
+            neighbors = copied_scores[start:i] + copied_scores[i + 1 : end + 1]
+            scores[i] = sum(neighbors) / len(neighbors)
+
+        total_score = sum(scores)
+
+        return total_score / max_score if max_score else 0
+
+
 class SoftPassFail(Soften, PassFail):
     """Same as `PassFail`, but shorter messages are subject to less harsh
     scoring."""
@@ -122,6 +188,11 @@ class SoftScaling(Soften, Scaling):
     scoring."""
 
 
+class SoftVoting(Soften, Voting):
+    """Same as `Voting`, but shorter messages are subject to less harsh
+    scoring."""
+
+
 class SentenceScorer(ABC):
     @classmethod
     @abstractmethod

sonatoki/__main__.py

@@ -60,11 +60,11 @@ def download_json(url: str) -> Dict[str, Any]:
 
 def regen_linku_data():
     data = download_json(LINKU_WORDS)
-    with open(os.path.join(HERE, "linku.json"), "w") as f:
+    with open(os.path.join(HERE, "linku.json"), "w", encoding="utf-8") as f:
         _ = f.write(json.dumps(data))
 
     data = download_json(LINKU_SANDBOX)
-    with open(os.path.join(HERE, "sandbox.json"), "w") as f:
+    with open(os.path.join(HERE, "sandbox.json"), "w", encoding="utf-8") as f:
         _ = f.write(json.dumps(data))
 
 
@@ -96,11 +96,11 @@ def regen_false_negatives():
             continue
 
     # TODO: include short matches or no?
-    with open(os.path.join(HERE, "syllabic.txt"), "w") as f:
+    with open(os.path.join(HERE, "syllabic.txt"), "w", encoding="utf-8") as f:
         syllabic_final = sorted([word + "\n" for word in syllabic_matches])
         f.writelines(syllabic_final)
 
-    with open(os.path.join(HERE, "alphabetic.txt"), "w") as f:
+    with open(os.path.join(HERE, "alphabetic.txt"), "w", encoding="utf-8") as f:
         alphabetic_final = sorted([word + "\n" for word in alphabetic_matches])
         f.writelines(alphabetic_final)
 

sonatoki/constants.py

@@ -7,7 +7,7 @@ from pathlib import Path
 from sonatoki.types import LinkuWord, LinkuUsageDate
 from sonatoki.utils import find_unicode_chars, find_unicode_ranges
 
-LATEST_DATE = "2023-09"
+LATEST_DATE = "2024-09"
 # hardcoding this seems bad, but it means the parser is stable w.r.t. Linku!
 
 
@@ -648,6 +648,7 @@ FALSE_POS_SYLLABIC = {
     "insolate",
     "insulate",
     "intense",
+    "saluton",
     # "june",
     "lemon",
     "manipulate",
@@ -667,10 +668,11 @@ FALSE_POS_ALPHABETIC: Set[str] = {
     "not",
     "link",
     "wait",
+    "just",
     "lol",
     "new",
     "also",
-    "isn",  # TODO: tokenizer....
+    "isnt",
     "mean",
     "means",
     "it",
@@ -680,6 +682,7 @@ FALSE_POS_ALPHABETIC: Set[str] = {
     "new",
     "wel",
     "makes",
+    "unles",
 }
 
 UCSUR_RANGES = [
@@ -698,9 +701,9 @@ def linku_data() -> Dict[str, LinkuWord]:
     # NOTE: this does open+read+parse two files each time you construct a filter
     # but i expect users to construct filters only at the start of runtime
     # there is no reason to waste your RAM by leaving the linku data in it
-    with open(LINKU) as f:
+    with open(LINKU, "r", encoding="utf-8") as f:
         linku: Dict[str, LinkuWord] = json.loads(f.read())
-    with open(SANDBOX) as f:
+    with open(SANDBOX, "r", encoding="utf-8") as f:
         sandbox: Dict[str, LinkuWord] = json.loads(f.read())
 
     return {**linku, **sandbox}
@@ -721,6 +724,10 @@ def words_by_usage(
 
     result: Set[str] = set()
     for word in data.values():
+        if usage == 0:
+            result.add(word["word"])
+            continue
+
         usages = word["usage"]
         if date in usages and usages[date] >= usage:
             result.add(word["word"])
@@ -731,10 +738,10 @@ def words_by_usage(
 NIMI_PU_SYNONYMS = {"namako", "kin", "oko"}
 
 
-# with open(SYLLABICS) as f:
+# with open(SYLLABICS, "r", encoding="utf-8") as f:
 #     FALSE_POS_SYLLABIC = {line.strip() for line in f}
 #
-# with open(ALPHABETICS) as f:
+# with open(ALPHABETICS, "r", encoding="utf-8") as f:
 #     FALSE_POS_ALPHABETIC = {line.strip() for line in f}
 
 __all__ = [