sonatoki 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl

sonatoki/Configs.py

@@ -0,0 +1,80 @@
+# STL
+from copy import deepcopy
+from typing import List, Type, TypedDict
+
+# PDM
+from typing_extensions import NotRequired
+
+# LOCAL
+from sonatoki.Filters import (
+    Filter,
+    NimiPu,
+    Numerics,
+    Syllabic,
+    NimiLinku,
+    NimiPuAle,
+    Alphabetic,
+    ProperName,
+    Phonotactic,
+    NimiLinkuAle,
+    Punctuations,
+)
+from sonatoki.Scorers import Number, Scorer, PassFail, SoftScaling, SoftPassFail
+from sonatoki.Cleaners import Cleaner, ConsecutiveDuplicates
+from sonatoki.Tokenizers import Tokenizer, WordTokenizerTok
+from sonatoki.Preprocessors import (
+    URLs,
+    Preprocessor,
+    DiscordEmotes,
+    DiscordSpecial,
+    DiscordChannels,
+    DiscordMentions,
+)
+
+
+class IloConfig(TypedDict):
+    preprocessors: List[Type[Preprocessor]]
+    word_tokenizer: Type[Tokenizer]
+    cleaners: List[Type[Cleaner]]
+    ignoring_filters: List[Type[Filter]]
+    scoring_filters: List[Type[Filter]]
+    scorer: Type[Scorer]
+    passing_score: Number
+
+
+BaseConfig: IloConfig = {
+    "preprocessors": [URLs],
+    "cleaners": [ConsecutiveDuplicates],
+    "ignoring_filters": [Numerics, Punctuations],
+    "scoring_filters": [],
+    "scorer": PassFail,
+    "passing_score": 0.8,
+    "word_tokenizer": WordTokenizerTok,
+}
+
+
+PrefConfig: IloConfig = deepcopy(BaseConfig)
+PrefConfig["scoring_filters"].extend([NimiLinku, Syllabic, ProperName, Alphabetic])
+PrefConfig["scorer"] = SoftScaling
+
+
+LazyConfig: IloConfig = deepcopy(BaseConfig)
+LazyConfig["scoring_filters"].extend([Alphabetic, ProperName])
+LazyConfig["scorer"] = SoftPassFail
+
+DiscordConfig: IloConfig = deepcopy(PrefConfig)
+DiscordConfig["preprocessors"].extend(
+    [DiscordEmotes, DiscordMentions, DiscordChannels, DiscordSpecial]
+)
+TelegramConfig: IloConfig = deepcopy(PrefConfig)
+ForumConfig: IloConfig = deepcopy(PrefConfig)
+
+__all__ = [
+    "IloConfig",
+    "BaseConfig",
+    "PrefConfig",
+    "LazyConfig",
+    "DiscordConfig",
+    "TelegramConfig",
+    "ForumConfig",
+]

sonatoki/Filters.py

@@ -17,6 +17,7 @@ from sonatoki.constants import (
     NIMI_LINKU_SET,
     NIMI_PU_ALE_SET,
     NIMI_LINKU_ALE_SET,
+    NIMI_LINKU_SANDBOX_SET,
 )
 
 re.DEFAULT_VERSION = re.VERSION1
@@ -87,6 +88,10 @@ class NimiLinkuAle(SetFilter):
     tokens = NIMI_LINKU_ALE_SET
 
 
+class NimiLinkuSandbox(SetFilter):
+    tokens = NIMI_LINKU_SANDBOX_SET
+
+
 class Phonotactic(RegexFilter):
     """Determines if a given token is phonotactically valid Toki Pona (or `n`).
     Excludes both consecutive nasals and the illegal syllables:

sonatoki/Preprocessors.py

@@ -13,7 +13,7 @@ There are currently two distinct types of Preprocessor:
   - ArrowQuote
 
 Order does not generally matter, but if there were two overlapping containers such as in the string "|| spoiler ` monospace || `", order would matter.
-As such, each Preprocessor exposes a .precedence attribute which is optionally usable for ordering them. Lower precedence means it should be applied first.
+It is up to the user to order them appropriately.
 """
 
 # STL
@@ -27,8 +27,6 @@ re.DEFAULT_VERSION = re.VERSION1
 
 
 class Preprocessor(ABC):
-    precedence: int = 0
-
     @classmethod  # order matters
     @abstractmethod
     def process(cls, msg: str) -> str:
@@ -104,7 +102,6 @@ class DoubleQuotes(RegexPreprocessor):
 class Backticks(RegexPreprocessor):
     """Remove paired backticks and their contents `like this`"""
 
-    precedence = -10
     pattern = re.compile(r"`[^`]+`", flags=re.S)
 
 
@@ -121,6 +118,9 @@ class ArrowQuote(RegexPreprocessor):
 
 
 __all__ = [
+    "DiscordChannels",
+    "DiscordMentions",
+    "DiscordSpecial",
     "DiscordEmotes",
     "SingleQuotes",
     "DoubleQuotes",

sonatoki/Scorers.py

@@ -16,6 +16,13 @@ Number = Union[int, float]
 Weights = Dict[str, Number]
 
 
+def sigmoid(n: int) -> Number:
+    return 1 / (1 + math.exp(-(0.30 * (n - 1))))
+    # n-1 makes sigmoid(1) == 0.5
+    # 0.30 softens scaling in favor of short input
+    # return n / (1+abs(n))   # too weak in 0.7+
+
+
 class Scorer(ABC):
     @classmethod
     @abstractmethod
@@ -27,7 +34,7 @@ class PassFail(Scorer):
     """The token passes any filter or fails all of them, scoring 1 or 0 respectively."""
 
     @classmethod
-    def __score(cls, token: str, filters: List[Type[Filter]]) -> Number:
+    def score_token(cls, token: str, filters: List[Type[Filter]]) -> Number:
         for f in filters:
             if f.filter(token):
                 score = 1
@@ -47,10 +54,27 @@ class PassFail(Scorer):
         total_score = 0
         len_tokens = len(tokens)
         for token in tokens:
-            total_score += cls.__score(token, filters)
+            total_score += cls.score_token(token, filters)
         return total_score / len_tokens if len_tokens else 0
 
 
+class SoftPassFail(PassFail):
+    @classmethod
+    @override
+    def score(cls, tokens: List[str], filters: List[Type[Filter]]) -> Number:
+        if not tokens:
+            return 1
+
+        total_score = 0
+        len_tokens = len(tokens)
+        for token in tokens:
+            total_score += cls.score_token(token, filters)
+
+        percentage = total_score / len_tokens if len_tokens else 0
+        percentage **= sigmoid(len_tokens)
+        return percentage
+
+
 class Scaling(Scorer):
     """
     The sooner a token matches a filter, the higher its score.
@@ -91,13 +115,6 @@ class SoftScaling(Scaling):
     For example, a single token scoring 0.64 will now score 0.8.
     """
 
-    @staticmethod
-    def sigmoid(n: int) -> Number:
-        return 1 / (1 + math.exp(-(0.30 * (n - 1))))
-        # n-1 makes sigmoid(1) == 0.5
-        # 0.30 softens scaling in favor of short input
-        # return n / (1+abs(n))   # too weak in 0.7+
-
     @classmethod
     @override
     def score(cls, tokens: List[str], filters: List[Type[Filter]]) -> Number:
@@ -113,11 +130,11 @@ class SoftScaling(Scaling):
             total_score += cls.score_token(token, filters, len_filters)
 
         percentage = total_score / max_score if max_score else 0
-        percentage **= cls.sigmoid(len_tokens)
+        percentage **= sigmoid(len_tokens)
         return percentage
 
 
 class Logarithmic(Scorer): ...
 
 
-__all__ = ["PassFail", "Scaling", "SoftScaling"]
+__all__ = ["PassFail", "SoftPassFail", "Scaling", "SoftScaling"]

sonatoki/Tokenizers.py

@@ -1,8 +1,10 @@
 # STL
-from typing import List, Callable
+from abc import ABC, abstractmethod
+from typing import List
 
 # PDM
 import regex as re
+from typing_extensions import override
 
 try:
     # PDM
@@ -15,50 +17,60 @@ except ImportError as e:
 
 LANGUAGE = "english"  # for NLTK
 
-SENT_DELIMS_RE = r"""(.*?[.?!;:])|(.+?$)"""
-SENT_DELIMS_RE = re.compile(SENT_DELIMS_RE)
 
-SENT_DELIMS_TOK = r"""(.*?[.?!;:-])|(.+?$)"""
-SENT_DELIMS_TOK = re.compile(SENT_DELIMS_TOK)
+class Tokenizer(ABC):
+    @classmethod
+    @abstractmethod
+    def tokenize(cls, s: str) -> List[str]: ...
 
 
-WORD_DELIMS_RE = r"""\s+|(?=[.?!;:'"-])"""
-WORD_DELIMS_RE = re.compile(WORD_DELIMS_RE)
+class NoOpTokenizer(Tokenizer):
+    """This is a special case that you do not want or need."""
 
-WORD_DELIMS_TOK = r"([\p{Punctuation}\p{posix_punct}]+|\s+)"
-WORD_DELIMS_TOK = re.compile(WORD_DELIMS_TOK)
+    @classmethod
+    @override
+    def tokenize(cls, s: str) -> List[str]:
+        return [s]
 
-Tokenizer = Callable[[str], List[str]]
 
+class RegexTokenizer(Tokenizer):
+    pattern: "re.Pattern[str]"
+
+    @classmethod
+    @override
+    def tokenize(cls, s: str) -> List[str]:
+        return [clean for word in re.split(cls.pattern, s) if (clean := word.strip())]
 
-if not isinstance(nltk, ImportError):
 
-    def sent_tokenize_nltk(s: str) -> List[str]:
-        return __sent_tokenize_nltk(text=s, language=LANGUAGE)
+class WordTokenizerTok(RegexTokenizer):
+    pattern = re.compile(r"""([\p{Punctuation}\p{posix_punct}]+|\s+)""")
+    # TODO: are <> or {} that common as *sentence* delims? [] are already a stretch
+    # TODO: do the typography characters matter?
+    # NOTE: | / and , are *not* sentence delimiters for my purpose
 
-    def word_tokenize_nltk(s: str) -> List[str]:
-        return __word_tokenize_nltk(text=s, language=LANGUAGE)
 
+class SentTokenizerTok(RegexTokenizer):
+    pattern = re.compile(r"""(?<=[.?!:;·…“”"'()\[\]\-]|$)""")
 
-def sent_tokenize_re(s: str) -> List[str]:
-    return [
-        clean
-        for sent in re.findall(SENT_DELIMS_RE, s)
-        if (clean := sent[0].strip() or sent[1].strip())
-    ]
 
+class WordTokenizerRe(RegexTokenizer):
+    pattern = re.compile(r"""(?<=[.?!;:'"-])""")
 
-def word_tokenize_re(s: str) -> List[str]:
-    return [clean for word in re.split(WORD_DELIMS_RE, s) if (clean := word.strip())]
 
+class SentTokenizerRe(RegexTokenizer):
+    pattern = re.compile(r"""(.*?[.?!;:])|(.+?$)""")
 
-def sent_tokenize_tok(s: str) -> List[str]:
-    return [
-        clean
-        for sent in re.findall(SENT_DELIMS_TOK, s)
-        if (clean := sent[0].strip() or sent[1].strip())
-    ]
 
+if not isinstance(nltk, ImportError):
 
-def word_tokenize_tok(s: str) -> List[str]:
-    return [clean for word in re.split(WORD_DELIMS_TOK, s) if (clean := word.strip())]
+    class WordTokenizerNLTK(Tokenizer):
+        @classmethod
+        @override
+        def tokenize(cls, s: str) -> List[str]:
+            return __word_tokenize_nltk(text=s, language=LANGUAGE)
+
+    class SentTokenizerNLTK(Tokenizer):
+        @classmethod
+        @override
+        def tokenize(cls, s: str) -> List[str]:
+            return __sent_tokenize_nltk(text=s, language=LANGUAGE)

sonatoki/constants.py

@@ -4,6 +4,7 @@ from typing import Dict, List
 from pathlib import Path
 
 LINKU = Path(__file__).resolve().parent / Path("linku.json")
+SANDBOX = Path(__file__).resolve().parent / Path("sandbox.json")
 
 VOWELS = "aeiou"
 CONSONANTS = "jklmnpstw"
@@ -29,10 +30,16 @@ with open(LINKU) as f:
     ]
     NIMI_LINKU_ALE: List[str] = [d["word"] for d in r.values()]
 
+with open(SANDBOX) as f:
+    r: Dict[str, Dict[str, str]] = json.loads(f.read())
+    NIMI_LINKU_SANDBOX: List[str] = [d["word"] for d in r.values()]
+
+
 NIMI_PU_SET = set(NIMI_PU)
 NIMI_PU_ALE_SET = set(NIMI_PU_ALE)
 NIMI_LINKU_SET = set(NIMI_LINKU)
 NIMI_LINKU_ALE_SET = set(NIMI_LINKU_ALE)
+NIMI_LINKU_SANDBOX_SET = set(NIMI_LINKU_SANDBOX)
 ALLOWABLES_SET = set(ALLOWABLES)
 
 __all__ = [
@@ -54,4 +61,7 @@ __all__ = [
     #
     "NIMI_LINKU_ALE",
     "NIMI_LINKU_ALE_SET",
+    #
+    "NIMI_LINKU_SANDBOX",
+    "NIMI_LINKU_SANDBOX_SET",
 ]

sonatoki/ilo.py

@@ -14,13 +14,13 @@ LOG = logging.getLogger(__name__)
 
 class Ilo:
     __preprocessors: List[Type[Preprocessor]]
+    __word_tokenizer: Type[Tokenizer]
     __cleaners: List[Type[Cleaner]]
     __ignoring_filters: List[Type[Filter]]
     __scoring_filters: List[Type[Filter]]
     __scorer: Type[Scorer]
-    __tokenize: Tokenizer
     __passing_score: Number
-    logging_threshold: Number = 1.0
+    logging_threshold: Number = -1
 
     def __init__(
         self,
@@ -29,61 +29,62 @@ class Ilo:
         ignoring_filters: List[Type[Filter]],
         scoring_filters: List[Type[Filter]],
         scorer: Type[Scorer],
-        tokenizer: Tokenizer,  # NOTE: no wrapper needed?
         passing_score: Number,
+        word_tokenizer: Type[Tokenizer],
     ):
         super().__init__()
         # avoid keeping a ref to user's list just in case
         self.__preprocessors = [*preprocessors]
+        self.__word_tokenizer = word_tokenizer
         self.__cleaners = [*cleaners]
         self.__ignoring_filters = [*ignoring_filters]
         self.__scoring_filters = [*scoring_filters]
         self.__scorer = scorer
-        self.__tokenize = tokenizer
         self.__passing_score = passing_score
 
-    def __preprocess(self, msg: str) -> str:
+    def preprocess(self, msg: str) -> str:
         for p in self.__preprocessors:
             msg = p.process(msg)
         return msg
 
-    def __clean_token(self, token: str) -> str:
+    def word_tokenize(self, msg: str) -> List[str]:
+        """It is *highly* recommended that you run `ilo.preprocess` first."""
+        return self.__word_tokenizer.tokenize(msg)
+
+    def clean_token(self, token: str) -> str:
         for c in self.__cleaners:
             token = c.clean(token)
         return token
 
-    def __clean_tokens(self, tokens: List[str]) -> List[str]:
-        # NOTE: tested, making a new list with a for loop *is* faster than
-        # - list comps
-        # - generator comps
-        # - in-place replacement/removal
-        # - in place replacement with result of generator comp
+    def clean_tokens(self, tokens: List[str]) -> List[str]:
+        # NOTE: tested, making a new list with a for loop *is* faster than:
+        # list comp, generator comp, in-place replacement
         cleaned_tokens: List[str] = list()
         for token in tokens:
-            cleaned_token = self.__clean_token(token)
+            cleaned_token = self.clean_token(token)
             if not cleaned_token:
                 # TODO: warn user?
                 continue
             cleaned_tokens.append(cleaned_token)
         return cleaned_tokens
 
-    def __filter_token(self, token: str) -> bool:
+    def _filter_token(self, token: str) -> bool:
         for f in self.__ignoring_filters:
             if f.filter(token):
                 return True
         return False
 
-    def __filter_tokens(self, tokens: List[str]) -> List[str]:
+    def filter_tokens(self, tokens: List[str]) -> List[str]:
         filtered_tokens: List[str] = []
         for token in tokens:
-            if self.__filter_token(token):
+            if self._filter_token(token):
                 continue
             # the ignoring filter is true if the token matches
             # the user wants to ignore these so keep non-matching tokens
             filtered_tokens.append(token)
         return filtered_tokens
 
-    def __score_tokens(self, tokens: List[str]) -> float:
+    def score_tokens(self, tokens: List[str]) -> float:
         return self.__scorer.score(tokens, self.__scoring_filters)
 
     def _is_toki_pona(
@@ -95,26 +96,25 @@ class Ilo:
         - Filtered message (list[str])
         - Cleaned message (list[str])
         - Score (float)
-        - Result (bool)
-        """
-        preprocessed = self.__preprocess(message)
-        tokenized = self.__tokenize(preprocessed)
-        filtered = self.__filter_tokens(tokenized)
-        cleaned = self.__clean_tokens(filtered)
-        score = self.__score_tokens(cleaned)
+        - Result (bool)"""
+        preprocessed = self.preprocess(message)
+        tokenized = self.word_tokenize(preprocessed)
+        filtered = self.filter_tokens(tokenized)
+        cleaned = self.clean_tokens(filtered)
+        score = self.score_tokens(cleaned)
         result = score >= self.__passing_score
 
-        # NOTE: this method may break if above funcs start sharing a list
         if score <= self.logging_threshold:
-            LOG.debug("Msg: %.2f  %s", score, repr(message))
-            LOG.debug("Preproc:   %s", repr(preprocessed))
-            LOG.debug("Tokenized: %s", tokenized)
-            LOG.debug("Filtered:  %s", filtered)
-            LOG.debug("Cleaned:   %s", cleaned)
+            LOG.debug("msg: %.2f  %s", score, repr(message))
+            LOG.debug("preproc:   %s", repr(preprocessed))
+            LOG.debug("tokenized: %s", tokenized)
+            LOG.debug("filtered:  %s", filtered)
+            LOG.debug("cleaned:   %s", cleaned)
         # TODO: Move to each function? Loses ability to control when logging occurs by threshold
 
         return preprocessed, tokenized, filtered, cleaned, score, result
 
     def is_toki_pona(self, message: str) -> bool:
+        """Determines whether a single statement is or is not Toki Pona."""
         *_, result = self._is_toki_pona(message)
         return result