sonatoki 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

sonatoki/Configs.py

@@ -0,0 +1,80 @@
+# STL
+from copy import deepcopy
+from typing import List, Type, TypedDict
+
+# PDM
+from typing_extensions import NotRequired
+
+# LOCAL
+from sonatoki.Filters import (
+    Filter,
+    NimiPu,
+    Numeric,
+    Syllabic,
+    NimiLinku,
+    NimiPuAle,
+    Alphabetic,
+    ProperName,
+    Phonotactic,
+    Punctuation,
+    NimiLinkuAle,
+)
+from sonatoki.Scorers import Number, Scorer, PassFail, SoftScaling, SoftPassFail
+from sonatoki.Cleaners import Cleaner, ConsecutiveDuplicates
+from sonatoki.Tokenizers import Tokenizer, WordTokenizerTok
+from sonatoki.Preprocessors import (
+    URLs,
+    Preprocessor,
+    DiscordEmotes,
+    DiscordSpecial,
+    DiscordChannels,
+    DiscordMentions,
+)
+
+
+class IloConfig(TypedDict):
+    preprocessors: List[Type[Preprocessor]]
+    word_tokenizer: Type[Tokenizer]
+    cleaners: List[Type[Cleaner]]
+    ignoring_filters: List[Type[Filter]]
+    scoring_filters: List[Type[Filter]]
+    scorer: Type[Scorer]
+    passing_score: Number
+
+
+BaseConfig: IloConfig = {
+    "preprocessors": [URLs],
+    "cleaners": [ConsecutiveDuplicates],
+    "ignoring_filters": [Numeric, Punctuation],
+    "scoring_filters": [],
+    "scorer": PassFail,
+    "passing_score": 0.8,
+    "word_tokenizer": WordTokenizerTok,
+}
+
+
+PrefConfig: IloConfig = deepcopy(BaseConfig)
+PrefConfig["scoring_filters"].extend([NimiLinku, Syllabic, ProperName, Alphabetic])
+PrefConfig["scorer"] = SoftScaling
+
+
+LazyConfig: IloConfig = deepcopy(BaseConfig)
+LazyConfig["scoring_filters"].extend([Alphabetic, ProperName])
+LazyConfig["scorer"] = SoftPassFail
+
+DiscordConfig: IloConfig = deepcopy(PrefConfig)
+DiscordConfig["preprocessors"].extend(
+    [DiscordEmotes, DiscordMentions, DiscordChannels, DiscordSpecial]
+)
+TelegramConfig: IloConfig = deepcopy(PrefConfig)
+ForumConfig: IloConfig = deepcopy(PrefConfig)
+
+__all__ = [
+    "IloConfig",
+    "BaseConfig",
+    "PrefConfig",
+    "LazyConfig",
+    "DiscordConfig",
+    "TelegramConfig",
+    "ForumConfig",
+]

sonatoki/Filters.py

@@ -17,6 +17,7 @@ from sonatoki.constants import (
     NIMI_LINKU_SET,
     NIMI_PU_ALE_SET,
     NIMI_LINKU_ALE_SET,
+    NIMI_LINKU_SANDBOX_SET,
 )
 
 re.DEFAULT_VERSION = re.VERSION1
@@ -87,6 +88,10 @@ class NimiLinkuAle(SetFilter):
     tokens = NIMI_LINKU_ALE_SET
 
 
+class NimiLinkuSandbox(SetFilter):
+    tokens = NIMI_LINKU_SANDBOX_SET
+
+
 class Phonotactic(RegexFilter):
     """Determines if a given token is phonotactically valid Toki Pona (or `n`).
     Excludes both consecutive nasals and the illegal syllables:
@@ -126,7 +131,7 @@ class Alphabetic(Filter):
         return set(token.lower()).issubset(ALPHABET_SET)
 
 
-class Numerics(Filter):
+class Numeric(Filter):
     """Determine if a given token is entirely numeric.
     Covers all numeric symbols in Unicode.
 
@@ -142,7 +147,7 @@ class Numerics(Filter):
         return msg.isnumeric()
 
 
-class Punctuations(RegexFilter):
+class Punctuation(RegexFilter):
     pattern = re.compile(r"[\p{Punctuation}\p{posix_punct}]+")
 
 
@@ -154,6 +159,6 @@ __all__ = [
     "Syllabic",
     "Alphabetic",
     "ProperName",
-    "Punctuations",
-    "Numerics",
+    "Punctuation",
+    "Numeric",
 ]

sonatoki/Preprocessors.py

@@ -13,7 +13,7 @@ There are currently two distinct types of Preprocessor:
   - ArrowQuote
 
 Order does not generally matter, but if there were two overlapping containers such as in the string "|| spoiler ` monospace || `", order would matter.
-As such, each Preprocessor exposes a .precedence attribute which is optionally usable for ordering them. Lower precedence means it should be applied first.
+It is up to the user to order them appropriately.
 """
 
 # STL
@@ -27,8 +27,6 @@ re.DEFAULT_VERSION = re.VERSION1
 
 
 class Preprocessor(ABC):
-    precedence: int = 0
-
     @classmethod  # order matters
     @abstractmethod
     def process(cls, msg: str) -> str:
@@ -64,6 +62,13 @@ class URLs(RegexPreprocessor):
     pattern = re.compile(r"https?:\/\/\S+")
 
 
+class Reference(RegexPreprocessor):
+    """Remove text contained in double brackets.
+    Often used to fetch articles on Wikipedia, or Magic the Gathering cards."""
+
+    pattern = re.compile(r"\[\[.+\]\]")
+
+
 class DiscordEmotes(RegexPreprocessor):
     """Remove text-formatted Discord emotes `<flags:name:id>`"""
 
@@ -82,6 +87,13 @@ class DiscordSpecial(RegexPreprocessor):
     pattern = re.compile(r"<id:[a-zA-Z0-9_]{4,}>")
 
 
+class AngleBracketObject(RegexPreprocessor):
+    """A generalized version of the Discord-specific angle bracket objects.
+    Removes any contiguous (not broken by whitespace) text in angle brackets."""
+
+    pattern = re.compile(r"<[^<>\s]+>")
+
+
 """
 The following classes are Containers.
 
@@ -94,24 +106,23 @@ would likely be using a language other than Toki Pona.
 
 
 class SingleQuotes(RegexPreprocessor):
-    pattern = re.compile(r"'[^']+'", flags=re.S)  # . matches newline
+    pattern = re.compile(r"'[^']+'", flags=re.DOTALL)
 
 
 class DoubleQuotes(RegexPreprocessor):
-    pattern = re.compile(r'"[^"]+"', flags=re.S)
+    pattern = re.compile(r'"[^"]+"', flags=re.DOTALL)
 
 
 class Backticks(RegexPreprocessor):
     """Remove paired backticks and their contents `like this`"""
 
-    precedence = -10
-    pattern = re.compile(r"`[^`]+`", flags=re.S)
+    pattern = re.compile(r"`[^`]+`", flags=re.DOTALL)
 
 
 class Spoilers(RegexPreprocessor):
     """Remove paired double bars and their contents `||like this||`"""
 
-    pattern = re.compile(r"\|\|(?:(?!\|\|).)+\|\|", flags=re.S)
+    pattern = re.compile(r"\|\|(?:(?!\|\|).)+\|\|", flags=re.DOTALL)
 
 
 class ArrowQuote(RegexPreprocessor):
@@ -120,7 +131,22 @@ class ArrowQuote(RegexPreprocessor):
     pattern = re.compile(r"^>\ .+$", re.MULTILINE)
 
 
+class AllQuotes(RegexPreprocessor):
+    pattern = re.compile(
+        "|".join(
+            [
+                SingleQuotes.pattern.pattern,
+                DoubleQuotes.pattern.pattern,
+                Backticks.pattern.pattern,
+                ArrowQuote.pattern.pattern,
+            ]
+        ),
+        flags=re.MULTILINE | re.DOTALL,
+    )
+
+
 __all__ = [
+    "AngleBracketObject",
     "DiscordChannels",
     "DiscordMentions",
     "DiscordSpecial",
@@ -128,7 +154,9 @@ __all__ = [
     "SingleQuotes",
     "DoubleQuotes",
     "ArrowQuote",
+    "AllQuotes",
     "Backticks",
+    "Reference",
     "Spoilers",
     "URLs",
 ]

sonatoki/Tokenizers.py

@@ -1,10 +1,10 @@
 # STL
-from typing import List, Callable
+from abc import ABC, abstractmethod
+from typing import List
 
 # PDM
 import regex as re
-
-# TODO: Entire module should be reworked to match the class scheme of the rest of the module, imo
+from typing_extensions import override
 
 try:
     # PDM
@@ -17,42 +17,60 @@ except ImportError as e:
 
 LANGUAGE = "english"  # for NLTK
 
-SENT_DELIMS_RE = re.compile(r"""(.*?[.?!;:])|(.+?$)""")
-SENT_DELIMS_TOK = re.compile(r"""(?<=[.?!:;·…“”"'()\[\]\-]|$)""")
-# TODO: are <> or {} that common as *sentence* delims? [] are already a stretch
-# TODO: do the typography characters matter?
-# NOTE: | / and , are *not* sentence delimiters for my purpose
 
-WORD_DELIMS_RE = re.compile(r"""\s+|(?=[.?!;:'"-])""")
-WORD_DELIMS_TOK = re.compile(r"([\p{Punctuation}\p{posix_punct}]+|\s+)")
+class Tokenizer(ABC):
+    @classmethod
+    @abstractmethod
+    def tokenize(cls, s: str) -> List[str]: ...
 
-Tokenizer = Callable[[str], List[str]]
 
+class NoOpTokenizer(Tokenizer):
+    """This is a special case that you do not want or need."""
 
-if not isinstance(nltk, ImportError):
+    @classmethod
+    @override
+    def tokenize(cls, s: str) -> List[str]:
+        return [s]
+
+
+class RegexTokenizer(Tokenizer):
+    pattern: "re.Pattern[str]"
 
-    def sent_tokenize_nltk(s: str) -> List[str]:
-        return __sent_tokenize_nltk(text=s, language=LANGUAGE)
+    @classmethod
+    @override
+    def tokenize(cls, s: str) -> List[str]:
+        return [clean for word in re.split(cls.pattern, s) if (clean := word.strip())]
 
-    def word_tokenize_nltk(s: str) -> List[str]:
-        return __word_tokenize_nltk(text=s, language=LANGUAGE)
 
+class WordTokenizerTok(RegexTokenizer):
+    pattern = re.compile(r"""([\p{Punctuation}\p{posix_punct}]+|\s+)""")
+    # TODO: are <> or {} that common as *sentence* delims? [] are already a stretch
+    # TODO: do the typography characters matter?
+    # NOTE: | / and , are *not* sentence delimiters for my purpose
 
-def sent_tokenize_re(s: str) -> List[str]:
-    return [
-        clean
-        for sent in re.findall(SENT_DELIMS_RE, s)
-        if (clean := sent[0].strip() or sent[1].strip())
-    ]
 
+class SentTokenizerTok(RegexTokenizer):
+    pattern = re.compile(r"""(?<=[.?!:;·…“”"'()\[\]\-]|$)""")
 
-def word_tokenize_re(s: str) -> List[str]:
-    return [clean for word in re.split(WORD_DELIMS_RE, s) if (clean := word.strip())]
 
+class WordTokenizerRe(RegexTokenizer):
+    pattern = re.compile(r"""(?<=[.?!;:'"-])""")
 
-def sent_tokenize_tok(s: str) -> List[str]:
-    return [clean for sent in re.split(SENT_DELIMS_TOK, s) if (clean := sent.strip())]
 
+class SentTokenizerRe(RegexTokenizer):
+    pattern = re.compile(r"""(.*?[.?!;:])|(.+?$)""")
+
+
+if not isinstance(nltk, ImportError):
 
-def word_tokenize_tok(s: str) -> List[str]:
-    return [clean for word in re.split(WORD_DELIMS_TOK, s) if (clean := word.strip())]
+    class WordTokenizerNLTK(Tokenizer):
+        @classmethod
+        @override
+        def tokenize(cls, s: str) -> List[str]:
+            return __word_tokenize_nltk(text=s, language=LANGUAGE)
+
+    class SentTokenizerNLTK(Tokenizer):
+        @classmethod
+        @override
+        def tokenize(cls, s: str) -> List[str]:
+            return __sent_tokenize_nltk(text=s, language=LANGUAGE)

sonatoki/constants.py

@@ -4,6 +4,7 @@ from typing import Dict, List
 from pathlib import Path
 
 LINKU = Path(__file__).resolve().parent / Path("linku.json")
+SANDBOX = Path(__file__).resolve().parent / Path("sandbox.json")
 
 VOWELS = "aeiou"
 CONSONANTS = "jklmnpstw"
@@ -29,10 +30,16 @@ with open(LINKU) as f:
     ]
     NIMI_LINKU_ALE: List[str] = [d["word"] for d in r.values()]
 
+with open(SANDBOX) as f:
+    r: Dict[str, Dict[str, str]] = json.loads(f.read())
+    NIMI_LINKU_SANDBOX: List[str] = [d["word"] for d in r.values()]
+
+
 NIMI_PU_SET = set(NIMI_PU)
 NIMI_PU_ALE_SET = set(NIMI_PU_ALE)
 NIMI_LINKU_SET = set(NIMI_LINKU)
 NIMI_LINKU_ALE_SET = set(NIMI_LINKU_ALE)
+NIMI_LINKU_SANDBOX_SET = set(NIMI_LINKU_SANDBOX)
 ALLOWABLES_SET = set(ALLOWABLES)
 
 __all__ = [
@@ -54,4 +61,7 @@ __all__ = [
     #
     "NIMI_LINKU_ALE",
     "NIMI_LINKU_ALE_SET",
+    #
+    "NIMI_LINKU_SANDBOX",
+    "NIMI_LINKU_SANDBOX_SET",
 ]

sonatoki/ilo.py

@@ -14,13 +14,13 @@ LOG = logging.getLogger(__name__)
 
 class Ilo:
     __preprocessors: List[Type[Preprocessor]]
+    __word_tokenizer: Type[Tokenizer]
     __cleaners: List[Type[Cleaner]]
     __ignoring_filters: List[Type[Filter]]
     __scoring_filters: List[Type[Filter]]
     __scorer: Type[Scorer]
-    __tokenize: Tokenizer
     __passing_score: Number
-    logging_threshold: Number = 1.0
+    logging_threshold: Number = -1
 
     def __init__(
         self,
@@ -29,61 +29,62 @@ class Ilo:
         ignoring_filters: List[Type[Filter]],
         scoring_filters: List[Type[Filter]],
         scorer: Type[Scorer],
-        tokenizer: Tokenizer,  # NOTE: no wrapper needed?
         passing_score: Number,
+        word_tokenizer: Type[Tokenizer],
     ):
         super().__init__()
         # avoid keeping a ref to user's list just in case
         self.__preprocessors = [*preprocessors]
+        self.__word_tokenizer = word_tokenizer
         self.__cleaners = [*cleaners]
         self.__ignoring_filters = [*ignoring_filters]
         self.__scoring_filters = [*scoring_filters]
         self.__scorer = scorer
-        self.__tokenize = tokenizer
         self.__passing_score = passing_score
 
-    def __preprocess(self, msg: str) -> str:
+    def preprocess(self, msg: str) -> str:
         for p in self.__preprocessors:
             msg = p.process(msg)
         return msg
 
-    def __clean_token(self, token: str) -> str:
+    def word_tokenize(self, msg: str) -> List[str]:
+        """It is *highly* recommended that you run `ilo.preprocess` first."""
+        return self.__word_tokenizer.tokenize(msg)
+
+    def clean_token(self, token: str) -> str:
         for c in self.__cleaners:
             token = c.clean(token)
         return token
 
-    def __clean_tokens(self, tokens: List[str]) -> List[str]:
-        # NOTE: tested, making a new list with a for loop *is* faster than
-        # - list comps
-        # - generator comps
-        # - in-place replacement/removal
-        # - in place replacement with result of generator comp
+    def clean_tokens(self, tokens: List[str]) -> List[str]:
+        # NOTE: tested, making a new list with a for loop *is* faster than:
+        # list comp, generator comp, in-place replacement
         cleaned_tokens: List[str] = list()
         for token in tokens:
-            cleaned_token = self.__clean_token(token)
+            cleaned_token = self.clean_token(token)
             if not cleaned_token:
                 # TODO: warn user?
                 continue
             cleaned_tokens.append(cleaned_token)
         return cleaned_tokens
 
-    def __filter_token(self, token: str) -> bool:
+    def _filter_token(self, token: str) -> bool:
         for f in self.__ignoring_filters:
             if f.filter(token):
                 return True
         return False
 
-    def __filter_tokens(self, tokens: List[str]) -> List[str]:
+    def filter_tokens(self, tokens: List[str]) -> List[str]:
         filtered_tokens: List[str] = []
         for token in tokens:
-            if self.__filter_token(token):
+            if self._filter_token(token):
                 continue
             # the ignoring filter is true if the token matches
             # the user wants to ignore these so keep non-matching tokens
             filtered_tokens.append(token)
         return filtered_tokens
 
-    def __score_tokens(self, tokens: List[str]) -> float:
+    def score_tokens(self, tokens: List[str]) -> float:
         return self.__scorer.score(tokens, self.__scoring_filters)
 
     def _is_toki_pona(
@@ -95,26 +96,25 @@ class Ilo:
         - Filtered message (list[str])
         - Cleaned message (list[str])
         - Score (float)
-        - Result (bool)
-        """
-        preprocessed = self.__preprocess(message)
-        tokenized = self.__tokenize(preprocessed)
-        filtered = self.__filter_tokens(tokenized)
-        cleaned = self.__clean_tokens(filtered)
-        score = self.__score_tokens(cleaned)
+        - Result (bool)"""
+        preprocessed = self.preprocess(message)
+        tokenized = self.word_tokenize(preprocessed)
+        filtered = self.filter_tokens(tokenized)
+        cleaned = self.clean_tokens(filtered)
+        score = self.score_tokens(cleaned)
         result = score >= self.__passing_score
 
-        # NOTE: this method may break if above funcs start sharing a list
         if score <= self.logging_threshold:
-            LOG.debug("Msg: %.2f  %s", score, repr(message))
-            LOG.debug("Preproc:   %s", repr(preprocessed))
-            LOG.debug("Tokenized: %s", tokenized)
-            LOG.debug("Filtered:  %s", filtered)
-            LOG.debug("Cleaned:   %s", cleaned)
+            LOG.debug("msg: %.2f  %s", score, repr(message))
+            LOG.debug("preproc:   %s", repr(preprocessed))
+            LOG.debug("tokenized: %s", tokenized)
+            LOG.debug("filtered:  %s", filtered)
+            LOG.debug("cleaned:   %s", cleaned)
         # TODO: Move to each function? Loses ability to control when logging occurs by threshold
 
         return preprocessed, tokenized, filtered, cleaned, score, result
 
     def is_toki_pona(self, message: str) -> bool:
+        """Determines whether a single statement is or is not Toki Pona."""
         *_, result = self._is_toki_pona(message)
         return result