sonatoki 0.3.2__py3-none-any.whl → 0.4.0__py3-none-any.whl

sonatoki/Cleaners.py

@@ -10,6 +10,7 @@ class Cleaner(ABC):
     @classmethod
     @abstractmethod
     def clean(cls, token: str) -> str:
+        """Transform a token to remove some undesirable part."""
         raise NotImplementedError
 
 
@@ -33,7 +34,8 @@ class ConsecutiveDuplicates(Cleaner):
     may be altered for emphasis or effect, such as in "sonaaaa" or "AAAAAA".
 
     This may be undesirable for moraic scripts like Hiragana, where `わわ` would be
-    incorrectly reduced to `わ`. This does preserve phonotactic validity, though."""
+    incorrectly reduced to `わ`. This does preserve phonotactic validity, though.
+    """
 
     @classmethod
     @override
@@ -69,4 +71,5 @@ class Lowercase(Cleaner):
 
 __all__ = [
     "ConsecutiveDuplicates",
+    "Lowercase",
 ]

sonatoki/Configs.py

@@ -2,6 +2,9 @@
 from copy import deepcopy
 from typing import List, Type, TypedDict
 
+# PDM
+from typing_extensions import NotRequired
+
 # LOCAL
 from sonatoki.Filters import (
     Filter,
@@ -9,6 +12,8 @@ from sonatoki.Filters import (
     Syllabic,
     NimiUCSUR,
     Alphabetic,
+    NimiKuLili,
+    NimiKuSuli,
     ProperName,
     Punctuation,
     LongSyllabic,
@@ -20,12 +25,11 @@ from sonatoki.Filters import (
     NimiLinkuCommon,
     NimiLinkuObscure,
     NimiLinkuSandbox,
-    EnglishIgnorables,
     NimiLinkuUncommon,
 )
 from sonatoki.Scorers import Number, Scorer, PassFail, SoftScaling, SoftPassFail
 from sonatoki.Cleaners import Cleaner, ConsecutiveDuplicates
-from sonatoki.Tokenizers import Tokenizer, WordTokenizer
+from sonatoki.Tokenizers import Tokenizer
 from sonatoki.Preprocessors import (
     URLs,
     Backticks,
@@ -37,15 +41,16 @@ from sonatoki.Preprocessors import (
 
 class IloConfig(TypedDict):
     preprocessors: List[Type[Preprocessor]]
-    word_tokenizer: Type[Tokenizer]
     cleaners: List[Type[Cleaner]]
     ignoring_filters: List[Type[Filter]]
     scoring_filters: List[Type[Filter]]
     scorer: Type[Scorer]
     passing_score: Number
+    word_tokenizer: NotRequired[Type[Tokenizer]]
+    sent_tokenizer: NotRequired[Type[Tokenizer]]
 
 
-# TODO: branching configs?
+# TODO: branching configs? config builder?
 
 BaseConfig: IloConfig = {
     "preprocessors": [URLs],
@@ -54,7 +59,6 @@ BaseConfig: IloConfig = {
     "scoring_filters": [],
     "scorer": PassFail,
     "passing_score": 0.8,
-    "word_tokenizer": WordTokenizer,
 }
 
 
@@ -70,7 +74,6 @@ PrefConfig: IloConfig = {
     ],
     "scorer": SoftScaling,
     "passing_score": 0.8,
-    "word_tokenizer": WordTokenizer,
 }
 
 CorpusConfig: IloConfig = {
@@ -93,13 +96,8 @@ CorpusConfig: IloConfig = {
     ],
     "scorer": SoftScaling,
     "passing_score": 0.8,
-    "word_tokenizer": WordTokenizer,
 }
-
-
-"""
-Mimics the previous implementation of ilo pi toki pona taso
-"""
+"""Mimics the previous implementation of ilo pi toki pona taso."""
 LazyConfig: IloConfig = {
     "preprocessors": [Backticks, URLs, AngleBracketObject, Reference],
     "cleaners": [ConsecutiveDuplicates],
@@ -107,27 +105,47 @@ LazyConfig: IloConfig = {
     "scoring_filters": [Alphabetic, NimiUCSUR, ProperName, Miscellaneous],
     "scorer": SoftPassFail,
     "passing_score": 0.8,
-    "word_tokenizer": WordTokenizer,
 }
+"""This is extremely silly."""
+IsipinEpikuConfig: IloConfig = {
+    "preprocessors": [Backticks, URLs, AngleBracketObject, Reference],
+    "cleaners": [ConsecutiveDuplicates],
+    "ignoring_filters": [Numeric, Punctuation],
+    "scoring_filters": [
+        OrMemberFilter(
+            NimiKuSuli,
+            NimiKuLili,
+            NimiLinkuUncommon,
+            NimiLinkuObscure,
+            NimiLinkuSandbox,
+        ),
+        LongSyllabic,
+        LongProperName,
+        LongAlphabetic,
+    ],
+    "scorer": SoftScaling,
+    "passing_score": 0.8,
+}
+
 
 DiscordConfig: IloConfig = {
     "preprocessors": [Backticks, URLs, AngleBracketObject, Reference],
     "cleaners": [ConsecutiveDuplicates],
-    "ignoring_filters": [Numeric, Punctuation, EnglishIgnorables],
+    "ignoring_filters": [Numeric, Punctuation],
     "scoring_filters": [
-        OrMemberFilter(NimiLinkuCore, NimiLinkuCommon, NimiUCSUR),
+        OrMemberFilter(NimiLinkuCore, NimiLinkuCommon, NimiUCSUR, Miscellaneous),
         LongSyllabic,
         LongProperName,
         LongAlphabetic,
     ],
     "scorer": SoftScaling,
     "passing_score": 0.8,
-    "word_tokenizer": WordTokenizer,
 }
 
 TelegramConfig: IloConfig = deepcopy(PrefConfig)
 ForumConfig: IloConfig = deepcopy(PrefConfig)
 
+
 __all__ = [
     "BaseConfig",
     "CorpusConfig",

sonatoki/Filters.py

@@ -127,9 +127,11 @@ class ProperName(Filter):
     When Toki Pona is written with the Latin alphabet, names are generally
     capitalized at their start. This filter identifies those tokens.
 
-    Note that this alone cannot determine if a token is a valid name, because
-    a standalone name is considered invalid in Toki Pona- names generally have head nouns.
-    This tool only examines one token at a time, so cannot detect names any better than identifying their capital letter.
+    Note that this alone cannot determine if a token is a valid name,
+    because a standalone name is considered invalid in Toki Pona- names
+    generally have head nouns. This tool only examines one token at a
+    time, so cannot detect names any better than identifying their
+    capital letter.
     """
 
     @classmethod
@@ -187,12 +189,14 @@ class NimiUCSUR(MemberFilter):
 
 class Phonotactic(RegexFilter):
     """Determines if a given token is phonotactically valid Toki Pona (or `n`).
+
     Excludes both consecutive nasals and the illegal syllables:
     - "nm", "nn"
     - "wu", "wo", "ji", "ti"
 
     Note that if this validator is used after `Cleaners.ConsecutiveDuplicates`,
-    "nn" cannot be found."""
+    "nn" cannot be found.
+    """
 
     pattern = re.compile(
         rf"^((^[{VOWELS}]|[klmnps][{VOWELS}]|[jt][aeou]|[w][aei])(n(?![mn]))?)+$|^n$",
@@ -208,8 +212,10 @@ class LongPhonotactic(MinLen, Phonotactic):
 
 class Syllabic(RegexFilter):
     """Determines if a given token is syllabically valid Toki Pona (or `n`).
-    Words must have correctly ordered vowels and consonants, but the phonotactic
-    exceptions are not considered."""
+
+    Words must have correctly ordered vowels and consonants, but the
+    phonotactic exceptions are not considered.
+    """
 
     # rf"^((^[{VOWELS}]|[{CONSONANTS}][{VOWELS}])n?)+$|^n$"
     # Alterative I was exploring takes ~15% more steps
@@ -236,13 +242,14 @@ class LongAlphabetic(MinLen, Alphabetic):
 
 
 class Numeric(Filter):
-    """Determine if a given token is entirely numeric.
-    Covers all numeric symbols in Unicode.
+    """Determine if a given token is entirely numeric. Covers all numeric
+    symbols in Unicode.
 
     This will fail to find numeric tokens such as "1.111" or "-42",
     but if used with the aggressive tokenizer designed for `tok`, these will be
     split into `["1", ".", "111"]` and `["-", "42"]` respectively. As such, the
-    numeric tokens will be split from their punctuation."""
+    numeric tokens will be split from their punctuation.
+    """
 
     @classmethod
     @override
@@ -252,13 +259,17 @@ class Numeric(Filter):
 
 
 class Punctuation(SubsetFilter):
-    """Identify whether a token is entirely punctuation. Fastest implementation."""
+    """Identify whether a token is entirely punctuation.
+
+    Fastest implementation.
+    """
 
     tokens = set(ALL_PUNCT)
 
 
 class PunctuationRe(RegexFilter):
     """Faster implementation of `PunctuationRe1`.
+
     Goes out of date compared to the `regex` library if UNICODE_PUNCT_RANGES is not updated.
     """
 
@@ -266,7 +277,8 @@ class PunctuationRe(RegexFilter):
 
 
 class PunctuationRe1(Regex1Filter):
-    """Reference implementation for identifying tokens made entirely of punctuation."""
+    """Reference implementation for identifying tokens made entirely of
+    punctuation."""
 
     pattern = regex.compile(
         rf"[\p{{Punctuation}}\p{{posix_punct}}{UCSUR_PUNCT_RANGES}]+"
@@ -278,14 +290,16 @@ class OrFilter:
     returning True when any individual filter matches or False otherwise.
     Requires at least two filters.
 
-    OrFilter exists as a compromise between the need to score some filters equally,
-    while not adding custom behavior to scorers.
-    I could have allowed a position to have a list of filters instead of one filter,
-    but this would require cleaning the user's input, and nested handling of lists.
-    It also would not have been as powerful- I would need another param for the and/or switch,
-    or to not give users the choice.
+    OrFilter exists as a compromise between the need to score some
+    filters equally, while not adding custom behavior to scorers. I
+    could have allowed a position to have a list of filters instead of
+    one filter, but this would require cleaning the user's input, and
+    nested handling of lists. It also would not have been as powerful- I
+    would need another param for the and/or switch, or to not give users
+    the choice.
 
-    Instead, the user is responsible for building an OrFilter out of their desired filters.
+    Instead, the user is responsible for building an OrFilter out of
+    their desired filters.
     """
 
     @staticmethod
@@ -336,10 +350,13 @@ class OrMemberFilter:
         return filter
 
 
-class AndFilter(Filter):
+class AndFilter:
     """Instantiate with more than one filter to compose them into one filter,
-    returning False when any individual filter fails to match or True otherwise.
-    Requires at least two filters."""
+    returning False when any individual filter fails to match or True
+    otherwise.
+
+    Requires at least two filters.
+    """
 
     def __new__(cls, *filters_: Type[Filter]) -> Type[Filter]:
         if not len(filters_) >= 2:

sonatoki/Preprocessors.py

@@ -2,7 +2,7 @@
 "Preprocessors" are classes which strip content from a given string prior to tokenization.
 There are currently two distinct types of Preprocessor:
 
-- Remove a token from a string which would be difficult to identify after tokenization. 
+- Remove a token from a string which would be difficult to identify after tokenization.
   - URLs
   - DiscordEmotes
 - Remove a section of a string which is contained in or marked by certain character(s). Also called "Containers"
@@ -61,21 +61,24 @@ Ignorables are tokens which do not count toward the accepted number of tokens
 or the total number of tokens.
 This is generally because they are considered external to Toki Pona.
 
-It is likely that every user will want to use these. 
+It is likely that every user will want to use these.
 Not having them will cause many false negatives, such as when a URL is divided
 into its parts and checked as a token.
 """
 
 
 class URLs(RegexPreprocessor):
-    """Remove http(s) protocol URLs"""
+    """Remove http(s) protocol URLs."""
 
     pattern = re.compile(r"https?:\/\/\S+")
 
 
 class Reference(RegexPreprocessor):
     """Remove text contained in double brackets.
-    Often used to fetch articles on Wikipedia, or Magic the Gathering cards."""
+
+    Often used to fetch articles on Wikipedia, or Magic the Gathering
+    cards.
+    """
 
     pattern = re.compile(r"\[\[.+\]\]")
 
@@ -100,7 +103,10 @@ class DiscordSpecial(RegexPreprocessor):
 
 class AngleBracketObject(RegexPreprocessor):
     """A generalized version of the Discord-specific angle bracket objects.
-    Removes any contiguous (not broken by whitespace) text in angle brackets."""
+
+    Removes any contiguous (not broken by whitespace) text in angle
+    brackets.
+    """
 
     pattern = re.compile(r"<[^<>\s]+>")
 
@@ -111,7 +117,7 @@ The following classes are Containers.
 Containers are a special case of Ignorables, where an entire segment of an input
 may be removed and not counted toward the accepted or total number of tokens.
 
-Some users may prefer to use these so that they may quote third parties who 
+Some users may prefer to use these so that they may quote third parties who
 would likely be using a language other than Toki Pona.
 """
 

sonatoki/Scorers.py

@@ -13,22 +13,52 @@ Number = Union[int, float]
 Weights = Dict[str, Number]
 
 
-def sigmoid(n: int) -> Number:
-    return 1 / (1 + math.exp(-(0.30 * (n - 1))))
-    # n-1 makes sigmoid(1) == 0.5
-    # 0.30 softens scaling in favor of short input
-    # return n / (1+abs(n))   # too weak in 0.7+
-
-
 class Scorer(ABC):
     @classmethod
     @abstractmethod
     def score(cls, tokens: List[str], filters: List[Type[Filter]]) -> Number:
+        """Score a list of tokens using the given `Filter`s, returning a
+        `Number` between 0 and 1 inclusive."""
         raise NotImplementedError
 
 
+class Soften(Scorer):
+    """Meta `Scorer` which scales the scores of short messages to reduce the
+    impact of shortness on scoring.
+
+    The scores of short messages are scaled by mapping the token count
+    to [0.5, 1.0] via the sigmoid function, then raising the score to
+    the resultant power.
+
+    For example, a single token scoring 0.64 will score 0.8 instead.
+    """
+
+    @staticmethod
+    def sigmoid(n: int) -> Number:
+        return 1 / (1 + math.exp(-(0.30 * (n - 1))))
+        # n-1 makes sigmoid(1) == 0.5
+        # 0.30 softens scaling in favor of short input
+        # return n / (1+abs(n))   # too weak in 0.7+
+
+    @classmethod
+    @override
+    def score(cls, tokens: List[str], filters: List[Type[Filter]]) -> Number:
+        percentage = super().score(tokens, filters)  # type: ignore [abstractmethod]
+        len_tokens = len(tokens)
+        percentage **= cls.sigmoid(len_tokens)
+        return percentage
+
+    def __new__(cls, scorer: Type[Scorer]) -> Type[Scorer]:
+        class SoftenedScorer(Soften, scorer): ...
+
+        return SoftenedScorer
+
+
 class PassFail(Scorer):
-    """The token passes any filter or fails all of them, scoring 1 or 0 respectively."""
+    """If a token matches any filter, it scores 1.
+
+    Otherwise, it scores 0.
+    """
 
     @classmethod
     def score_token(cls, token: str, filters: List[Type[Filter]]) -> Number:
@@ -50,28 +80,17 @@ class PassFail(Scorer):
         return total_score / len_tokens if len_tokens else 0
 
 
-class SoftPassFail(PassFail):
-    @classmethod
-    @override
-    def score(cls, tokens: List[str], filters: List[Type[Filter]]) -> Number:
-        if not tokens:
-            return 1
-
-        total_score = 0
-        len_tokens = len(tokens)
-        for token in tokens:
-            total_score += cls.score_token(token, filters)
-
-        percentage = total_score / len_tokens if len_tokens else 0
-        percentage **= sigmoid(len_tokens)
-        return percentage
+class Scaling(Scorer):
+    """Tokens score 1 for matching the first filter, and a linearly reduced
+    amount for matching later filters based on how many filters there are.
 
+    For example, if there are 4 filters, a token scores 1.0, 0.75, 0.50,
+    and 0.25 for matching each respectively.
 
-class Scaling(Scorer):
-    """
-    The sooner a token matches a filter, the higher its score.
-    In other words, filter order matters, weighing earlier listed filters higher than later ones.
-    This is desirable to avoid messages which would only match weaker filters, as these are less likely to be Toki Pona.
+    In other words, filter order matters, weighing earlier listed
+    filters higher than later ones. This is desirable to avoid messages
+    which would only match weaker filters, as these are less likely to
+    be Toki Pona.
     """
 
     @classmethod
@@ -95,33 +114,17 @@ class Scaling(Scorer):
         return total_score / max_score if max_score else 0
 
 
-class SoftScaling(Scaling):
-    """Shorter messages are subject to less harsh scoring
-    by mapping the token count to [0.5, 1.0] via the sigmoid function,
-    then raising the score to the resultant power.
-    For example, a single token scoring 0.64 will now score 0.8.
-    """
+class SoftPassFail(Soften, PassFail):
+    """Same as `PassFail`, but shorter messages are subject to less harsh
+    scoring."""
 
-    @classmethod
-    @override
-    def score(cls, tokens: List[str], filters: List[Type[Filter]]) -> Number:
-        if not tokens:
-            return 1
 
-        total_score = 0
-        len_filters = len(filters)
-        len_tokens = len(tokens)
-
-        max_score = len_tokens * len_filters
-        for token in tokens:
-            total_score += cls.score_token(token, filters, len_filters)
-
-        percentage = total_score / max_score if max_score else 0
-        percentage **= sigmoid(len_tokens)
-        return percentage
+class SoftScaling(Soften, Scaling):
+    """Same as `Scaling`, but shorter messages are subject to less harsh
+    scoring."""
 
 
-class Logarithmic(Scorer): ...
+# class Logarithmic(Scorer): ...
 
 
 __all__ = ["PassFail", "SoftPassFail", "Scaling", "SoftScaling"]

sonatoki/constants.py

@@ -380,62 +380,29 @@ CONSONANTS = "jklmnpstw"
 ALPHABET = VOWELS + CONSONANTS
 
 LANGUAGE = "english"  # for NLTK
-
-"""Commonly occurring strings which are some kind of valid Toki Pona or external token"""
+"""Commonly occurring strings which are some kind of valid Toki Pona or
+external token."""
 ALLOWABLES = {
     "x",  # ala
     "y",  # anu
     "kxk",  # ken ala ken
     "wxw",  # wile ala wile
+    "msa",
 }
 
 PHONOMATCHES = {
-    # "a",  # ignore
-    # "an",  # against
-    # "i",  # against
-    # "in",  # against
+    "non",
+    "nope",
     "some",
-    "like",  # against
-    # "me",  # against
-    # "no",  # against
-    # "on",  # against
-    # "se",  # against
-    # "so",  # against
-    # "some",  # against
-    "to",  # ignore
-    # "u",  # against
-    # "un",  # against
-    "use",  # against
-    # "we",  # against
+    "like",
+    "use",
+    "imo",
+    "time",
+    "man",
+    "also",
 }
 
-ALPHABETIC_MATCHES = PHONOMATCHES | {
-    "a",
-    # "am",
-    # "as",
-    # "at",
-    # "aw",  # aww
-    # "ek",  # eek
-    # "ew",
-    # "ik",
-    # "il",  # ill
-    # "im",
-    # "im",
-    # "ip",
-    # "is",
-    # "it",
-    # "l",  # they'll
-    # "m",  # i'm
-    # "ok",
-    # "op",
-    # "ow",
-    # "s",  # let's
-    # "t",  # don't
-    # "up",
-    # "us",
-    # "ut",
-    # "uw",
-}
+ALPHABETIC_MATCHES: Set[str] = set()
 
 IGNORABLES = PHONOMATCHES | ALPHABETIC_MATCHES
 

sonatoki/ilo.py

@@ -5,12 +5,17 @@ from typing import List, Type, Tuple
 from sonatoki.Filters import Filter
 from sonatoki.Scorers import Number, Scorer
 from sonatoki.Cleaners import Cleaner
-from sonatoki.Tokenizers import Tokenizer
+from sonatoki.Tokenizers import Tokenizer, SentTokenizer, WordTokenizer
 from sonatoki.Preprocessors import Preprocessor
 
+# tokenized, filtered, cleaned, score, result
+Scorecard = Tuple[List[str], List[str], List[str], Number, bool]
+# TODO: scorecard kinda sucks as a name
+
 
 class Ilo:
     __preprocessors: List[Type[Preprocessor]]
+    __sent_tokenizer: Type[Tokenizer]
     __word_tokenizer: Type[Tokenizer]
     __cleaners: List[Type[Cleaner]]
     __ignoring_filters: List[Type[Filter]]
@@ -26,11 +31,13 @@ class Ilo:
         scoring_filters: List[Type[Filter]],
         scorer: Type[Scorer],
         passing_score: Number,
-        word_tokenizer: Type[Tokenizer],
+        word_tokenizer: Type[Tokenizer] = WordTokenizer,
+        sent_tokenizer: Type[Tokenizer] = SentTokenizer,
     ):
         super().__init__()
         # avoid keeping a ref to user's list just in case
         self.__preprocessors = [*preprocessors]
+        self.__sent_tokenizer = sent_tokenizer
         self.__word_tokenizer = word_tokenizer
         self.__cleaners = [*cleaners]
         self.__ignoring_filters = [*ignoring_filters]
@@ -47,6 +54,9 @@ class Ilo:
         """It is *highly* recommended that you run `ilo.preprocess` first."""
         return self.__word_tokenizer.tokenize(msg)
 
+    def sent_tokenize(self, msg: str) -> List[str]:
+        return self.__sent_tokenizer.tokenize(msg)
+
     def clean_token(self, token: str) -> str:
         for c in self.__cleaners:
             token = c.clean(token)
@@ -83,26 +93,60 @@ class Ilo:
     def score_tokens(self, tokens: List[str]) -> float:
         return self.__scorer.score(tokens, self.__scoring_filters)
 
-    def _is_toki_pona(
-        self, message: str
-    ) -> Tuple[str, List[str], List[str], List[str], Number, bool]:
-        """Returns all components of the processing algorithm:
-        - Preprocessed message (str)
+    def _is_toki_pona(self, message: str) -> Scorecard:
+        """Process a message into its tokens, then filters, cleans, and scores
+        them. Returns all parts. Message must already be preprocessed, normally
+        done in `self.is_toki_pona(message)`.
+
+        Returns all components of the processing algorithm except preprocessing:
         - Tokenized message (list[str])
         - Filtered message (list[str])
         - Cleaned message (list[str])
         - Score (float)
-        - Result (bool)"""
-        preprocessed = self.preprocess(message)
-        tokenized = self.word_tokenize(preprocessed)
+        - Result (bool)
+        """
+        tokenized = self.word_tokenize(message)
         filtered = self.filter_tokens(tokenized)
         cleaned = self.clean_tokens(filtered)
         score = self.score_tokens(cleaned)
         result = score >= self.__passing_score
 
-        return preprocessed, tokenized, filtered, cleaned, score, result
+        return tokenized, filtered, cleaned, score, result
 
     def is_toki_pona(self, message: str) -> bool:
         """Determines whether a single statement is or is not Toki Pona."""
+        message = self.preprocess(message)
         *_, result = self._is_toki_pona(message)
         return result
+
+    def _are_toki_pona(self, message: str):
+        """Split a message into sentences, then return a list each sentence's
+        results via `self._is_toki_pona()`.
+
+        Message must already be preprocessed, normally done in
+        `self.are_toki_pona(message)`.
+        """
+        results: List[Scorecard] = list()
+        for sentence in self.sent_tokenize(message):
+            result = self._is_toki_pona(sentence)
+            results.append(result)
+        return results
+
+    def are_toki_pona(self, message: str) -> List[bool]:
+        """Splits a statement into sentences, then determines if each is or is not Toki Pona.
+        NOTE: You will need to decide how to score the result. Examples:
+
+        ```
+        def all_must_pass(message: str) -> bool:
+            return all(ILO.are_toki_pona(message))
+
+        def portion_must_pass(message: str, score: Number = 0.8) -> bool:
+            results = ILO.are_toki_pona(message)
+            sent_count = len(results)
+            passing = results.count(True)
+            return (passing / sent_count) >= score
+        ```
+        """
+        message = self.preprocess(message)
+        results = self._are_toki_pona(message)
+        return [res[-1] for res in results]