sonatoki 0.1.0__py3-none-any.whl

sonatoki/Cleaners.py

@@ -0,0 +1,42 @@
+# STL
+import re
+from abc import ABC, abstractmethod
+
+# PDM
+from typing_extensions import override
+
+
+class Cleaner(ABC):
+    @classmethod
+    @abstractmethod
+    def clean(cls, token: str) -> str:
+        raise NotImplementedError
+
+
+class RegexCleaner(Cleaner):
+    pattern: "re.Pattern[str]"
+    replace: str
+
+    @classmethod
+    @override
+    def clean(cls, token: str) -> str:
+        return re.sub(cls.pattern, cls.replace, token)
+
+
+class ConsecutiveDuplicates(RegexCleaner):
+    """Remove consecutive duplicates from an input string, ignoring case.
+
+    The first match of any 2+ will become `\\1`, preserving initial case.
+    For example, `FfFoo` will reduce to `Foo`, and `bBAR` will reduce to `bAR`.
+
+    This is desirable for Toki Pona written with the Latin alphabet because strings
+    may be altered for emphasis or effect, such as in "sonaaaa" or "AAAAAA".
+
+    This may be undesirable for moraic scripts like Hiragana, where `わわ` would be
+    incorrectly reduced to `わ`. This does preserve phonotactic validity, though."""
+
+    pattern = re.compile(r"(.)\1+", flags=re.IGNORECASE)
+    replace = r"\1"
+
+
+__all__ = ["ConsecutiveDuplicates"]

sonatoki/Filters.py

@@ -0,0 +1,159 @@
+# STL
+from abc import ABC, abstractmethod
+from typing import Set
+from functools import lru_cache as cache  # cache comes in 3.9
+
+# PDM
+import regex as re
+from typing_extensions import override
+
+# LOCAL
+from sonatoki.constants import (
+    VOWELS,
+    CONSONANTS,
+    NIMI_PU_SET,
+    ALPHABET_SET,
+    ALLOWABLES_SET,
+    NIMI_LINKU_SET,
+    NIMI_PU_ALE_SET,
+    NIMI_LINKU_ALE_SET,
+)
+
+re.DEFAULT_VERSION = re.VERSION1
+
+
+class Filter(ABC):
+    @classmethod
+    @abstractmethod
+    @cache(maxsize=None)
+    def filter(cls, token: str) -> bool:
+        raise NotImplementedError
+
+
+class RegexFilter(Filter):
+    pattern: "re.Pattern[str]"
+
+    @classmethod
+    @override
+    @cache(maxsize=None)
+    def filter(cls, token: str) -> bool:
+        return not not re.fullmatch(cls.pattern, token)
+
+
+class SetFilter(Filter):
+    tokens: Set[str]
+
+    @classmethod
+    @override
+    @cache(maxsize=None)
+    def filter(cls, token: str) -> bool:
+        return token.lower() in cls.tokens
+
+
+class Miscellaneous(SetFilter):
+    tokens = ALLOWABLES_SET
+
+
+class ProperName(Filter):
+    """Determines if a given token is a valid name (also called a loan word).
+    When Toki Pona is written with the Latin alphabet, names are generally
+    capitalized at their start. This filter identifies those tokens.
+
+    Note that this alone cannot determine if a token is a valid name, because
+    a standalone name is considered invalid in Toki Pona- names generally have head nouns.
+    This tool only examines one token at a time, so cannot detect names any better than identifying their capital letter.
+    """
+
+    @classmethod
+    @override
+    @cache(maxsize=None)
+    def filter(cls, token: str) -> bool:
+        return token == token.capitalize()
+
+
+class NimiPu(SetFilter):
+    tokens = NIMI_PU_SET
+
+
+class NimiPuAle(SetFilter):
+    tokens = NIMI_PU_ALE_SET
+
+
+class NimiLinku(SetFilter):
+    tokens = NIMI_LINKU_SET
+
+
+class NimiLinkuAle(SetFilter):
+    tokens = NIMI_LINKU_ALE_SET
+
+
+class Phonotactic(RegexFilter):
+    """Determines if a given token is phonotactically valid Toki Pona (or `n`).
+    Excludes both consecutive nasals and the illegal syllables:
+    - "nm", "nn"
+    - "wu", "wo", "ji", "ti"
+
+    Note that if this validator is used after `Cleaners.ConsecutiveDuplicates`,
+    "nn" cannot be found."""
+
+    pattern = re.compile(
+        rf"^((^[{VOWELS}]|[klmnps][{VOWELS}]|[jt][aeou]|[w][aei])(n(?![mn]))?)+$|^n$",
+        # Can't split initial vowel group off like in Syllabics because of
+        # consecutive nasal detection; it is costly to duplicate
+        flags=re.IGNORECASE,
+    )
+
+
+class Syllabic(RegexFilter):
+    """Determines if a given token is syllabically valid Toki Pona (or `n`).
+    Words must have correctly ordered vowels and consonants, but the phonotactic
+    exceptions are not considered."""
+
+    # rf"^((^[{VOWELS}]|[{CONSONANTS}][{VOWELS}])n?)+$|^n$"
+    # Alterative I was exploring takes ~15% more steps
+    pattern = re.compile(
+        rf"^(?:^[{VOWELS}]n?)?(?:[{CONSONANTS}][{VOWELS}]n?)*$|^n$",
+        flags=re.IGNORECASE,
+    )
+
+
+class Alphabetic(Filter):
+    @classmethod
+    @override
+    @cache(maxsize=None)
+    def filter(cls, token: str) -> bool:
+        # Faster than regex version
+        return set(token.lower()).issubset(ALPHABET_SET)
+
+
+class Numerics(Filter):
+    """Determine if a given token is entirely numeric.
+    Covers all numeric symbols in Unicode.
+
+    This will fail to find numeric tokens such as "1.111" or "-42",
+    but if used with the aggressive tokenizer designed for `tok`, these will be
+    split into `["1", ".", "111"]` and `["-", "42"]` respectively. As such, the
+    numeric tokens will be split from their punctuation."""
+
+    @classmethod
+    @override
+    @cache(maxsize=None)
+    def filter(cls, msg: str) -> bool:
+        return msg.isnumeric()
+
+
+class Punctuations(RegexFilter):
+    pattern = re.compile(r"[\p{Punctuation}\p{posix_punct}]+")
+
+
+__all__ = [
+    "NimiPu",
+    "NimiLinku",
+    "NimiLinkuAle",
+    "Phonotactic",
+    "Syllabic",
+    "Alphabetic",
+    "ProperName",
+    "Punctuations",
+    "Numerics",
+]

sonatoki/Preprocessors.py

@@ -0,0 +1,131 @@
+"""
+"Preprocessors" are classes which strip content from a given string prior to tokenization.
+There are currently two distinct types of Preprocessor:
+
+- Remove a token from a string which would be difficult to identify after tokenization. 
+  - URLs
+  - DiscordEmotes
+- Remove a section of a string which is contained in or marked by certain character(s). Also called "Containers"
+  - SingleQuotes
+  - DoubleQuotes
+  - Backticks
+  - Spoilers
+  - ArrowQuote
+
+Order does not generally matter, but if there were two overlapping containers such as in the string "|| spoiler ` monospace || `", order would matter.
+As such, each Preprocessor exposes a .precedence attribute which is optionally usable for ordering them. Lower precedence means it should be applied first.
+"""
+
+# STL
+from abc import ABC, abstractmethod
+
+# PDM
+import regex as re
+from typing_extensions import override
+
+re.DEFAULT_VERSION = re.VERSION1
+
+
+class Preprocessor(ABC):
+    precedence: int = 0
+
+    @classmethod  # order matters
+    @abstractmethod
+    def process(cls, msg: str) -> str:
+        raise NotImplementedError
+
+
+class RegexPreprocessor(Preprocessor):
+    pattern: "re.Pattern[str]"
+    replace: str = " "
+
+    @classmethod
+    @override
+    def process(cls, msg: str) -> str:
+        return re.sub(cls.pattern, cls.replace, msg)
+
+
+"""
+The following classes are Ignorables.
+
+Ignorables are tokens which do not count toward the accepted number of tokens
+or the total number of tokens.
+This is generally because they are considered external to Toki Pona.
+
+It is likely that every user will want to use these. 
+Not having them will cause many false negatives, such as when a URL is divided
+into its parts and checked as a token.
+"""
+
+
+class URLs(RegexPreprocessor):
+    """Remove http(s) protocol URLs"""
+
+    pattern = re.compile(r"https?:\/\/\S+")
+
+
+class DiscordEmotes(RegexPreprocessor):
+    """Remove text-formatted Discord emotes `<flags:name:id>`"""
+
+    pattern = re.compile(r"<a?:[a-zA-Z0-9_]{2,}:[0-9]{2,}>")
+
+
+class DiscordMentions(RegexPreprocessor):
+    pattern = re.compile(r"<@[\!\&]?[0-9]{2,}>")
+
+
+class DiscordChannels(RegexPreprocessor):
+    pattern = re.compile(r"<#[0-9]{2,}>")
+
+
+class DiscordSpecial(RegexPreprocessor):
+    pattern = re.compile(r"<id:[a-zA-Z0-9_]{4,}>")
+
+
+"""
+The following classes are Containers.
+
+Containers are a special case of Ignorables, where an entire segment of an input
+may be removed and not counted toward the accepted or total number of tokens.
+
+Some users may prefer to use these so that they may quote third parties who 
+would likely be using a language other than Toki Pona.
+"""
+
+
+class SingleQuotes(RegexPreprocessor):
+    pattern = re.compile(r"'[^']+'", flags=re.S)  # . matches newline
+
+
+class DoubleQuotes(RegexPreprocessor):
+    pattern = re.compile(r'"[^"]+"', flags=re.S)
+
+
+class Backticks(RegexPreprocessor):
+    """Remove paired backticks and their contents `like this`"""
+
+    precedence = -10
+    pattern = re.compile(r"`[^`]+`", flags=re.S)
+
+
+class Spoilers(RegexPreprocessor):
+    """Remove paired double bars and their contents `||like this||`"""
+
+    pattern = re.compile(r"\|\|(?:(?!\|\|).)+\|\|", flags=re.S)
+
+
+class ArrowQuote(RegexPreprocessor):
+    """Remove lines beginning with `> `"""
+
+    pattern = re.compile(r"^>\ .+$", re.MULTILINE)
+
+
+__all__ = [
+    "DiscordEmotes",
+    "SingleQuotes",
+    "DoubleQuotes",
+    "ArrowQuote",
+    "Backticks",
+    "Spoilers",
+    "URLs",
+]

sonatoki/Scorers.py

@@ -0,0 +1,123 @@
+# STL
+import math
+from abc import ABC, abstractmethod
+from typing import Dict, List, Type, Union
+
+# PDM
+from typing_extensions import override
+
+# LOCAL
+from sonatoki.Filters import Filter
+
+Number = Union[int, float]
+Weights = Dict[str, Number]
+
+
+class Scorer(ABC):
+    weights: Weights
+
+    # @classmethod
+    # def __score(cls, token: str, filters: List[Type[Filter]]) -> Tuple[int, Number]:
+    #     for filter in filters:
+    #         if not filter.filter(token):
+    #             continue
+    #         # NOTE: We assume the filters are ordered by their score
+    #         # Thus the first match is also the highest scoring
+    #         return filter.counts, cls.weights[filter.__name__]
+    #         # TODO: override weight if count is 0?
+    #     return 1, 0
+
+    @classmethod
+    @abstractmethod
+    def score(cls, tokens: List[str], filters: List[Type[Filter]]) -> Number:
+        raise NotImplementedError
+
+
+class PassFail(Scorer):
+    """The token passes any filter or fails all of them, scoring 1 or 0 respectively."""
+
+    @classmethod
+    def __score(cls, token: str, filters: List[Type[Filter]]) -> Number:
+        for f in filters:
+            if f.filter(token):
+                return 1
+        return 0
+
+    @classmethod
+    @override
+    def score(cls, tokens: List[str], filters: List[Type[Filter]]) -> Number:
+        if not tokens:
+            return 1
+
+        total_score = 0
+        len_tokens = len(tokens)
+        for token in tokens:
+            total_score += cls.__score(token, filters)
+        return total_score / len_tokens if len_tokens else 0
+
+
+class Scaling(Scorer):
+    """
+    The sooner a token matches a filter, the higher its score.
+    In other words, filter order matters, weighing earlier listed filters higher than later ones.
+    This is desirable to avoid messages which would only match weaker filters, as these are less likely to be Toki Pona.
+    """
+
+    @classmethod
+    def score_token(cls, token: str, filters: List[Type[Filter]], scale: int):
+        for i, f in enumerate(filters):
+            if f.filter(token):
+                return scale - i
+        return 0
+
+    @classmethod
+    @override
+    def score(cls, tokens: List[str], filters: List[Type[Filter]]) -> Number:
+        if not tokens:
+            return 1
+
+        total_score = 0
+        len_filters = len(filters)
+        max_score = len(tokens) * len_filters
+        for token in tokens:
+            total_score += cls.score_token(token, filters, len_filters)
+        return total_score / max_score if max_score else 0
+
+
+class SoftScaling(Scaling):
+    """Shorter messages are subject to less harsh scoring
+    by mapping the token count to [0.5, 1.0] via the sigmoid function,
+    then raising the score to the resultant power.
+    For example, a single token scoring 0.64 will now score 0.8.
+    """
+
+    @staticmethod
+    def sigmoid(n: int) -> Number:
+        return 1 / (1 + math.exp(-(0.30 * (n - 1))))
+        # n-1 makes sigmoid(1) == 0.5
+        # 0.30 softens scaling against input
+        # return n / (1+abs(n))   # too weak in 0.7+
+
+    @classmethod
+    @override
+    def score(cls, tokens: List[str], filters: List[Type[Filter]]) -> Number:
+        if not tokens:
+            return 1
+
+        total_score = 0
+        len_filters = len(filters)
+        len_tokens = len(tokens)
+
+        max_score = len_tokens * len_filters
+        for token in tokens:
+            total_score += cls.score_token(token, filters, len_filters)
+
+        percentage = total_score / max_score if max_score else 0
+        percentage **= cls.sigmoid(len_tokens)
+        return percentage
+
+
+class Logarithmic(Scorer): ...
+
+
+__all__ = ["PassFail", "Scaling", "SoftScaling"]

sonatoki/Tokenizers.py

@@ -0,0 +1,64 @@
+# STL
+from typing import List, Callable
+
+# PDM
+import regex as re
+
+try:
+    # PDM
+    import nltk
+    from nltk.tokenize import sent_tokenize as __sent_tokenize_nltk
+    from nltk.tokenize import word_tokenize as __word_tokenize_nltk
+except ImportError as e:
+    nltk = e
+
+
+LANGUAGE = "english"  # for NLTK
+
+SENT_DELIMS_RE = r"""(.*?[.?!;:])|(.+?$)"""
+SENT_DELIMS_RE = re.compile(SENT_DELIMS_RE)
+
+SENT_DELIMS_TOK = r"""(.*?[.?!;:-])|(.+?$)"""
+SENT_DELIMS_TOK = re.compile(SENT_DELIMS_TOK)
+
+
+WORD_DELIMS_RE = r"""\s+|(?=[.?!;:'"-])"""
+WORD_DELIMS_RE = re.compile(WORD_DELIMS_RE)
+
+WORD_DELIMS_TOK = r"([\p{Punctuation}\p{posix_punct}]+|\s+)"
+WORD_DELIMS_TOK = re.compile(WORD_DELIMS_TOK)
+
+Tokenizer = Callable[[str], List[str]]
+
+
+if not isinstance(nltk, ImportError):
+
+    def sent_tokenize_nltk(s: str) -> List[str]:
+        return __sent_tokenize_nltk(text=s, language=LANGUAGE)
+
+    def word_tokenize_nltk(s: str) -> List[str]:
+        return __word_tokenize_nltk(text=s, language=LANGUAGE)
+
+
+def sent_tokenize_re(s: str) -> List[str]:
+    return [
+        clean
+        for sent in re.findall(SENT_DELIMS_RE, s)
+        if (clean := sent[0].strip() or sent[1].strip())
+    ]
+
+
+def word_tokenize_re(s: str) -> List[str]:
+    return [clean for word in re.split(WORD_DELIMS_RE, s) if (clean := word.strip())]
+
+
+def sent_tokenize_tok(s: str) -> List[str]:
+    return [
+        clean
+        for sent in re.findall(SENT_DELIMS_TOK, s)
+        if (clean := sent[0].strip() or sent[1].strip())
+    ]
+
+
+def word_tokenize_tok(s: str) -> List[str]:
+    return [clean for word in re.split(WORD_DELIMS_TOK, s) if (clean := word.strip())]

sonatoki/__main__.py

@@ -0,0 +1,9 @@
+#!/bin/env python3
+
+
+def open():
+    pass
+
+
+if __name__ == "__main__":
+    open()

sonatoki/constants.py

@@ -0,0 +1,57 @@
+# STL
+import json
+from typing import Dict, List
+from pathlib import Path
+
+LINKU = Path(__file__).resolve().parent / Path("linku.json")
+
+VOWELS = "aeiou"
+CONSONANTS = "jklmnpstw"
+ALPHABET = VOWELS + CONSONANTS
+ALPHABET_SET = set(ALPHABET)
+
+"""Commonly occurring strings which are some kind of valid Toki Pona or external token"""
+ALLOWABLES = {
+    "cw",  # Content Warning
+    "x",  # ala
+    "y",  # anu
+    "kxk",  # ken ala ken
+    "wxw",  # wile ala wile
+}
+
+
+with open(LINKU) as f:
+    r: Dict[str, Dict[str, str]] = json.loads(f.read())
+    NIMI_PU: List[str] = [d["word"] for d in r.values() if d["book"] == "pu"]
+    NIMI_PU_ALE: List[str] = NIMI_PU + ["namako", "kin", "oko"]
+    NIMI_LINKU: List[str] = [
+        d["word"] for d in r.values() if d["usage_category"] in ["core", "common"]
+    ]
+    NIMI_LINKU_ALE: List[str] = [d["word"] for d in r.values()]
+
+NIMI_PU_SET = set(NIMI_PU)
+NIMI_PU_ALE_SET = set(NIMI_PU_ALE)
+NIMI_LINKU_SET = set(NIMI_LINKU)
+NIMI_LINKU_ALE_SET = set(NIMI_LINKU_ALE)
+ALLOWABLES_SET = set(ALLOWABLES)
+
+__all__ = [
+    "VOWELS",
+    #
+    "CONSONANTS",
+    #
+    "ALPHABET",
+    "ALPHABET_SET",
+    #
+    "NIMI_PU",
+    "NIMI_PU_SET",
+    #
+    "NIMI_PU_ALE",
+    "NIMI_PU_ALE_SET",
+    #
+    "NIMI_LINKU",
+    "NIMI_LINKU_SET",
+    #
+    "NIMI_LINKU_ALE",
+    "NIMI_LINKU_ALE_SET",
+]

sonatoki/ilo.py

@@ -0,0 +1,101 @@
+# STL
+from typing import List, Type
+
+# LOCAL
+from sonatoki.Filters import Filter
+from sonatoki.Scorers import Number, Scorer
+from sonatoki.Cleaners import Cleaner
+from sonatoki.Tokenizers import Tokenizer
+from sonatoki.Preprocessors import Preprocessor
+
+
+class Ilo:
+    __preprocessors: List[Type[Preprocessor]]
+    __cleaners: List[Type[Cleaner]]
+    __ignoring_filters: List[Type[Filter]]
+    __scoring_filters: List[Type[Filter]]
+    __scorer: Type[Scorer]
+    __tokenize: Tokenizer
+    __passing_score: Number
+    debug: bool = False
+
+    def __init__(
+        self,
+        preprocessors: List[Type[Preprocessor]],
+        cleaners: List[Type[Cleaner]],
+        ignoring_filters: List[Type[Filter]],
+        scoring_filters: List[Type[Filter]],
+        scorer: Type[Scorer],
+        tokenizer: Tokenizer,  # NOTE: no wrapper needed?
+        passing_score: Number,
+    ):
+        super().__init__()
+        # avoid keeping a ref to user's list just in case
+        self.__preprocessors = [*preprocessors]
+        self.__cleaners = [*cleaners]
+        self.__ignoring_filters = [*ignoring_filters]
+        self.__scoring_filters = [*scoring_filters]
+        self.__scorer = scorer
+        self.__tokenize = tokenizer
+        self.__passing_score = passing_score
+
+    def __preprocess(self, msg: str) -> str:
+        for p in self.__preprocessors:
+            msg = p.process(msg)
+        return msg
+
+    def __clean_token(self, token: str) -> str:
+        for c in self.__cleaners:
+            token = c.clean(token)
+        return token
+
+    def __clean_tokens(self, tokens: List[str]) -> List[str]:
+        # NOTE: tested, making a new list with a for loop *is* faster than
+        # - list comps
+        # - generator comps
+        # - in-place replacement/removal
+        # - in place replacement with result of generator comp
+        cleaned_tokens: List[str] = list()
+        for token in tokens:
+            cleaned_token = self.__clean_token(token)
+            if not cleaned_token:
+                # TODO: warn user?
+                continue
+            cleaned_tokens.append(cleaned_token)
+        return cleaned_tokens
+
+    def __filter_token(self, token: str) -> bool:
+        for f in self.__ignoring_filters:
+            if f.filter(token):
+                return True
+        return False
+
+    def __filter_tokens(self, tokens: List[str]) -> List[str]:
+        filtered_tokens: List[str] = []
+        for token in tokens:
+            if self.__filter_token(token):
+                continue
+            # the ignoring filter is true if the token matches
+            # the user wants to ignore these so keep non-matching tokens
+            filtered_tokens.append(token)
+        return filtered_tokens
+
+    def __score_tokens(self, tokens: List[str]) -> float:
+        return self.__scorer.score(tokens, self.__scoring_filters)
+
+    def is_toki_pona(self, message: str) -> bool:
+        preprocessed = self.__preprocess(message)
+        tokenized = self.__tokenize(preprocessed)
+        filtered = self.__filter_tokens(tokenized)
+        cleaned = self.__clean_tokens(filtered)
+        score = self.__score_tokens(cleaned)
+
+        if self.debug:
+            print("msg: %.2f  %s" % (score, repr(message)))
+            print("Preproc:   %s" % repr(preprocessed))
+            print("Tokenized: %s" % tokenized)
+            print("Filtered:  %s" % filtered)
+            print("Cleaned:   %s" % cleaned)
+            print()
+
+        return score >= self.__passing_score