sonatoki 0.6.2__py3-none-any.whl → 0.7.0__py3-none-any.whl

sonatoki/Configs.py

@@ -1,49 +1,74 @@
 # STL
-from copy import deepcopy
-from typing import Set, List, Type, TypedDict, cast
+from typing import List, Type, TypedDict
 
 # PDM
 from typing_extensions import NotRequired
 
 # LOCAL
+from sonatoki.types import Number
 from sonatoki.Filters import (
     Or,
     And,
     Not,
     Filter,
     Numeric,
-    Syllabic,
     NimiUCSUR,
     Alphabetic,
     NimiKuLili,
     NimiKuSuli,
     ProperName,
-    Phonotactic,
     Punctuation,
     LongSyllabic,
     Miscellaneous,
-    NimiLinkuCore,
     LongAlphabetic,
     LongProperName,
-    NimiLinkuCommon,
     FalsePosSyllabic,
+    NimiLinkuByUsage,
     NimiLinkuObscure,
     NimiLinkuSandbox,
     NimiLinkuUncommon,
     FalsePosAlphabetic,
 )
-from sonatoki.Scorers import Number, Scorer, PassFail, SoftScaling, SoftPassFail
+from sonatoki.Scorers import Scorer, PassFail, SoftScaling, SoftPassFail
 from sonatoki.Cleaners import Cleaner, ConsecutiveDuplicates
 from sonatoki.Tokenizers import Tokenizer, WordTokenizerRe
 from sonatoki.Preprocessors import (
     URLs,
     Emoji,
-    Backticks,
+    Codeblock,
     Reference,
     Preprocessor,
     AngleBracketObject,
 )
 
+__DICT_PHONOMATCHES = {
+    # Sandbox words are removed from the CorpusConfig if they appear more frequently in English than Toki Pona by a factor of at least 3.
+    # In this case, all of these appear more often in English by a factor of at least 10.
+    "aka",  # also known as
+    "an",  # article
+    "api",  # API
+    "i",  # 1st person
+    "kana",  # japanese script
+    "me",  # 1st person singular, english
+    "ne",  # "no" in several languages
+    "nu",  # "new" in english, "now" in dutch
+    "se",  # spanish particle, english "see"
+    "take",  # acquire, perhaps forcefully or without permission
+    "ten",  # 10
+    "to",  # to, too
+    "je",  # 1st person pronoun, french
+    "u",  # no u
+    "we",  # 1st person plural, english
+    "wi",  # wii and discussions of syllables
+    "sole",  # singular, of shoe
+    # unexplored candidates for removal
+    # "omen",  # ominous
+    # "papa",  # father
+    # "lo",  # "lo" and "loo"
+    # "ewe",  # sheep
+    # "pa",  # father- eh?
+}
+
 
 class IloConfig(TypedDict):
     preprocessors: List[Type[Preprocessor]]
@@ -69,11 +94,11 @@ BaseConfig: IloConfig = {
 
 
 PrefConfig: IloConfig = {
-    "preprocessors": [Emoji, Backticks, URLs, AngleBracketObject, Reference],
+    "preprocessors": [Emoji, Codeblock, URLs, AngleBracketObject, Reference],
     "cleaners": [ConsecutiveDuplicates],
     "ignoring_filters": [Numeric, Punctuation],
     "scoring_filters": [
-        Or(NimiLinkuCore, NimiLinkuCommon, NimiLinkuUncommon, NimiUCSUR),
+        Or(NimiLinkuByUsage(30), NimiUCSUR),
         And(LongSyllabic, Not(FalsePosSyllabic)),
         # NOTE: These are allowed to pass name and alphabetic below, because they *could* be wrong
         LongProperName,
@@ -84,16 +109,13 @@ PrefConfig: IloConfig = {
 }
 
 CorpusConfig: IloConfig = {
-    "preprocessors": [Emoji, Backticks, URLs, AngleBracketObject, Reference],
+    "preprocessors": [Emoji, Codeblock, URLs, AngleBracketObject, Reference],
     "cleaners": [ConsecutiveDuplicates],
     "ignoring_filters": [Numeric, Punctuation],
     "scoring_filters": [
         Or(
-            NimiLinkuCore,
-            NimiLinkuCommon,
-            NimiLinkuUncommon,
-            NimiLinkuObscure,
-            NimiLinkuSandbox,
+            # awkward but efficient syntax
+            NimiLinkuByUsage(0)(sub=__DICT_PHONOMATCHES),
             NimiUCSUR,
             Miscellaneous,
         ),
@@ -104,43 +126,9 @@ CorpusConfig: IloConfig = {
     "scorer": SoftScaling,
     "passing_score": 0.8,
 }
-
-# TODO: create a mechanism to omit tokens from a filter with more granularity
-__corpus_tokens_dict: Set[str] = cast(
-    Set[str],
-    CorpusConfig["scoring_filters"][
-        0
-    ].tokens,  # pyright: ignore[reportAttributeAccessIssue]
-)
-__corpus_tokens_dict -= {
-    # Sandbox words are removed from the CorpusConfig if they appear more frequently in English than Toki Pona by a factor of at least 3.
-    # In this case, all of these appear more often in English by a factor of at least 10.
-    "aka",  # also known as
-    "an",  # article
-    "api",  # API
-    "i",  # 1st person
-    "kana",  # japanese script
-    "me",  # 1st person
-    "ne",  # "no" in several languages
-    "nu",  # "new", now in dutch
-    "se",  # spanish particle, "see"
-    "take",  # acquire, perhaps forcefully or without permission
-    "ten",  # 10
-    "to",  # to, too
-    "u",  # no u
-    "we",  # 1st person plural
-    "wi",  # wii and discussions of syllables
-    "sole",  # singular, of shoe
-    # unexplored candidates for removal
-    # "omen",  # ominous
-    # "papa",  # father
-    # "lo",  # "lo" and "loo"
-    # "ewe",  # sheep
-    # "pa",  # father- eh?
-}
 """Mimics the previous implementation of ilo pi toki pona taso."""
 LazyConfig: IloConfig = {
-    "preprocessors": [Emoji, Backticks, URLs, AngleBracketObject, Reference],
+    "preprocessors": [Emoji, Codeblock, URLs, AngleBracketObject, Reference],
     "cleaners": [ConsecutiveDuplicates],
     "ignoring_filters": [Numeric, Punctuation],
     "scoring_filters": [Alphabetic, NimiUCSUR, ProperName, Miscellaneous],
@@ -150,7 +138,7 @@ LazyConfig: IloConfig = {
 }
 """This is extremely silly."""
 IsipinEpikuConfig: IloConfig = {
-    "preprocessors": [Emoji, Backticks, URLs, AngleBracketObject, Reference],
+    "preprocessors": [Emoji, Codeblock, URLs, AngleBracketObject, Reference],
     "cleaners": [ConsecutiveDuplicates],
     "ignoring_filters": [Numeric, Punctuation],
     "scoring_filters": [
@@ -170,31 +158,10 @@ IsipinEpikuConfig: IloConfig = {
 }
 
 
-DiscordConfig: IloConfig = {
-    "preprocessors": [Emoji, Backticks, URLs, AngleBracketObject, Reference],
-    "cleaners": [ConsecutiveDuplicates],
-    "ignoring_filters": [Numeric, Punctuation],
-    "scoring_filters": [
-        Or(NimiLinkuCore, NimiLinkuCommon, NimiUCSUR, Miscellaneous),
-        And(LongSyllabic, Not(FalsePosSyllabic)),
-        LongProperName,
-        And(LongAlphabetic, Not(FalsePosAlphabetic)),
-    ],
-    "scorer": SoftScaling,
-    "passing_score": 0.8,
-}
-
-TelegramConfig: IloConfig = deepcopy(PrefConfig)
-ForumConfig: IloConfig = deepcopy(PrefConfig)
-
-
 __all__ = [
     "BaseConfig",
     "CorpusConfig",
-    "DiscordConfig",
-    "ForumConfig",
     "IloConfig",
     "LazyConfig",
     "PrefConfig",
-    "TelegramConfig",
 ]

sonatoki/Filters.py

@@ -1,37 +1,33 @@
 # STL
 import re
 from abc import ABC, abstractmethod
-from typing import Set, List, Type
+from copy import deepcopy
+from typing import Set, List, Type, Union, Literal, Optional
 from functools import lru_cache as cache  # cache comes in 3.9
 
 # PDM
 import regex
-from typing_extensions import override, deprecated
+from typing_extensions import override
 
 # LOCAL
+from sonatoki.types import LinkuBooks, LinkuUsageDate, LinkuUsageCategory
 from sonatoki.utils import prep_dictionary
 from sonatoki.constants import (
     VOWELS,
-    NIMI_PU,
     ALPHABET,
     ALL_PUNCT,
     ALLOWABLES,
     CONSONANTS,
     NIMI_UCSUR,
-    NIMI_KU_LILI,
-    NIMI_KU_SULI,
-    NIMI_LINKU_CORE,
     NIMI_PU_SYNONYMS,
-    NIMI_LINKU_COMMON,
     FALSE_POS_SYLLABIC,
-    NIMI_LINKU_OBSCURE,
-    NIMI_LINKU_SANDBOX,
     NOT_IN_PUNCT_CLASS,
-    NIMI_LINKU_UNCOMMON,
     ALL_PUNCT_RANGES_STR,
     FALSE_POS_ALPHABETIC,
     UCSUR_PUNCT_RANGES_STR,
     EMOJI_VARIATION_SELECTOR_RANGES_STR,
+    words_by_tag,
+    words_by_usage,
 )
 
 regex.DEFAULT_VERSION = regex.VERSION1
@@ -101,6 +97,20 @@ class MemberFilter(Filter):
     def filter(cls, token: str) -> bool:
         return token.lower() in cls.tokens
 
+    def __new__(
+        cls, add: Optional[Set[str]] = None, sub: Optional[Set[str]] = None
+    ) -> Type[Filter]:
+        parent_tokens = deepcopy(cls.tokens)
+        if add:
+            parent_tokens = parent_tokens.union(add)
+        if sub:
+            parent_tokens -= sub
+
+        class AnonMemberFilter(MemberFilter):
+            tokens = parent_tokens
+
+        return AnonMemberFilter
+
 
 class SubsetFilter(Filter):
     tokens: Set[str]
@@ -155,40 +165,46 @@ class LongProperName(MinLen, ProperName):
     length = 2  # reject "names" of length 1
 
 
-class NimiPu(MemberFilter):
-    tokens = prep_dictionary(NIMI_PU)
-
-
-class NimiPuSynonyms(MemberFilter):
-    tokens = prep_dictionary(NIMI_PU_SYNONYMS)
+class NimiLinkuByUsage:
+    def __new__(
+        cls,
+        usage: int,
+        date: Optional[LinkuUsageDate] = None,
+    ) -> Type[MemberFilter]:
+        words = words_by_usage(usage, date)
 
+        class AnonLinkuMemberFilter(MemberFilter):
+            tokens = prep_dictionary(words)
 
-class NimiKuSuli(MemberFilter):
-    tokens = prep_dictionary(NIMI_KU_SULI)
+        return AnonLinkuMemberFilter
 
 
-class NimiKuLili(MemberFilter):
-    tokens = prep_dictionary(NIMI_KU_LILI)
+class NimiLinkuByTag:
+    def __new__(
+        cls,
+        tag: Union[Literal["usage_category"], Literal["book"]],
+        category: Union[LinkuUsageCategory, LinkuBooks],
+    ) -> Type[MemberFilter]:
+        words = words_by_tag(tag, category)
 
+        class AnonLinkuMemberFilter(MemberFilter):
+            tokens = prep_dictionary(words)
 
-class NimiLinkuCore(MemberFilter):
-    tokens = prep_dictionary(NIMI_LINKU_CORE)
+        return AnonLinkuMemberFilter
 
 
-class NimiLinkuCommon(MemberFilter):
-    tokens = prep_dictionary(NIMI_LINKU_COMMON)
+NimiPu = NimiLinkuByTag("book", "pu")
+NimiKuSuli = NimiLinkuByTag("book", "ku suli")
+NimiKuLili = NimiLinkuByTag("book", "ku lili")
+NimiLinkuCore = NimiLinkuByTag("usage_category", "core")
+NimiLinkuCommon = NimiLinkuByTag("usage_category", "common")
+NimiLinkuUncommon = NimiLinkuByTag("usage_category", "uncommon")
+NimiLinkuObscure = NimiLinkuByTag("usage_category", "obscure")
+NimiLinkuSandbox = NimiLinkuByTag("usage_category", "sandbox")
 
 
-class NimiLinkuUncommon(MemberFilter):
-    tokens = prep_dictionary(NIMI_LINKU_UNCOMMON)
-
-
-class NimiLinkuObscure(MemberFilter):
-    tokens = prep_dictionary(NIMI_LINKU_OBSCURE)
-
-
-class NimiLinkuSandbox(MemberFilter):
-    tokens = prep_dictionary(NIMI_LINKU_SANDBOX)
+class NimiPuSynonyms(MemberFilter):
+    tokens = prep_dictionary(NIMI_PU_SYNONYMS)
 
 
 class NimiUCSUR(MemberFilter):

sonatoki/Preprocessors.py

@@ -143,6 +143,15 @@ class Backticks(RegexPreprocessor):
     pattern = re.compile(r"`[^`]+`", flags=re.DOTALL)
 
 
+class Codeblock(RegexPreprocessor):
+    """Remove codeblocks marked by a set of three backticks on their own lines.
+
+    Subset of what would be removed by Backticks, but may be preferable.
+    """
+
+    pattern = re.compile(r"```\n(?:(?!```).*?)?```", flags=re.DOTALL)
+
+
 class Spoilers(RegexPreprocessor):
     """Remove paired double bars and their contents `||like this||`"""
 

sonatoki/Scorers.py

@@ -1,17 +1,15 @@
 # STL
 import math
 from abc import ABC, abstractmethod
-from typing import Dict, List, Type, Union
+from typing import List, Type
 
 # PDM
 from typing_extensions import override
 
 # LOCAL
+from sonatoki.types import Number, Scorecard
 from sonatoki.Filters import Filter
 
-Number = Union[int, float]
-Weights = Dict[str, Number]
-
 
 class Scorer(ABC):
     @classmethod
@@ -124,7 +122,64 @@ class SoftScaling(Soften, Scaling):
     scoring."""
 
 
-# class Logarithmic(Scorer): ...
+class SentenceScorer(ABC):
+    @classmethod
+    @abstractmethod
+    def score(cls, scorecards: List[Scorecard]) -> List[Scorecard]:
+        """Re-score a list of sentences (scorecards, sentences with all their
+        metadata) and return them."""
+        raise NotImplementedError
+
+
+class SentNoOp(SentenceScorer):
+    @classmethod
+    @override
+    def score(cls, scorecards: List[Scorecard]) -> List[Scorecard]:
+        return scorecards
 
 
-__all__ = ["PassFail", "SoftPassFail", "Scaling", "SoftScaling"]
+class SentAvg(SentenceScorer):
+    @classmethod
+    @override
+    def score(cls, scorecards: List[Scorecard]) -> List[Scorecard]:
+        if not scorecards:
+            return scorecards
+
+        total = sum(card["score"] for card in scorecards)
+        avg = total / len(scorecards)
+        for card in scorecards:
+            card["score"] = avg
+        return scorecards
+
+
+class SentWeightedAvg(SentenceScorer):
+    @classmethod
+    @override
+    def score(cls, scorecards: List[Scorecard]) -> List[Scorecard]:
+        if not scorecards:
+            return scorecards
+
+        weighted_total = 0
+        total_len = 0
+        for card in scorecards:
+            cardlen = len(card["cleaned"])
+            cardscore = card["score"]
+
+            weighted_total += cardlen * cardscore
+            total_len += cardlen
+
+        weighted_avg = weighted_total / total_len
+        for card in scorecards:
+            card["score"] = weighted_avg
+        return scorecards
+
+
+__all__ = [
+    "PassFail",
+    "Scaling",
+    "SoftPassFail",
+    "SoftScaling",
+    "Soften",
+    "SentAvg",
+    "SentWeightedAvg",
+]

sonatoki/constants.py

@@ -1,11 +1,16 @@
 # STL
 import json
-from typing import Set, Dict
+from typing import Set, Dict, Optional
 from pathlib import Path
 
 # LOCAL
+from sonatoki.types import LinkuWord, LinkuUsageDate
 from sonatoki.utils import find_unicode_chars, find_unicode_ranges
 
+LATEST_DATE = "2023-09"
+# hardcoding this seems bad, but it means the parser is stable w.r.t. Linku!
+
+
 # `\p{Punctuation}` character class
 # https://www.compart.com/en/unicode/category
 # https://unicode.org/Public/UNIDATA/UnicodeData.txt
@@ -638,6 +643,7 @@ FALSE_POS_SYLLABIC = {
     "iluminate",
     "imense",
     "imitate",
+    "inanimate",
     "injoke",
     "insane",
     "insolate",
@@ -689,26 +695,42 @@ NIMI_UCSUR = find_unicode_chars(UCSUR_RANGES)
 # NIMI_PU_ALE_UCSUR_RANGES = NIMI_PU_UCSUR_RANGES + ["\\U000F1978-\\U000F197A"]
 
 
-def category_helper(data: Dict[str, Dict[str, str]], key: str, value: str) -> Set[str]:
-    return {d["word"] for d in data.values() if d[key] == value}
+def linku_data() -> Dict[str, LinkuWord]:
+    # NOTE: this does open+read+parse two files each time you construct a filter
+    # but i expect users to construct filters only at the start of runtime
+    # there is no reason to waste your RAM by leaving the linku data in it
+    with open(LINKU) as f:
+        linku: Dict[str, LinkuWord] = json.loads(f.read())
+    with open(SANDBOX) as f:
+        sandbox: Dict[str, LinkuWord] = json.loads(f.read())
+
+    return {**linku, **sandbox}
+
 
+def words_by_tag(tag: str, value: str) -> Set[str]:
+    data = linku_data()
+    return {d["word"] for d in data.values() if d[tag] == value}
 
-with open(LINKU) as f:
-    linku: Dict[str, Dict[str, str]] = json.loads(f.read())
-    NIMI_PU = category_helper(linku, "book", "pu")
-    NIMI_PU_SYNONYMS = {"namako", "kin", "oko"}
 
-    NIMI_KU_SULI = category_helper(linku, "book", "ku suli")
-    NIMI_KU_LILI = category_helper(linku, "book", "ku lili")
+def words_by_usage(
+    usage: int,
+    date: Optional[LinkuUsageDate] = None,
+) -> Set[str]:
+    if not date:
+        date = LATEST_DATE
+    data = linku_data()
 
-    NIMI_LINKU_CORE = category_helper(linku, "usage_category", "core")
-    NIMI_LINKU_COMMON = category_helper(linku, "usage_category", "common")
-    NIMI_LINKU_UNCOMMON = category_helper(linku, "usage_category", "uncommon")
-    NIMI_LINKU_OBSCURE = category_helper(linku, "usage_category", "obscure")
+    result: Set[str] = set()
+    for word in data.values():
+        usages = word["usage"]
+        if date in usages and usages[date] >= usage:
+            result.add(word["word"])
+
+    return result
+
+
+NIMI_PU_SYNONYMS = {"namako", "kin", "oko"}
 
-with open(SANDBOX) as f:
-    sandbox: Dict[str, Dict[str, str]] = json.loads(f.read())
-    NIMI_LINKU_SANDBOX = {d["word"] for d in sandbox.values()}
 
 # with open(SYLLABICS) as f:
 #     FALSE_POS_SYLLABIC = {line.strip() for line in f}
@@ -716,9 +738,6 @@ with open(SANDBOX) as f:
 # with open(ALPHABETICS) as f:
 #     FALSE_POS_ALPHABETIC = {line.strip() for line in f}
 
-del linku
-del sandbox
-
 __all__ = [
     "ALLOWABLES",
     "ALL_PUNCT",
@@ -727,14 +746,6 @@ __all__ = [
     "CONSONANTS",
     "EMOJI_VARIATION_SELECTOR_RANGES",
     "EMOJI_VARIATION_SELECTOR_RANGES_STR",
-    "NIMI_KU_LILI",
-    "NIMI_KU_SULI",
-    "NIMI_LINKU_COMMON",
-    "NIMI_LINKU_CORE",
-    "NIMI_LINKU_OBSCURE",
-    "NIMI_LINKU_SANDBOX",
-    "NIMI_LINKU_UNCOMMON",
-    "NIMI_PU",
     "NIMI_PU_SYNONYMS",
     "POSIX_PUNCT",
     "POSIX_PUNCT_RANGES",

sonatoki/ilo.py

@@ -1,17 +1,14 @@
 # STL
-from typing import List, Type, Tuple
+from typing import List, Type
 
 # LOCAL
+from sonatoki.types import Number, Scorecard
 from sonatoki.Filters import Filter
-from sonatoki.Scorers import Number, Scorer
+from sonatoki.Scorers import Scorer, SentNoOp, SentenceScorer
 from sonatoki.Cleaners import Cleaner
 from sonatoki.Tokenizers import Tokenizer, SentTokenizer, WordTokenizer
 from sonatoki.Preprocessors import Preprocessor
 
-# tokenized, filtered, cleaned, score, result
-Scorecard = Tuple[List[str], List[str], List[str], Number, bool]
-# TODO: scorecard kinda sucks as a name
-
 
 class Ilo:
     __preprocessors: List[Type[Preprocessor]]
@@ -21,6 +18,7 @@ class Ilo:
     __ignoring_filters: List[Type[Filter]]
     __scoring_filters: List[Type[Filter]]
     __scorer: Type[Scorer]
+    __sentence_scorer: Type[SentenceScorer]
     __passing_score: Number
 
     def __init__(
@@ -31,6 +29,7 @@ class Ilo:
         scoring_filters: List[Type[Filter]],
         scorer: Type[Scorer],
         passing_score: Number,
+        sentence_scorer: Type[SentenceScorer] = SentNoOp,
         word_tokenizer: Type[Tokenizer] = WordTokenizer,
         sent_tokenizer: Type[Tokenizer] = SentTokenizer,
     ):
@@ -43,6 +42,7 @@ class Ilo:
         self.__ignoring_filters = [*ignoring_filters]
         self.__scoring_filters = [*scoring_filters]
         self.__scorer = scorer
+        self.__sentence_scorer = sentence_scorer
         self.__passing_score = passing_score
 
     def preprocess(self, msg: str) -> str:
@@ -55,6 +55,7 @@ class Ilo:
         return self.__word_tokenizer.tokenize(msg)
 
     def sent_tokenize(self, msg: str) -> List[str]:
+        """It is *highly* recommended that you run `ilo.preprocess` first."""
         return self.__sent_tokenizer.tokenize(msg)
 
     def clean_token(self, token: str) -> str:
@@ -93,44 +94,50 @@ class Ilo:
     def score_tokens(self, tokens: List[str]) -> float:
         return self.__scorer.score(tokens, self.__scoring_filters)
 
+    def score_sentences(self, scorecards: List[Scorecard]) -> List[Scorecard]:
+        return self.__sentence_scorer.score(scorecards)
+
     def _is_toki_pona(self, message: str) -> Scorecard:
         """Process a message into its tokens, then filters, cleans, and scores
-        them. Returns all parts. Message must already be preprocessed, normally
-        done in `self.is_toki_pona(message)`.
-
-        Returns all components of the processing algorithm except preprocessing:
-        - Tokenized message (list[str])
-        - Filtered message (list[str])
-        - Cleaned message (list[str])
-        - Score (float)
-        - Result (bool)
+        them. Message must already be preprocessed, normally done in
+        `self.is_toki_pona(message)`.
+
+        Returns a `Scorecard` with all changes to the input text and a score.
         """
         tokenized = self.word_tokenize(message)
         filtered = self.filter_tokens(tokenized)
         cleaned = self.clean_tokens(filtered)
         score = self.score_tokens(cleaned)
-        result = score >= self.__passing_score
 
-        return tokenized, filtered, cleaned, score, result
+        scorecard: Scorecard = {
+            "text": message,
+            "tokenized": tokenized,
+            "filtered": filtered,
+            "cleaned": cleaned,
+            "score": score,
+        }
+
+        return scorecard
 
     def is_toki_pona(self, message: str) -> bool:
-        """Determines whether a single statement is or is not Toki Pona."""
+        """Determines whether a text is or is not Toki Pona."""
         message = self.preprocess(message)
-        *_, result = self._is_toki_pona(message)
-        return result
+        scorecard = self._is_toki_pona(message)
+        return scorecard["score"] >= self.__passing_score
 
     def _are_toki_pona(self, message: str) -> List[Scorecard]:
-        """Split a message into sentences, then return a list each sentence's
-        results via `self._is_toki_pona()`.
+        """Split a message into sentences, then return a list with each
+        sentence's scorecard from `self._is_toki_pona()`.
 
         Message must already be preprocessed, normally done in
         `self.are_toki_pona(message)`.
         """
-        results: List[Scorecard] = list()
+        scorecards: List[Scorecard] = list()
         for sentence in self.sent_tokenize(message):
             result = self._is_toki_pona(sentence)
-            results.append(result)
-        return results
+            scorecards.append(result)
+        scorecards = self.score_sentences(scorecards)
+        return scorecards
 
     def are_toki_pona(self, message: str) -> List[bool]:
         """Splits a statement into sentences, then determines if each is or is not Toki Pona.
@@ -148,5 +155,5 @@ class Ilo:
         ```
         """
         message = self.preprocess(message)
-        results = self._are_toki_pona(message)
-        return [res[-1] for res in results]
+        scorecards = self._are_toki_pona(message)
+        return [card["score"] >= self.__passing_score for card in scorecards]

sonatoki/types.py

@@ -0,0 +1,60 @@
+# STL
+from typing import Dict, List, Union, Literal, TypedDict
+
+Number = Union[int, float]
+
+
+# TODO: scorecard kinda sucks as a name
+class Scorecard(TypedDict):
+    text: str
+    tokenized: List[str]
+    filtered: List[str]
+    cleaned: List[str]
+    score: Number
+
+
+LinkuUsageDate = Union[
+    Literal["2020-04"],
+    Literal["2021-10"],
+    Literal["2022-08"],
+    Literal["2023-09"],
+    # Literal["2024-09"],
+]
+
+LinkuUsageCategory = Union[
+    Literal["core"],
+    Literal["common"],
+    Literal["uncommon"],
+    Literal["obscure"],
+    Literal["sandbox"],
+]
+
+LinkuBooks = Union[
+    Literal["pu"],
+    Literal["ku suli"],
+    Literal["ku lili"],
+    Literal["none"],
+]
+
+
+class LinkuWord(TypedDict):
+    id: str
+    author_verbatim: str
+    author_verbatim_source: str
+    book: str
+    coined_era: str
+    coined_year: str
+    creator: List[str]
+    ku_data: Dict[str, int]
+    see_also: List[str]
+    resources: Dict[str, str]
+    representations: Dict[str, Union[str, List[str]]]
+    source_language: str
+    usage_category: LinkuUsageCategory
+    word: str
+    deprecated: bool
+    etymology: List[Dict[str, str]]
+    audio: List[Dict[str, str]]
+    pu_verbatim: Dict[str, str]
+    usage: Dict[LinkuUsageDate, int]
+    translations: Dict[str, Dict[str, str]]

{sonatoki-0.6.2.dist-info → sonatoki-0.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sonatoki
-Version: 0.6.2
+Version: 0.7.0
 Summary: ilo li moku e toki li pana e sona ni: ni li toki ala toki pona?
 Author-Email: "jan Kekan San (@gregdan3)" <gregory.danielson3@gmail.com>
 License: AGPL-3.0-or-later

{sonatoki-0.6.2.dist-info → sonatoki-0.7.0.dist-info}/RECORD

@@ -1,20 +1,21 @@
-sonatoki-0.6.2.dist-info/METADATA,sha256=DfNErx2CBzvtmLA-ANWO6LeGNyR3bywqb_ITgOhc4ew,6517
-sonatoki-0.6.2.dist-info/WHEEL,sha256=rSwsxJWe3vzyR5HCwjWXQruDgschpei4h_giTm0dJVE,90
-sonatoki-0.6.2.dist-info/licenses/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
+sonatoki-0.7.0.dist-info/METADATA,sha256=s6w7_WaARQijvFIFIWtg8hL2WzAkj19N7-DsKgfhi3s,6517
+sonatoki-0.7.0.dist-info/WHEEL,sha256=rSwsxJWe3vzyR5HCwjWXQruDgschpei4h_giTm0dJVE,90
+sonatoki-0.7.0.dist-info/licenses/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
 sonatoki/Cleaners.py,sha256=x2dT3MpDUfbrHA0EP2D3n1sTiKFFi5jw9ha-1dX973o,1958
-sonatoki/Configs.py,sha256=RD6YUYW45pwIFx8ebJgGs5PhIhL9sjn_VqIg4zf3VUE,5697
-sonatoki/Filters.py,sha256=nVSmw5M4sEYA_8KI1fI53rMHkd9KO6yWbKfdxxExxN8,11700
-sonatoki/Preprocessors.py,sha256=nN6xL6mvVAnWZjSNW8CaeLm8x4kK3dCoB-1WYqi0ANU,5763
-sonatoki/Scorers.py,sha256=LRQLgXKTU2VqhkMHFPVxyVt83DXf85_zrpDGk4ThU24,3811
+sonatoki/Configs.py,sha256=rIvrkYjeJeCuWwJIjvmJX6keRZcUJ0pt7h7KdYT5IFI,4766
+sonatoki/Filters.py,sha256=cJ5skX9yeqd4HvjzPxIAswigRWvO0ZV2nepQksFedtk,12575
+sonatoki/Preprocessors.py,sha256=nY0_cmF4aEmGZxXc7ZEvhvf2BZO6GnrMUC8IqDwu47A,6034
+sonatoki/Scorers.py,sha256=aCU3p9rD4QOy-uu851FGGw-ARqUCG_l4V_z5rtRL420,5236
 sonatoki/Tokenizers.py,sha256=8lpC70bzXOpHyhVr5bmqpYKmdmQvJdf7X5-Icc9RRCw,5040
 sonatoki/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sonatoki/__main__.py,sha256=6n4kUF80APl6a0jV46h_ncHNuQbrLpZ_nAmiNAakiag,5673
 sonatoki/alphabetic.txt,sha256=duyqAKilD2vLIr75RShCIAnktNJcGeEoQIk18V6czmg,11702
-sonatoki/constants.py,sha256=mPbU-X9PNzelOHVZn-8ZqR_ewKYNjDA6lj2XQpnuoRw,19212
-sonatoki/ilo.py,sha256=PWZa202Q4h7IjnLxmfgT93iAPJL7dqJbA97L9kQDPiA,5658
+sonatoki/constants.py,sha256=BxE_MME2XZUZLg9ZezPirUO2sxw4JkujsrKoENeYORc,19313
+sonatoki/ilo.py,sha256=Dsn0yagkwjqpAQoCj6mkZ6NqWeanRF2lxNDNoqjWGLo,5993
 sonatoki/linku.json,sha256=d72Dvht-a4gBmdqLLI8mElvo83zSpbxDmxJj05hOudM,295413
 sonatoki/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sonatoki/sandbox.json,sha256=44csrQDaVtV-n8OyewabX1J9MmUFCsPct5C8E5Xuc58,140197
 sonatoki/syllabic.txt,sha256=HnqY4TrZ3tPcHah3TsvG9F9gjMrnAGdJ8hHJNHyyUPc,1712
+sonatoki/types.py,sha256=zoVJeaDLOPstREiHtoD9pv-AOCsJq2C4_GG3nTYd114,1267
 sonatoki/utils.py,sha256=sT5xLMEj0aLpy8GP92HKblJU1Wt1m8NUlMgCFWB32xQ,2265
-sonatoki-0.6.2.dist-info/RECORD,,
+sonatoki-0.7.0.dist-info/RECORD,,