sonatoki 0.3.3__tar.gz → 0.4.0__tar.gz

{sonatoki-0.3.3 → sonatoki-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sonatoki
-Version: 0.3.3
+Version: 0.4.0
 Summary: ilo li moku e toki li pana e sona ni: ni li toki ala toki pona?
 Author-Email: "jan Kekan San (@gregdan3)" <gregory.danielson3@gmail.com>
 License: AGPL-3.0-or-later

{sonatoki-0.3.3 → sonatoki-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sonatoki"
-version = "0.3.3"
+version = "0.4.0"
 description = "ilo li moku e toki li pana e sona ni: ni li toki ala toki pona?"
 authors = [
     { name = "jan Kekan San (@gregdan3)", email = "gregory.danielson3@gmail.com" },

{sonatoki-0.3.3 → sonatoki-0.4.0}/src/sonatoki/Configs.py

@@ -2,6 +2,9 @@
 from copy import deepcopy
 from typing import List, Type, TypedDict
 
+# PDM
+from typing_extensions import NotRequired
+
 # LOCAL
 from sonatoki.Filters import (
     Filter,
@@ -26,7 +29,7 @@ from sonatoki.Filters import (
 )
 from sonatoki.Scorers import Number, Scorer, PassFail, SoftScaling, SoftPassFail
 from sonatoki.Cleaners import Cleaner, ConsecutiveDuplicates
-from sonatoki.Tokenizers import Tokenizer, WordTokenizer
+from sonatoki.Tokenizers import Tokenizer
 from sonatoki.Preprocessors import (
     URLs,
     Backticks,
@@ -38,12 +41,13 @@ from sonatoki.Preprocessors import (
 
 class IloConfig(TypedDict):
     preprocessors: List[Type[Preprocessor]]
-    word_tokenizer: Type[Tokenizer]
     cleaners: List[Type[Cleaner]]
     ignoring_filters: List[Type[Filter]]
     scoring_filters: List[Type[Filter]]
     scorer: Type[Scorer]
     passing_score: Number
+    word_tokenizer: NotRequired[Type[Tokenizer]]
+    sent_tokenizer: NotRequired[Type[Tokenizer]]
 
 
 # TODO: branching configs? config builder?
@@ -55,7 +59,6 @@ BaseConfig: IloConfig = {
     "scoring_filters": [],
     "scorer": PassFail,
     "passing_score": 0.8,
-    "word_tokenizer": WordTokenizer,
 }
 
 
@@ -71,7 +74,6 @@ PrefConfig: IloConfig = {
     ],
     "scorer": SoftScaling,
     "passing_score": 0.8,
-    "word_tokenizer": WordTokenizer,
 }
 
 CorpusConfig: IloConfig = {
@@ -94,7 +96,6 @@ CorpusConfig: IloConfig = {
     ],
     "scorer": SoftScaling,
     "passing_score": 0.8,
-    "word_tokenizer": WordTokenizer,
 }
 """Mimics the previous implementation of ilo pi toki pona taso."""
 LazyConfig: IloConfig = {
@@ -104,7 +105,6 @@ LazyConfig: IloConfig = {
     "scoring_filters": [Alphabetic, NimiUCSUR, ProperName, Miscellaneous],
     "scorer": SoftPassFail,
     "passing_score": 0.8,
-    "word_tokenizer": WordTokenizer,
 }
 """This is extremely silly."""
 IsipinEpikuConfig: IloConfig = {
@@ -125,7 +125,6 @@ IsipinEpikuConfig: IloConfig = {
     ],
     "scorer": SoftScaling,
     "passing_score": 0.8,
-    "word_tokenizer": WordTokenizer,
 }
 
 
@@ -141,7 +140,6 @@ DiscordConfig: IloConfig = {
     ],
     "scorer": SoftScaling,
     "passing_score": 0.8,
-    "word_tokenizer": WordTokenizer,
 }
 
 TelegramConfig: IloConfig = deepcopy(PrefConfig)

{sonatoki-0.3.3 → sonatoki-0.4.0}/src/sonatoki/ilo.py

@@ -5,12 +5,17 @@ from typing import List, Type, Tuple
 from sonatoki.Filters import Filter
 from sonatoki.Scorers import Number, Scorer
 from sonatoki.Cleaners import Cleaner
-from sonatoki.Tokenizers import Tokenizer
+from sonatoki.Tokenizers import Tokenizer, SentTokenizer, WordTokenizer
 from sonatoki.Preprocessors import Preprocessor
 
+# tokenized, filtered, cleaned, score, result
+Scorecard = Tuple[List[str], List[str], List[str], Number, bool]
+# TODO: scorecard kinda sucks as a name
+
 
 class Ilo:
     __preprocessors: List[Type[Preprocessor]]
+    __sent_tokenizer: Type[Tokenizer]
     __word_tokenizer: Type[Tokenizer]
     __cleaners: List[Type[Cleaner]]
     __ignoring_filters: List[Type[Filter]]
@@ -26,11 +31,13 @@ class Ilo:
         scoring_filters: List[Type[Filter]],
         scorer: Type[Scorer],
         passing_score: Number,
-        word_tokenizer: Type[Tokenizer],
+        word_tokenizer: Type[Tokenizer] = WordTokenizer,
+        sent_tokenizer: Type[Tokenizer] = SentTokenizer,
     ):
         super().__init__()
         # avoid keeping a ref to user's list just in case
         self.__preprocessors = [*preprocessors]
+        self.__sent_tokenizer = sent_tokenizer
         self.__word_tokenizer = word_tokenizer
         self.__cleaners = [*cleaners]
         self.__ignoring_filters = [*ignoring_filters]
@@ -47,6 +54,9 @@ class Ilo:
         """It is *highly* recommended that you run `ilo.preprocess` first."""
         return self.__word_tokenizer.tokenize(msg)
 
+    def sent_tokenize(self, msg: str) -> List[str]:
+        return self.__sent_tokenizer.tokenize(msg)
+
     def clean_token(self, token: str) -> str:
         for c in self.__cleaners:
             token = c.clean(token)
@@ -83,26 +93,60 @@ class Ilo:
     def score_tokens(self, tokens: List[str]) -> float:
         return self.__scorer.score(tokens, self.__scoring_filters)
 
-    def _is_toki_pona(
-        self, message: str
-    ) -> Tuple[str, List[str], List[str], List[str], Number, bool]:
-        """Returns all components of the processing algorithm:
-        - Preprocessed message (str)
+    def _is_toki_pona(self, message: str) -> Scorecard:
+        """Process a message into its tokens, then filters, cleans, and scores
+        them. Returns all parts. Message must already be preprocessed, normally
+        done in `self.is_toki_pona(message)`.
+
+        Returns all components of the processing algorithm except preprocessing:
         - Tokenized message (list[str])
         - Filtered message (list[str])
         - Cleaned message (list[str])
         - Score (float)
-        - Result (bool)"""
-        preprocessed = self.preprocess(message)
-        tokenized = self.word_tokenize(preprocessed)
+        - Result (bool)
+        """
+        tokenized = self.word_tokenize(message)
         filtered = self.filter_tokens(tokenized)
         cleaned = self.clean_tokens(filtered)
         score = self.score_tokens(cleaned)
         result = score >= self.__passing_score
 
-        return preprocessed, tokenized, filtered, cleaned, score, result
+        return tokenized, filtered, cleaned, score, result
 
     def is_toki_pona(self, message: str) -> bool:
         """Determines whether a single statement is or is not Toki Pona."""
+        message = self.preprocess(message)
         *_, result = self._is_toki_pona(message)
         return result
+
+    def _are_toki_pona(self, message: str):
+        """Split a message into sentences, then return a list each sentence's
+        results via `self._is_toki_pona()`.
+
+        Message must already be preprocessed, normally done in
+        `self.are_toki_pona(message)`.
+        """
+        results: List[Scorecard] = list()
+        for sentence in self.sent_tokenize(message):
+            result = self._is_toki_pona(sentence)
+            results.append(result)
+        return results
+
+    def are_toki_pona(self, message: str) -> List[bool]:
+        """Splits a statement into sentences, then determines if each is or is not Toki Pona.
+        NOTE: You will need to decide how to score the result. Examples:
+
+        ```
+        def all_must_pass(message: str) -> bool:
+            return all(ILO.are_toki_pona(message))
+
+        def portion_must_pass(message: str, score: Number = 0.8) -> bool:
+            results = ILO.are_toki_pona(message)
+            sent_count = len(results)
+            passing = results.count(True)
+            return (passing / sent_count) >= score
+        ```
+        """
+        message = self.preprocess(message)
+        results = self._are_toki_pona(message)
+        return [res[-1] for res in results]