sonatoki 0.3.3__py3-none-any.whl → 0.4.0__py3-none-any.whl

sonatoki/Configs.py

@@ -2,6 +2,9 @@
 from copy import deepcopy
 from typing import List, Type, TypedDict
 
+# PDM
+from typing_extensions import NotRequired
+
 # LOCAL
 from sonatoki.Filters import (
     Filter,
@@ -26,7 +29,7 @@ from sonatoki.Filters import (
 )
 from sonatoki.Scorers import Number, Scorer, PassFail, SoftScaling, SoftPassFail
 from sonatoki.Cleaners import Cleaner, ConsecutiveDuplicates
-from sonatoki.Tokenizers import Tokenizer, WordTokenizer
+from sonatoki.Tokenizers import Tokenizer
 from sonatoki.Preprocessors import (
     URLs,
     Backticks,
@@ -38,12 +41,13 @@ from sonatoki.Preprocessors import (
 
 class IloConfig(TypedDict):
     preprocessors: List[Type[Preprocessor]]
-    word_tokenizer: Type[Tokenizer]
     cleaners: List[Type[Cleaner]]
     ignoring_filters: List[Type[Filter]]
     scoring_filters: List[Type[Filter]]
     scorer: Type[Scorer]
     passing_score: Number
+    word_tokenizer: NotRequired[Type[Tokenizer]]
+    sent_tokenizer: NotRequired[Type[Tokenizer]]
 
 
 # TODO: branching configs? config builder?
@@ -55,7 +59,6 @@ BaseConfig: IloConfig = {
     "scoring_filters": [],
     "scorer": PassFail,
     "passing_score": 0.8,
-    "word_tokenizer": WordTokenizer,
 }
 
 
@@ -71,7 +74,6 @@ PrefConfig: IloConfig = {
     ],
     "scorer": SoftScaling,
     "passing_score": 0.8,
-    "word_tokenizer": WordTokenizer,
 }
 
 CorpusConfig: IloConfig = {
@@ -94,7 +96,6 @@ CorpusConfig: IloConfig = {
     ],
     "scorer": SoftScaling,
     "passing_score": 0.8,
-    "word_tokenizer": WordTokenizer,
 }
 """Mimics the previous implementation of ilo pi toki pona taso."""
 LazyConfig: IloConfig = {
@@ -104,7 +105,6 @@ LazyConfig: IloConfig = {
     "scoring_filters": [Alphabetic, NimiUCSUR, ProperName, Miscellaneous],
     "scorer": SoftPassFail,
     "passing_score": 0.8,
-    "word_tokenizer": WordTokenizer,
 }
 """This is extremely silly."""
 IsipinEpikuConfig: IloConfig = {
@@ -125,7 +125,6 @@ IsipinEpikuConfig: IloConfig = {
     ],
     "scorer": SoftScaling,
     "passing_score": 0.8,
-    "word_tokenizer": WordTokenizer,
 }
 
 
@@ -141,7 +140,6 @@ DiscordConfig: IloConfig = {
     ],
     "scorer": SoftScaling,
     "passing_score": 0.8,
-    "word_tokenizer": WordTokenizer,
 }
 
 TelegramConfig: IloConfig = deepcopy(PrefConfig)

sonatoki/ilo.py

@@ -5,12 +5,17 @@ from typing import List, Type, Tuple
 from sonatoki.Filters import Filter
 from sonatoki.Scorers import Number, Scorer
 from sonatoki.Cleaners import Cleaner
-from sonatoki.Tokenizers import Tokenizer
+from sonatoki.Tokenizers import Tokenizer, SentTokenizer, WordTokenizer
 from sonatoki.Preprocessors import Preprocessor
 
+# tokenized, filtered, cleaned, score, result
+Scorecard = Tuple[List[str], List[str], List[str], Number, bool]
+# TODO: scorecard kinda sucks as a name
+
 
 class Ilo:
     __preprocessors: List[Type[Preprocessor]]
+    __sent_tokenizer: Type[Tokenizer]
     __word_tokenizer: Type[Tokenizer]
     __cleaners: List[Type[Cleaner]]
     __ignoring_filters: List[Type[Filter]]
@@ -26,11 +31,13 @@ class Ilo:
         scoring_filters: List[Type[Filter]],
         scorer: Type[Scorer],
         passing_score: Number,
-        word_tokenizer: Type[Tokenizer],
+        word_tokenizer: Type[Tokenizer] = WordTokenizer,
+        sent_tokenizer: Type[Tokenizer] = SentTokenizer,
     ):
         super().__init__()
         # avoid keeping a ref to user's list just in case
         self.__preprocessors = [*preprocessors]
+        self.__sent_tokenizer = sent_tokenizer
         self.__word_tokenizer = word_tokenizer
         self.__cleaners = [*cleaners]
         self.__ignoring_filters = [*ignoring_filters]
@@ -47,6 +54,9 @@ class Ilo:
         """It is *highly* recommended that you run `ilo.preprocess` first."""
         return self.__word_tokenizer.tokenize(msg)
 
+    def sent_tokenize(self, msg: str) -> List[str]:
+        return self.__sent_tokenizer.tokenize(msg)
+
     def clean_token(self, token: str) -> str:
         for c in self.__cleaners:
             token = c.clean(token)
@@ -83,26 +93,60 @@ class Ilo:
     def score_tokens(self, tokens: List[str]) -> float:
         return self.__scorer.score(tokens, self.__scoring_filters)
 
-    def _is_toki_pona(
-        self, message: str
-    ) -> Tuple[str, List[str], List[str], List[str], Number, bool]:
-        """Returns all components of the processing algorithm:
-        - Preprocessed message (str)
+    def _is_toki_pona(self, message: str) -> Scorecard:
+        """Process a message into its tokens, then filters, cleans, and scores
+        them. Returns all parts. Message must already be preprocessed, normally
+        done in `self.is_toki_pona(message)`.
+
+        Returns all components of the processing algorithm except preprocessing:
         - Tokenized message (list[str])
         - Filtered message (list[str])
         - Cleaned message (list[str])
         - Score (float)
-        - Result (bool)"""
-        preprocessed = self.preprocess(message)
-        tokenized = self.word_tokenize(preprocessed)
+        - Result (bool)
+        """
+        tokenized = self.word_tokenize(message)
         filtered = self.filter_tokens(tokenized)
         cleaned = self.clean_tokens(filtered)
         score = self.score_tokens(cleaned)
         result = score >= self.__passing_score
 
-        return preprocessed, tokenized, filtered, cleaned, score, result
+        return tokenized, filtered, cleaned, score, result
 
     def is_toki_pona(self, message: str) -> bool:
         """Determines whether a single statement is or is not Toki Pona."""
+        message = self.preprocess(message)
         *_, result = self._is_toki_pona(message)
         return result
+
+    def _are_toki_pona(self, message: str):
+        """Split a message into sentences, then return a list each sentence's
+        results via `self._is_toki_pona()`.
+
+        Message must already be preprocessed, normally done in
+        `self.are_toki_pona(message)`.
+        """
+        results: List[Scorecard] = list()
+        for sentence in self.sent_tokenize(message):
+            result = self._is_toki_pona(sentence)
+            results.append(result)
+        return results
+
+    def are_toki_pona(self, message: str) -> List[bool]:
+        """Splits a statement into sentences, then determines if each is or is not Toki Pona.
+        NOTE: You will need to decide how to score the result. Examples:
+
+        ```
+        def all_must_pass(message: str) -> bool:
+            return all(ILO.are_toki_pona(message))
+
+        def portion_must_pass(message: str, score: Number = 0.8) -> bool:
+            results = ILO.are_toki_pona(message)
+            sent_count = len(results)
+            passing = results.count(True)
+            return (passing / sent_count) >= score
+        ```
+        """
+        message = self.preprocess(message)
+        results = self._are_toki_pona(message)
+        return [res[-1] for res in results]

{sonatoki-0.3.3.dist-info → sonatoki-0.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sonatoki
-Version: 0.3.3
+Version: 0.4.0
 Summary: ilo li moku e toki li pana e sona ni: ni li toki ala toki pona?
 Author-Email: "jan Kekan San (@gregdan3)" <gregory.danielson3@gmail.com>
 License: AGPL-3.0-or-later

{sonatoki-0.3.3.dist-info → sonatoki-0.4.0.dist-info}/RECORD

@@ -1,8 +1,8 @@
-sonatoki-0.3.3.dist-info/METADATA,sha256=b78h2-lsc4aBzkEWrzXTslfvJb-ZVbFjCpgPOF_kYrg,6341
-sonatoki-0.3.3.dist-info/WHEEL,sha256=SOP-4bEE0jbVaCHQGVvF08uWxk5rcSsfEybvoQVHlD8,90
-sonatoki-0.3.3.dist-info/licenses/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
+sonatoki-0.4.0.dist-info/METADATA,sha256=Z89tIHyGG9RRAgcr_3E4XW2IMX9NyT9mawcCeMQfXPU,6341
+sonatoki-0.4.0.dist-info/WHEEL,sha256=SOP-4bEE0jbVaCHQGVvF08uWxk5rcSsfEybvoQVHlD8,90
+sonatoki-0.4.0.dist-info/licenses/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
 sonatoki/Cleaners.py,sha256=x2dT3MpDUfbrHA0EP2D3n1sTiKFFi5jw9ha-1dX973o,1958
-sonatoki/Configs.py,sha256=ZNKJGeAxgolAxqNSeS7iADvQEcN4X3tY0iaoiAaL95U,4160
+sonatoki/Configs.py,sha256=tOeJSlYXMBHbRPBxERGWGT5AjvCxNb3ZGu8GA4BYve4,4034
 sonatoki/Filters.py,sha256=mpJBl-YPMF-Yl6mKFXf0D6DwkPR6H424RlvrkSeh4Dc,10714
 sonatoki/Preprocessors.py,sha256=nvAzxpWP9WwT6gOCKcuiz5F8xYDdKIt9bOVUvy9o-G0,4459
 sonatoki/Scorers.py,sha256=LRQLgXKTU2VqhkMHFPVxyVt83DXf85_zrpDGk4ThU24,3811
@@ -10,9 +10,9 @@ sonatoki/Tokenizers.py,sha256=So5_Tu6J98MD3yVcwB_X3lw2uMG0TN6XHcTbQjFCu5Q,4254
 sonatoki/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sonatoki/__main__.py,sha256=6xc-wIrrFo9wTyn4zRQNAmqwmJBtVvCMwV-CrM-hueA,82
 sonatoki/constants.py,sha256=wH3iR32-Ic7vSkrMjAZIvmIysTtkJ-KBVU5zv3Oamqs,12656
-sonatoki/ilo.py,sha256=yyLgNPI0Hmb4f1BzX6IRHr11FPChfL2xDR_9odlr8_8,3849
+sonatoki/ilo.py,sha256=7KwTZgczzU2gbhC69yZbxtpTHy_fGtg_MnG_bDpiSxM,5639
 sonatoki/linku.json,sha256=fm4-dks5s9x1bs7q82GNngAedVCWilMPCQ_o-j35QL0,270950
 sonatoki/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sonatoki/sandbox.json,sha256=zPtZgJ_CpJa-2Den0gTNlk52f-YEwFVcjMarQXeeu5U,77563
 sonatoki/utils.py,sha256=L984aXxvzfJaZ6GSWRKs7LweOGZYTLK11CdAhpLQr0g,4067
-sonatoki-0.3.3.dist-info/RECORD,,
+sonatoki-0.4.0.dist-info/RECORD,,