sonatoki 0.3.3__py3-none-any.whl → 0.5.0__py3-none-any.whl

sonatoki/constants.py

@@ -1,6 +1,6 @@
 # STL
 import json
-from typing import Set, Dict, List
+from typing import Set, Dict
 from pathlib import Path
 
 # LOCAL
@@ -15,9 +15,9 @@ UNICODE_PUNCT_RANGES = [
     "\\U0000003a-\\U00000040",
     "\\U0000005b-\\U00000060",
     "\\U0000007b-\\U0000007e",
-    "\\U000000a1-\\U000000a9",
+    "\\U000000a1-\\U000000a8",
     "\\U000000ab-\\U000000ac",
-    "\\U000000ae-\\U000000b1",
+    "\\U000000af-\\U000000b1",
     "\\U000000b4",
     "\\U000000b6-\\U000000b8",
     "\\U000000bb",
@@ -118,7 +118,9 @@ UNICODE_PUNCT_RANGES = [
     "\\U00001fed-\\U00001fef",
     "\\U00001ffd-\\U00001ffe",
     "\\U00002010-\\U00002027",
-    "\\U00002030-\\U0000205e",
+    "\\U00002030-\\U0000203b",
+    "\\U0000203d-\\U00002048",
+    "\\U0000204a-\\U0000205e",
     "\\U0000207a-\\U0000207e",
     "\\U0000208a-\\U0000208e",
     "\\U000020a0-\\U000020c0",
@@ -127,7 +129,8 @@ UNICODE_PUNCT_RANGES = [
     "\\U00002108-\\U00002109",
     "\\U00002114",
     "\\U00002116-\\U00002118",
-    "\\U0000211e-\\U00002123",
+    "\\U0000211e-\\U00002121",
+    "\\U00002123",
     "\\U00002125",
     "\\U00002127",
     "\\U00002129",
@@ -137,11 +140,88 @@ UNICODE_PUNCT_RANGES = [
     "\\U0000214a-\\U0000214d",
     "\\U0000214f",
     "\\U0000218a-\\U0000218b",
-    "\\U00002190-\\U00002426",
+    "\\U00002190-\\U00002193",
+    "\\U0000219a-\\U000021a8",
+    "\\U000021ab-\\U00002319",
+    "\\U0000231c-\\U00002327",
+    "\\U00002329-\\U000023ce",
+    "\\U000023d0-\\U000023e8",
+    "\\U000023f4-\\U000023f7",
+    "\\U000023fb-\\U00002426",
     "\\U00002440-\\U0000244a",
-    "\\U0000249c-\\U000024b5",
-    "\\U00002500-\\U00002775",
-    "\\U00002794-\\U00002b73",
+    "\\U0000249c-\\U000024c1",
+    "\\U000024c3-\\U000024e9",
+    "\\U00002500-\\U000025a9",
+    "\\U000025ac-\\U000025b5",
+    "\\U000025b7-\\U000025bf",
+    "\\U000025c1-\\U000025fa",
+    "\\U000025ff",
+    "\\U00002605-\\U0000260d",
+    "\\U0000260f-\\U00002610",
+    "\\U00002612-\\U00002613",
+    "\\U00002616-\\U00002617",
+    "\\U00002619-\\U0000261c",
+    "\\U0000261e-\\U0000261f",
+    "\\U00002621",
+    "\\U00002624-\\U00002625",
+    "\\U00002627-\\U00002629",
+    "\\U0000262b-\\U0000262d",
+    "\\U00002630-\\U00002637",
+    "\\U0000263b-\\U0000263f",
+    "\\U00002641",
+    "\\U00002643-\\U00002647",
+    "\\U00002654-\\U0000265e",
+    "\\U00002661-\\U00002662",
+    "\\U00002664",
+    "\\U00002667",
+    "\\U00002669-\\U0000267a",
+    "\\U0000267c-\\U0000267d",
+    "\\U00002680-\\U00002691",
+    "\\U00002698",
+    "\\U0000269a",
+    "\\U0000269d-\\U0000269f",
+    "\\U000026a2-\\U000026a6",
+    "\\U000026a8-\\U000026a9",
+    "\\U000026ac-\\U000026af",
+    "\\U000026b2-\\U000026bc",
+    "\\U000026bf-\\U000026c3",
+    "\\U000026c6-\\U000026c7",
+    "\\U000026c9-\\U000026cd",
+    "\\U000026d0",
+    "\\U000026d2",
+    "\\U000026d5-\\U000026e8",
+    "\\U000026eb-\\U000026ef",
+    "\\U000026f6",
+    "\\U000026fb-\\U000026fc",
+    "\\U000026fe-\\U00002701",
+    "\\U00002703-\\U00002704",
+    "\\U00002706-\\U00002707",
+    "\\U0000270e",
+    "\\U00002710-\\U00002711",
+    "\\U00002713",
+    "\\U00002715",
+    "\\U00002717-\\U0000271c",
+    "\\U0000271e-\\U00002720",
+    "\\U00002722-\\U00002727",
+    "\\U00002729-\\U00002732",
+    "\\U00002735-\\U00002743",
+    "\\U00002745-\\U00002746",
+    "\\U00002748-\\U0000274b",
+    "\\U0000274d",
+    "\\U0000274f-\\U00002752",
+    "\\U00002756",
+    "\\U00002758-\\U00002762",
+    "\\U00002765-\\U00002775",
+    "\\U00002794",
+    "\\U00002798-\\U000027a0",
+    "\\U000027a2-\\U000027af",
+    "\\U000027b1-\\U000027be",
+    "\\U000027c0-\\U00002933",
+    "\\U00002936-\\U00002b04",
+    "\\U00002b08-\\U00002b1a",
+    "\\U00002b1d-\\U00002b4f",
+    "\\U00002b51-\\U00002b54",
+    "\\U00002b56-\\U00002b73",
     "\\U00002b76-\\U00002b95",
     "\\U00002b97-\\U00002bff",
     "\\U00002ce5-\\U00002cea",
@@ -156,9 +236,8 @@ UNICODE_PUNCT_RANGES = [
     "\\U00002ff0-\\U00002fff",
     "\\U00003001-\\U00003004",
     "\\U00003008-\\U00003020",
-    "\\U00003030",
     "\\U00003036-\\U00003037",
-    "\\U0000303d-\\U0000303f",
+    "\\U0000303e-\\U0000303f",
     "\\U0000309b-\\U0000309c",
     "\\U000030a0",
     "\\U000030fb",
@@ -170,7 +249,9 @@ UNICODE_PUNCT_RANGES = [
     "\\U0000322a-\\U00003247",
     "\\U00003250",
     "\\U00003260-\\U0000327f",
-    "\\U0000328a-\\U000032b0",
+    "\\U0000328a-\\U00003296",
+    "\\U00003298",
+    "\\U0000329a-\\U000032b0",
     "\\U000032c0-\\U000033ff",
     "\\U00004dc0-\\U00004dff",
     "\\U0000a490-\\U0000a4c6",
@@ -314,49 +395,97 @@ UNICODE_PUNCT_RANGES = [
     "\\U0001ecb0",
     "\\U0001ed2e",
     "\\U0001eef0-\\U0001eef1",
-    "\\U0001f000-\\U0001f02b",
+    "\\U0001f000-\\U0001f003",
+    "\\U0001f005-\\U0001f02b",
     "\\U0001f030-\\U0001f093",
     "\\U0001f0a0-\\U0001f0ae",
     "\\U0001f0b1-\\U0001f0bf",
-    "\\U0001f0c1-\\U0001f0cf",
+    "\\U0001f0c1-\\U0001f0ce",
     "\\U0001f0d1-\\U0001f0f5",
-    "\\U0001f10d-\\U0001f12f",
-    "\\U0001f14a-\\U0001f14f",
-    "\\U0001f16a-\\U0001f16f",
-    "\\U0001f18a-\\U0001f1ad",
-    "\\U0001f1e6-\\U0001f202",
-    "\\U0001f210-\\U0001f23b",
+    "\\U0001f10d-\\U0001f16f",
+    "\\U0001f172-\\U0001f17d",
+    "\\U0001f180-\\U0001f18d",
+    "\\U0001f18f-\\U0001f190",
+    "\\U0001f19b-\\U0001f1ad",
+    "\\U0001f1e6-\\U0001f1e7",
+    "\\U0001f1ea-\\U0001f1eb",
+    "\\U0001f1ee-\\U0001f1f1",
+    "\\U0001f1f4-\\U0001f1f6",
+    "\\U0001f1f9-\\U0001f200",
+    "\\U0001f210-\\U0001f219",
+    "\\U0001f21b-\\U0001f22e",
+    "\\U0001f230-\\U0001f231",
+    "\\U0001f23b",
     "\\U0001f240-\\U0001f248",
-    "\\U0001f250-\\U0001f251",
     "\\U0001f260-\\U0001f265",
-    "\\U0001f300-\\U0001f6d7",
-    "\\U0001f6dc-\\U0001f6ec",
-    "\\U0001f6f0-\\U0001f6fc",
+    "\\U0001f322-\\U0001f323",
+    "\\U0001f394-\\U0001f395",
+    "\\U0001f398",
+    "\\U0001f39c-\\U0001f39d",
+    "\\U0001f3f1-\\U0001f3f2",
+    "\\U0001f3f6",
+    "\\U0001f4fe",
+    "\\U0001f53e-\\U0001f548",
+    "\\U0001f54f",
+    "\\U0001f568-\\U0001f56e",
+    "\\U0001f571-\\U0001f572",
+    "\\U0001f57b-\\U0001f586",
+    "\\U0001f588-\\U0001f589",
+    "\\U0001f58e-\\U0001f58f",
+    "\\U0001f591-\\U0001f594",
+    "\\U0001f597-\\U0001f5a3",
+    "\\U0001f5a6-\\U0001f5a7",
+    "\\U0001f5a9-\\U0001f5b0",
+    "\\U0001f5b3-\\U0001f5bb",
+    "\\U0001f5bd-\\U0001f5c1",
+    "\\U0001f5c5-\\U0001f5d0",
+    "\\U0001f5d4-\\U0001f5db",
+    "\\U0001f5df-\\U0001f5e0",
+    "\\U0001f5e2",
+    "\\U0001f5e4-\\U0001f5e7",
+    "\\U0001f5e9-\\U0001f5ee",
+    "\\U0001f5f0-\\U0001f5f2",
+    "\\U0001f5f4-\\U0001f5f9",
+    "\\U0001f650-\\U0001f67f",
+    "\\U0001f6c6-\\U0001f6ca",
+    "\\U0001f6d3-\\U0001f6d4",
+    "\\U0001f6e6-\\U0001f6e8",
+    "\\U0001f6ea",
+    "\\U0001f6f1-\\U0001f6f2",
     "\\U0001f700-\\U0001f776",
     "\\U0001f77b-\\U0001f7d9",
-    "\\U0001f7e0-\\U0001f7eb",
-    "\\U0001f7f0",
     "\\U0001f800-\\U0001f80b",
     "\\U0001f810-\\U0001f847",
     "\\U0001f850-\\U0001f859",
     "\\U0001f860-\\U0001f887",
     "\\U0001f890-\\U0001f8ad",
     "\\U0001f8b0-\\U0001f8b1",
-    "\\U0001f900-\\U0001fa53",
+    "\\U0001f900-\\U0001f90b",
+    "\\U0001f93b",
+    "\\U0001f946",
+    "\\U0001fa00-\\U0001fa53",
     "\\U0001fa60-\\U0001fa6d",
-    "\\U0001fa70-\\U0001fa7c",
-    "\\U0001fa80-\\U0001fa88",
-    "\\U0001fa90-\\U0001fabd",
-    "\\U0001fabf-\\U0001fac5",
-    "\\U0001face-\\U0001fadb",
-    "\\U0001fae0-\\U0001fae8",
-    "\\U0001faf0-\\U0001faf8",
     "\\U0001fb00-\\U0001fb92",
     "\\U0001fb94-\\U0001fbca",
-    "\\U000f1990-\\U000f199d",  # UCSUR punctuation
+    "\\U000f1990-\\U000f199d",
 ]
 
-UCSUR_PUNCT_RANGES = UNICODE_PUNCT_RANGES[-1]  # NOTE: THIS CAN CHANGE
+
+NOT_IN_PUNCT_CLASS = r"Ⓐ-ⓩ🄰-🅉🅐-🅩🅰-🆉"
+ALL_VARIATION_SELECTOR_RANGES = ["\\U0000fe00-\\U0000fe0f", "\\U000e0100-\\U000e01ef"]
+EMOJI_VARIATION_SELECTOR_RANGES = ["\\U0000fe0e-\\U0000fe0f"]
+EMOJI_VARIATION_SELECTOR_RANGES_STR = "".join(EMOJI_VARIATION_SELECTOR_RANGES)
+"""All variation selectors are in Nonspacing Mark (Mn), but it is more apt to
+mark these two as punctuation, since they are used exclusively for rendering
+emoji.
+
+But it's even better to use the Emoji filter.
+"""
+
+UCSUR_PUNCT_RANGES = ["\\U000f1990-\\U000f199d"]
+UCSUR_PUNCT_RANGES_STR = "".join(UCSUR_PUNCT_RANGES)
+"""Private Use Area glyphs are given the apt but unhelpful 'Private Use'
+class."""
 
 UNICODE_PUNCT = find_unicode_chars(UNICODE_PUNCT_RANGES)
 # this is a large string.
@@ -366,7 +495,7 @@ POSIX_PUNCT = r"""-!"#$%&'()*+,./:;<=>?@[\]^_`{|}~"""
 POSIX_PUNCT_RANGES = find_unicode_ranges(POSIX_PUNCT)
 
 ALL_PUNCT = "".join(sorted(list(set(POSIX_PUNCT + UNICODE_PUNCT))))
-ALL_PUNCT_RANGES = "".join(find_unicode_ranges(ALL_PUNCT))
+ALL_PUNCT_RANGES_STR = "".join(find_unicode_ranges(ALL_PUNCT))
 # combined bc the result could be simpler
 
 SENTENCE_PUNCT = """.?!:;'"()[-]“”·…"""
@@ -374,6 +503,8 @@ SENTENCE_PUNCT = """.?!:;'"()[-]“”·…"""
 
 LINKU = Path(__file__).resolve().parent / Path("linku.json")
 SANDBOX = Path(__file__).resolve().parent / Path("sandbox.json")
+SYLLABICS = Path(__file__).resolve().parent / Path("syllabic.txt")
+ALPHABETICS = Path(__file__).resolve().parent / Path("alphabetic.txt")
 
 VOWELS = "aeiou"
 CONSONANTS = "jklmnpstw"
@@ -390,21 +521,69 @@ ALLOWABLES = {
     "msa",
 }
 
-PHONOMATCHES = {
-    "non",
-    "nope",
-    "some",
+FALSE_POS_SYLLABIC = {
+    # ordered by frequency in previous TPT data
     "like",
+    "same",
+    "nope",
+    "uwu",  # TODO: emoticon?? uhh?
+    "non",
+    "owo",  # TODO: emoticon??
+    "one",
+    "to",
+    "i",
+    "awesome",
     "use",
-    "imo",
+    "name",
     "time",
+    "imo",  # "in my opinion"
     "man",
-    "also",
+    # "son",  # sona typo?
+    "joke",
+    "so",
+    "ten",
+    "make",
+    "pin",
+    "note",
+    # "aka" # in sandbox
+    "into",
+    "in",
+    "some",
+    "on",
+    "me",
+    "ipa",
+    "sun",
+    "sense",
+    "none",
+    "meme",
+    "wise",
+    # "ono", # TODO: what is this
+    "mon",
+    "take",
+    "luna",
+    "anti",
+    "elo",
+    "an",
+    "win",
+    "won",
+    "we",
+    "men",
+    "ton",
+    "woke",
+    "semi",
+    "male",
 }
 
-ALPHABETIC_MATCHES: Set[str] = set()
-
-IGNORABLES = PHONOMATCHES | ALPHABETIC_MATCHES
+FALSE_POS_ALPHABETIC: Set[str] = {
+    "t",
+    "is",
+    "not",
+    "lol",
+    "also",
+    "isn",  # TODO: tokenizer....
+    "mean",
+    "means",
+}
 
 UCSUR_RANGES = [
     "\\U000F1900-\\U000F1977",  # pu
@@ -439,15 +618,23 @@ with open(SANDBOX) as f:
     sandbox: Dict[str, Dict[str, str]] = json.loads(f.read())
     NIMI_LINKU_SANDBOX = {d["word"] for d in sandbox.values()}
 
+# with open(SYLLABICS) as f:
+#     FALSE_POS_SYLLABIC = {line.strip() for line in f}
+#
+# with open(ALPHABETICS) as f:
+#     FALSE_POS_ALPHABETIC = {line.strip() for line in f}
+
 del linku
 del sandbox
 
 __all__ = [
     "ALLOWABLES",
     "ALL_PUNCT",
-    "ALL_PUNCT_RANGES",
+    "ALL_PUNCT_RANGES_STR",
     "ALPHABET",
     "CONSONANTS",
+    "EMOJI_VARIATION_SELECTOR_RANGES",
+    "EMOJI_VARIATION_SELECTOR_RANGES_STR",
     "NIMI_KU_LILI",
     "NIMI_KU_SULI",
     "NIMI_LINKU_COMMON",
@@ -459,6 +646,8 @@ __all__ = [
     "NIMI_PU_SYNONYMS",
     "POSIX_PUNCT",
     "POSIX_PUNCT_RANGES",
+    "UCSUR_PUNCT_RANGES",
+    "UCSUR_PUNCT_RANGES_STR",
     "UNICODE_PUNCT",
     "UNICODE_PUNCT_RANGES",
     "VOWELS",

sonatoki/ilo.py

@@ -5,12 +5,17 @@ from typing import List, Type, Tuple
 from sonatoki.Filters import Filter
 from sonatoki.Scorers import Number, Scorer
 from sonatoki.Cleaners import Cleaner
-from sonatoki.Tokenizers import Tokenizer
+from sonatoki.Tokenizers import Tokenizer, SentTokenizer, WordTokenizer
 from sonatoki.Preprocessors import Preprocessor
 
+# tokenized, filtered, cleaned, score, result
+Scorecard = Tuple[List[str], List[str], List[str], Number, bool]
+# TODO: scorecard kinda sucks as a name
+
 
 class Ilo:
     __preprocessors: List[Type[Preprocessor]]
+    __sent_tokenizer: Type[Tokenizer]
     __word_tokenizer: Type[Tokenizer]
     __cleaners: List[Type[Cleaner]]
     __ignoring_filters: List[Type[Filter]]
@@ -26,11 +31,13 @@ class Ilo:
         scoring_filters: List[Type[Filter]],
         scorer: Type[Scorer],
         passing_score: Number,
-        word_tokenizer: Type[Tokenizer],
+        word_tokenizer: Type[Tokenizer] = WordTokenizer,
+        sent_tokenizer: Type[Tokenizer] = SentTokenizer,
     ):
         super().__init__()
         # avoid keeping a ref to user's list just in case
         self.__preprocessors = [*preprocessors]
+        self.__sent_tokenizer = sent_tokenizer
         self.__word_tokenizer = word_tokenizer
         self.__cleaners = [*cleaners]
         self.__ignoring_filters = [*ignoring_filters]
@@ -47,6 +54,9 @@ class Ilo:
         """It is *highly* recommended that you run `ilo.preprocess` first."""
         return self.__word_tokenizer.tokenize(msg)
 
+    def sent_tokenize(self, msg: str) -> List[str]:
+        return self.__sent_tokenizer.tokenize(msg)
+
     def clean_token(self, token: str) -> str:
         for c in self.__cleaners:
             token = c.clean(token)
@@ -83,26 +93,60 @@ class Ilo:
     def score_tokens(self, tokens: List[str]) -> float:
         return self.__scorer.score(tokens, self.__scoring_filters)
 
-    def _is_toki_pona(
-        self, message: str
-    ) -> Tuple[str, List[str], List[str], List[str], Number, bool]:
-        """Returns all components of the processing algorithm:
-        - Preprocessed message (str)
+    def _is_toki_pona(self, message: str) -> Scorecard:
+        """Process a message into its tokens, then filters, cleans, and scores
+        them. Returns all parts. Message must already be preprocessed, normally
+        done in `self.is_toki_pona(message)`.
+
+        Returns all components of the processing algorithm except preprocessing:
         - Tokenized message (list[str])
         - Filtered message (list[str])
         - Cleaned message (list[str])
         - Score (float)
-        - Result (bool)"""
-        preprocessed = self.preprocess(message)
-        tokenized = self.word_tokenize(preprocessed)
+        - Result (bool)
+        """
+        tokenized = self.word_tokenize(message)
         filtered = self.filter_tokens(tokenized)
         cleaned = self.clean_tokens(filtered)
         score = self.score_tokens(cleaned)
         result = score >= self.__passing_score
 
-        return preprocessed, tokenized, filtered, cleaned, score, result
+        return tokenized, filtered, cleaned, score, result
 
     def is_toki_pona(self, message: str) -> bool:
         """Determines whether a single statement is or is not Toki Pona."""
+        message = self.preprocess(message)
         *_, result = self._is_toki_pona(message)
         return result
+
+    def _are_toki_pona(self, message: str) -> List[Scorecard]:
+        """Split a message into sentences, then return a list each sentence's
+        results via `self._is_toki_pona()`.
+
+        Message must already be preprocessed, normally done in
+        `self.are_toki_pona(message)`.
+        """
+        results: List[Scorecard] = list()
+        for sentence in self.sent_tokenize(message):
+            result = self._is_toki_pona(sentence)
+            results.append(result)
+        return results
+
+    def are_toki_pona(self, message: str) -> List[bool]:
+        """Splits a statement into sentences, then determines if each is or is not Toki Pona.
+        NOTE: You will need to decide how to score the result. Examples:
+
+        ```
+        def all_must_pass(message: str) -> bool:
+            return all(ILO.are_toki_pona(message))
+
+        def portion_must_pass(message: str, score: Number = 0.8) -> bool:
+            results = ILO.are_toki_pona(message)
+            sent_count = len(results)
+            passing = results.count(True)
+            return (passing / sent_count) >= score
+        ```
+        """
+        message = self.preprocess(message)
+        results = self._are_toki_pona(message)
+        return [res[-1] for res in results]