sonatoki 0.9.2__py3-none-any.whl → 0.10.0__py3-none-any.whl

sonatoki/Tokenizers.py

@@ -12,9 +12,13 @@ from sonatoki.utils import regex_escape
 from sonatoki.Filters import NimiUCSUR  # seriously this sucks
 from sonatoki.constants import (
     ALL_PUNCT,
-    SENTENCE_PUNCT,
     INTRA_WORD_PUNCT,
+    ALL_SENTENCE_PUNCT,
+    UNICODE_WHITESPACE,
     ALL_PUNCT_RANGES_STR,
+    UCSUR_CARTOUCHE_LEFT,
+    UCSUR_CARTOUCHE_RIGHT,
+    UCSUR_MINUS_CARTOUCHE,
 )
 
 regex.DEFAULT_VERSION = regex.VERSION1
@@ -146,7 +150,9 @@ class WordTokenizerRe1(Regex1Tokenizer):
 
 
 class SentTokenizer(SetTokenizer):
-    delimiters = set(SENTENCE_PUNCT + "\n")  # regex does \n with a flag
+    delimiters: Set[str] = set(ALL_SENTENCE_PUNCT + "\n")  # regex does \n with a flag
+    intra_word_punct: Set[str] = set(INTRA_WORD_PUNCT)
+    all_punct: Set[str] = set(ALL_PUNCT + UNICODE_WHITESPACE)
 
     @classmethod
     @override
@@ -155,16 +161,43 @@ class SentTokenizer(SetTokenizer):
             return []
 
         tokens: List[str] = []
+
+        slen = len(s)
         last_match = 0
-        for i, char in enumerate(s):
-            if char not in cls.delimiters:
+        i = 0
+        while i < slen:
+            # if a cartouche appears, we do not want to split on its punctuation
+            if s[i] == UCSUR_CARTOUCHE_LEFT:
+                right_i = s.find(UCSUR_CARTOUCHE_RIGHT, i)
+                contained: set[str] = set()
+                if right_i > 0:
+                    contained = set(s[i + 1 : right_i])
+                # but it must contain only non-cartouche UCSUR chars
+                if contained and contained.issubset(UCSUR_MINUS_CARTOUCHE):
+                    i = right_i + 1
+                    continue
+            if s[i] not in cls.delimiters:
+                i += 1
                 continue
+            if s[i] in cls.intra_word_punct:
+                prev = s[i - 1] if i > 0 else ""
+                next = s[i + 1] if i + 1 < slen else ""
+                if (
+                    prev
+                    and next
+                    and prev not in cls.all_punct
+                    and next not in cls.all_punct
+                ):
+                    i += 2
+                    continue
 
             match = s[last_match : i + 1].strip()
             last_match = i + 1  # newlines can strip but idc
             if not match:
+                i += 1
                 continue
             tokens.append(match)
+            i += 1
 
         match = s[last_match:].strip()
         if match:
@@ -173,18 +206,24 @@ class SentTokenizer(SetTokenizer):
         return tokens
 
 
+@deprecated(
+    "SentTokenizerRe is a previous reference implementation. Its behavior has diverged from SentTokenizer and it may not be restored."
+)
 class SentTokenizerRe(RegexTokenizer):
     pattern = re.compile(
-        rf"""(?<=[{regex_escape(SENTENCE_PUNCT)}])|$""", flags=re.MULTILINE
+        rf"""(?<=[{regex_escape(ALL_SENTENCE_PUNCT)}])|$""", flags=re.MULTILINE
     )
     # TODO: are <> or {} that common as *sentence* delims? [] are already a stretch
     # TODO: do the typography characters matter?
     # NOTE: | / and , are *not* sentence delimiters for my purpose
 
 
+@deprecated(
+    "SentTokenizerRe1 is a previous reference implementation. Its behavior has diverged from SentTokenizer and it may not be restored."
+)
 class SentTokenizerRe1(Regex1Tokenizer):
     pattern = regex.compile(
-        rf"""(?<=[{regex_escape(SENTENCE_PUNCT)}]|$)""", flags=regex.MULTILINE
+        rf"""(?<=[{regex_escape(ALL_SENTENCE_PUNCT)}]|$)""", flags=regex.MULTILINE
     )
 
 

sonatoki/__main__.py

@@ -24,6 +24,7 @@ from sonatoki.Cleaners import ConsecutiveDuplicates
 from sonatoki.constants import (
     UCSUR_PUNCT_RANGES,
     UNICODE_PUNCT_RANGES,
+    UNICODE_WHITESPACE_RANGES,
     EMOJI_VARIATION_SELECTOR_RANGES,
 )
 
@@ -121,6 +122,11 @@ def regen_unicode_data():
         "Sc",  # Currency
         "So",  # Other
     }
+    WHITESPACE_CATEGORIES = {
+        "Zl",  # Line Separator
+        "Zp",  # Paragraph Separator
+        "Zs",  # Space Separator
+    }
     r"""These characters are in Symbol other (So) but are not in
     `\p{Punctuation}` However, I began excluding them again, because it turns
     out that some sequences of latin alphabet emoji."""
@@ -134,11 +140,15 @@ def regen_unicode_data():
     def is_punctuation(data: List[str]):
         return data[2] in PUNCT_CATEGORIES
 
+    def is_whitespace(data: List[str]):
+        return data[2] in WHITESPACE_CATEGORIES
+
     def get_character(data: List[str]):
         return chr(int(data[0], 16))
 
     unicode_data = download(UNICODE_DATA)
     unicode_punctuation = ""
+    unicode_whitespace = ""
     for line in unicode_data.split("\n"):
         if not line:  # damn you, trailing newline
             continue
@@ -147,24 +157,35 @@ def regen_unicode_data():
         # This does not apply to any currently defined punctuation category.
 
         unicode_data = line.split(";")
-        if not is_punctuation(unicode_data):
+        if is_punctuation(unicode_data):
+            char = get_character(unicode_data)
+            unicode_punctuation += char
+            continue
+        if is_whitespace((unicode_data)):
+            char = get_character(unicode_data)
+            unicode_whitespace += char
             continue
-
-        char = get_character(unicode_data)
-
-        unicode_punctuation += char
 
     unicode_punctuation = emoji.replace_emoji(unicode_punctuation)
 
-    unicode_ranges = find_unicode_ranges(unicode_punctuation)
-    unicode_ranges.extend(UCSUR_PUNCT_RANGES)
-    # unicode_ranges.extend(EMOJI_VARIATION_SELECTOR_RANGES)  # made unnecessary by emoji library
-    unicode_ranges = sorted(unicode_ranges)
+    unicode_punct_ranges = find_unicode_ranges(unicode_punctuation)
+    unicode_punct_ranges.extend(UCSUR_PUNCT_RANGES)
+    unicode_punct_ranges = sorted(unicode_punct_ranges)
     # sorted in case my manual additions are out of order
 
-    if unicode_ranges != UNICODE_PUNCT_RANGES:
-        output = json.dumps(unicode_ranges, indent=4, ensure_ascii=True)
-        print(output)
+    # TODO: can i push these outputs directly into the constants.py file?
+
+    if unicode_punct_ranges != UNICODE_PUNCT_RANGES:
+        output = json.dumps(unicode_punct_ranges, indent=4, ensure_ascii=True)
+        with open("updated_unicode_punct_ranges.txt", "w") as f:
+            f.write(output)
+
+    unicode_whitespace_ranges = find_unicode_ranges(unicode_whitespace)
+    unicode_whitespace_ranges = sorted(unicode_whitespace_ranges)
+    if unicode_whitespace_ranges != UNICODE_WHITESPACE_RANGES:
+        output = json.dumps(unicode_whitespace_ranges, indent=4, ensure_ascii=True)
+        with open("updated_unicode_whitespace_ranges.txt", "w") as f:
+            f.write(output)
 
 
 def main(argv: argparse.Namespace):

sonatoki/constants.py

@@ -109,8 +109,9 @@ UNICODE_PUNCT_RANGES = [
     "\\U00001a1e-\\U00001a1f",
     "\\U00001aa0-\\U00001aa6",
     "\\U00001aa8-\\U00001aad",
+    "\\U00001b4e-\\U00001b4f",
     "\\U00001b5a-\\U00001b6a",
-    "\\U00001b74-\\U00001b7e",
+    "\\U00001b74-\\U00001b7f",
     "\\U00001bfc-\\U00001bff",
     "\\U00001c3b-\\U00001c3f",
     "\\U00001c7e-\\U00001c7f",
@@ -152,7 +153,7 @@ UNICODE_PUNCT_RANGES = [
     "\\U00002329-\\U000023ce",
     "\\U000023d0-\\U000023e8",
     "\\U000023f4-\\U000023f7",
-    "\\U000023fb-\\U00002426",
+    "\\U000023fb-\\U00002429",
     "\\U00002440-\\U0000244a",
     "\\U0000249c-\\U000024c1",
     "\\U000024c3-\\U000024e9",
@@ -248,7 +249,7 @@ UNICODE_PUNCT_RANGES = [
     "\\U000030fb",
     "\\U00003190-\\U00003191",
     "\\U00003196-\\U0000319f",
-    "\\U000031c0-\\U000031e3",
+    "\\U000031c0-\\U000031e5",
     "\\U000031ef",
     "\\U00003200-\\U0000321e",
     "\\U0000322a-\\U00003247",
@@ -321,6 +322,8 @@ UNICODE_PUNCT_RANGES = [
     "\\U00010af0-\\U00010af6",
     "\\U00010b39-\\U00010b3f",
     "\\U00010b99-\\U00010b9c",
+    "\\U00010d6e",
+    "\\U00010d8e-\\U00010d8f",
     "\\U00010ead",
     "\\U00010f55-\\U00010f59",
     "\\U00010f86-\\U00010f89",
@@ -335,6 +338,8 @@ UNICODE_PUNCT_RANGES = [
     "\\U000111dd-\\U000111df",
     "\\U00011238-\\U0001123d",
     "\\U000112a9",
+    "\\U000113d4-\\U000113d5",
+    "\\U000113d7-\\U000113d8",
     "\\U0001144b-\\U0001144f",
     "\\U0001145a-\\U0001145b",
     "\\U0001145d",
@@ -351,6 +356,7 @@ UNICODE_PUNCT_RANGES = [
     "\\U00011a9a-\\U00011a9c",
     "\\U00011a9e-\\U00011aa2",
     "\\U00011b00-\\U00011b09",
+    "\\U00011be1",
     "\\U00011c41-\\U00011c45",
     "\\U00011c70-\\U00011c71",
     "\\U00011ef7-\\U00011ef8",
@@ -363,10 +369,13 @@ UNICODE_PUNCT_RANGES = [
     "\\U00016af5",
     "\\U00016b37-\\U00016b3f",
     "\\U00016b44-\\U00016b45",
+    "\\U00016d6d-\\U00016d6f",
     "\\U00016e97-\\U00016e9a",
     "\\U00016fe2",
     "\\U0001bc9c",
     "\\U0001bc9f",
+    "\\U0001cc00-\\U0001ccef",
+    "\\U0001cd00-\\U0001ceb3",
     "\\U0001cf50-\\U0001cfc3",
     "\\U0001d000-\\U0001d0f5",
     "\\U0001d100-\\U0001d126",
@@ -395,6 +404,7 @@ UNICODE_PUNCT_RANGES = [
     "\\U0001da85-\\U0001da8b",
     "\\U0001e14f",
     "\\U0001e2ff",
+    "\\U0001e5ff",
     "\\U0001e95e-\\U0001e95f",
     "\\U0001ecac",
     "\\U0001ecb0",
@@ -464,16 +474,41 @@ UNICODE_PUNCT_RANGES = [
     "\\U0001f850-\\U0001f859",
     "\\U0001f860-\\U0001f887",
     "\\U0001f890-\\U0001f8ad",
-    "\\U0001f8b0-\\U0001f8b1",
+    "\\U0001f8b0-\\U0001f8bb",
+    "\\U0001f8c0-\\U0001f8c1",
     "\\U0001f900-\\U0001f90b",
     "\\U0001f93b",
     "\\U0001f946",
     "\\U0001fa00-\\U0001fa53",
     "\\U0001fa60-\\U0001fa6d",
     "\\U0001fb00-\\U0001fb92",
-    "\\U0001fb94-\\U0001fbca",
+    "\\U0001fb94-\\U0001fbef",
     "\\U000f1990-\\U000f199d",
 ]
+UNICODE_PUNCT = find_unicode_chars(UNICODE_PUNCT_RANGES)
+# this is a large string.
+
+# `\p{posix_punct}` character class
+POSIX_PUNCT = r"""-!"#$%&'()*+,./:;<=>?@[\]^_`{|}~"""
+POSIX_PUNCT_RANGES = find_unicode_ranges(POSIX_PUNCT)
+
+ALL_PUNCT = "".join(sorted(list(set(POSIX_PUNCT + UNICODE_PUNCT))))
+ALL_PUNCT_RANGES_STR = "".join(find_unicode_ranges(ALL_PUNCT))
+# combined bc the result could be simpler
+
+
+UNICODE_WHITESPACE_RANGES = [
+    "\\U00000020",
+    "\\U000000a0",
+    "\\U00001680",
+    "\\U00002000-\\U0000200a",
+    "\\U00002028-\\U00002029",
+    "\\U0000202f",
+    "\\U0000205f",
+    "\\U00003000",
+]
+UNICODE_WHITESPACE = find_unicode_chars(UNICODE_WHITESPACE_RANGES)
+UNICODE_WHITESPACE_RANGES_STR = "".join(UNICODE_WHITESPACE_RANGES)
 
 
 NOT_IN_PUNCT_CLASS = r"Ⓐ-ⓩ🄰-🅉🅐-🅩🅰-🆉"
@@ -482,9 +517,7 @@ EMOJI_VARIATION_SELECTOR_RANGES = ["\\U0000fe0e-\\U0000fe0f"]
 EMOJI_VARIATION_SELECTOR_RANGES_STR = "".join(EMOJI_VARIATION_SELECTOR_RANGES)
 """All variation selectors are in Nonspacing Mark (Mn), but it is more apt to
 mark these two as punctuation, since they are used exclusively for rendering
-emoji.
-
-But it's even better to use the Emoji filter.
+emoji. But it's best to use the Emoji filter.
 """
 
 UCSUR_PUNCT_RANGES = ["\\U000f1990-\\U000f199d"]
@@ -492,20 +525,15 @@ UCSUR_PUNCT_RANGES_STR = "".join(UCSUR_PUNCT_RANGES)
 """Private Use Area glyphs are given the apt but unhelpful 'Private Use'
 class."""
 
-UNICODE_PUNCT = find_unicode_chars(UNICODE_PUNCT_RANGES)
-# this is a large string.
-
-# `\p{posix_punct}` character class
-POSIX_PUNCT = r"""-!"#$%&'()*+,./:;<=>?@[\]^_`{|}~"""
-POSIX_PUNCT_RANGES = find_unicode_ranges(POSIX_PUNCT)
-
-ALL_PUNCT = "".join(sorted(list(set(POSIX_PUNCT + UNICODE_PUNCT))))
-ALL_PUNCT_RANGES_STR = "".join(find_unicode_ranges(ALL_PUNCT))
-# combined bc the result could be simpler
+UCSUR_CARTOUCHE_LEFT = "󱦐"
+UCSUR_CARTOUCHE_RIGHT = "󱦑"
 
-SENTENCE_PUNCT = """.?!:;"()[-]«»‹›“”‟„⹂‽·•…「」『』"""
 # single quotes are word boundaries if not intra-word, but double quotes are sentence
 # boundaries
+BASIC_SENTENCE_PUNCT = """.?!:;()[-]‽·•…"""
+QUOTATIVE_PUNCT = """"«»‹›“”‟„⹂「」『』"""
+UCSUR_SENTENCE_PUNCT = """󱦜󱦝"""
+ALL_SENTENCE_PUNCT = BASIC_SENTENCE_PUNCT + UCSUR_SENTENCE_PUNCT
 
 INTRA_WORD_PUNCT = """-'’."""
 
@@ -691,7 +719,11 @@ UCSUR_RANGES = [
     "\\U000F19A0-\\U000F19A3",  # ku lili
 ]
 NIMI_UCSUR = find_unicode_chars(UCSUR_RANGES)
-
+ALL_UCSUR = NIMI_UCSUR + find_unicode_chars(UCSUR_PUNCT_RANGES)
+UCSUR_MINUS_CARTOUCHE = set(ALL_UCSUR).difference(
+    {UCSUR_CARTOUCHE_LEFT, UCSUR_CARTOUCHE_RIGHT}
+)
+print(UCSUR_MINUS_CARTOUCHE)
 
 # NIMI_PU_UCSUR_RANGES = ["\\U000F1900-\\U000F1977"]
 # NIMI_PU_ALE_UCSUR_RANGES = NIMI_PU_UCSUR_RANGES + ["\\U000F1978-\\U000F197A"]
@@ -757,7 +789,9 @@ __all__ = [
     "POSIX_PUNCT_RANGES",
     "UCSUR_PUNCT_RANGES",
     "UCSUR_PUNCT_RANGES_STR",
+    "UCSUR_SENTENCE_PUNCT",
     "UNICODE_PUNCT",
     "UNICODE_PUNCT_RANGES",
+    "UNICODE_WHITESPACE",
     "VOWELS",
 ]

{sonatoki-0.9.2.dist-info → sonatoki-0.10.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sonatoki
-Version: 0.9.2
+Version: 0.10.0
 Summary: ilo li moku e toki li pana e sona ni: ni li toki ala toki pona?
 Author-Email: "jan Kekan San (@gregdan3)" <gregory.danielson3@gmail.com>
 License: AGPL-3.0-or-later

{sonatoki-0.9.2.dist-info → sonatoki-0.10.0.dist-info}/RECORD

@@ -1,17 +1,17 @@
-sonatoki-0.9.2.dist-info/METADATA,sha256=nTqR-hm823FWnDVMJCgoWwmhSU4RaE2fdayXQcixd4o,6893
-sonatoki-0.9.2.dist-info/WHEEL,sha256=thaaA2w1JzcGC48WYufAs8nrYZjJm8LqNfnXFOFyCC4,90
-sonatoki-0.9.2.dist-info/entry_points.txt,sha256=6OYgBcLyFCUgeqLgnvMyOJxPCWzgy7se4rLPKtNonMs,34
-sonatoki-0.9.2.dist-info/licenses/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
+sonatoki-0.10.0.dist-info/METADATA,sha256=FS4LM5QUcxvHUY5Zq1IyT85MRAtJiq_sNWArztUI8D8,6894
+sonatoki-0.10.0.dist-info/WHEEL,sha256=thaaA2w1JzcGC48WYufAs8nrYZjJm8LqNfnXFOFyCC4,90
+sonatoki-0.10.0.dist-info/entry_points.txt,sha256=6OYgBcLyFCUgeqLgnvMyOJxPCWzgy7se4rLPKtNonMs,34
+sonatoki-0.10.0.dist-info/licenses/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
 sonatoki/Cleaners.py,sha256=x2dT3MpDUfbrHA0EP2D3n1sTiKFFi5jw9ha-1dX973o,1958
 sonatoki/Configs.py,sha256=6TY-G1nZFGv5EcElatWvI5MagwVCo92D5TTl7s2PX_s,4899
 sonatoki/Filters.py,sha256=8HAtR6_Rk6GPboaS_MHwSjZBJxYnAA8kYbRPI0eR6sM,14823
 sonatoki/Preprocessors.py,sha256=RmzkvPVo6Kdx1rZ5HeR9cTtx6oxpp2iLKrOMCUEqIrM,7107
 sonatoki/Scorers.py,sha256=zkdWc0hbtCX1HPdhI2tu2mL4Z5_S5sv7T83MefE4Yik,7756
-sonatoki/Tokenizers.py,sha256=cfWWZCfvn2tNJChDrofHrORZExp17g0rPmH5ydWgTQY,5219
+sonatoki/Tokenizers.py,sha256=yAHqVF7G-bH5i7nsvYH-dMV2qjeKvLW2W2F-fgyUnR4,6783
 sonatoki/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sonatoki/__main__.py,sha256=QIWRLYS1jb7OBUBK5s8kYoeiMv6MLBlt_I7H7tIVjpU,5745
+sonatoki/__main__.py,sha256=394ldEB4tFpw1UJLV4S4jJ55NfyLgH8rE7o3VWJoGik,6650
 sonatoki/alphabetic.txt,sha256=duyqAKilD2vLIr75RShCIAnktNJcGeEoQIk18V6czmg,11702
-sonatoki/constants.py,sha256=BrU45haroW-ya3qmFsVk7fdTUGyoYVw1MdVVnpiWjt8,19517
+sonatoki/constants.py,sha256=0MWyk7a6Hq8hVxV-WJ6LEkuq8LnJy-qKUah6I3PbRns,20602
 sonatoki/ilo.py,sha256=Dsn0yagkwjqpAQoCj6mkZ6NqWeanRF2lxNDNoqjWGLo,5993
 sonatoki/linku.json,sha256=U5KVxFJSageQydXXDsQCT8X_QoNAK2OaZhJmbu0eoZo,299939
 sonatoki/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -19,4 +19,4 @@ sonatoki/sandbox.json,sha256=QAviQZ7_nwstUr1ejKegxiIoYmBL2YJIoiZovDYNFL8,147485
 sonatoki/syllabic.txt,sha256=HnqY4TrZ3tPcHah3TsvG9F9gjMrnAGdJ8hHJNHyyUPc,1712
 sonatoki/types.py,sha256=VjYSGAzsbR_d3mg8n-VHg__7LyXpmGdEIMDsbPHyxFw,1265
 sonatoki/utils.py,sha256=sT5xLMEj0aLpy8GP92HKblJU1Wt1m8NUlMgCFWB32xQ,2265
-sonatoki-0.9.2.dist-info/RECORD,,
+sonatoki-0.10.0.dist-info/RECORD,,