sonatoki 0.11.0__py3-none-any.whl → 0.11.2__py3-none-any.whl

sonatoki/Cleaners.py

@@ -1,6 +1,7 @@
 # STL
 import re
 from abc import ABC, abstractmethod
+from sys import intern
 
 # PDM
 from typing_extensions import override
@@ -21,7 +22,7 @@ class RegexCleaner(Cleaner):
     @classmethod
     @override
     def clean(cls, token: str) -> str:
-        return re.sub(cls.pattern, cls.replace, token)
+        return intern(re.sub(cls.pattern, cls.replace, token))
 
 
 class ConsecutiveDuplicates(Cleaner):
@@ -44,29 +45,30 @@ class ConsecutiveDuplicates(Cleaner):
             return token
 
         output = token[0]
-
         last_output = output.lower()  # ignore case in comparison
         for i in range(1, len(token)):
-            cur_char = token[i].lower()
-            if cur_char == last_output:
+            cur_char = intern(token[i])
+            lower_cur_char = intern(cur_char.lower())
+            if lower_cur_char == last_output:
                 continue
-            output += token[i]  # preserve case of string
-            last_output = cur_char
+            output += cur_char  # preserve case of string
+            last_output = lower_cur_char
+        output = intern(output)
         return output
 
 
 class ConsecutiveDuplicatesRe(RegexCleaner):
     """Reference implementation for `ConsecutiveDuplicates`."""
 
-    pattern = re.compile(r"(.)\1+", flags=re.IGNORECASE)
-    replace = r"\1"
+    pattern: "re.Pattern[str]" = re.compile(r"(.)\1+", flags=re.IGNORECASE)
+    replace: str = r"\1"
 
 
 class Lowercase(Cleaner):
     @classmethod
     @override
     def clean(cls, token: str) -> str:
-        return token.lower()
+        return intern(token.lower())
 
 
 __all__ = [

sonatoki/Tokenizers.py

@@ -1,6 +1,7 @@
 # STL
 import re
 from abc import ABC, abstractmethod
+from sys import intern
 from typing import Set, List
 
 # PDM
@@ -40,7 +41,11 @@ class RegexTokenizer(Tokenizer):
     @classmethod
     @override
     def tokenize(cls, s: str) -> List[str]:
-        return [clean for word in re.split(cls.pattern, s) if (clean := word.strip())]
+        return [
+            intern(clean)
+            for word in re.split(cls.pattern, s)
+            if (clean := word.strip())
+        ]
 
 
 class Regex1Tokenizer(Tokenizer):
@@ -50,7 +55,9 @@ class Regex1Tokenizer(Tokenizer):
     @override
     def tokenize(cls, s: str) -> List[str]:
         return [
-            clean for word in regex.split(cls.pattern, s) if (clean := word.strip())
+            intern(clean)
+            for word in regex.split(cls.pattern, s)
+            if (clean := word.strip())
         ]
 
 
@@ -65,7 +72,8 @@ class WordTokenizer(SetTokenizer):
     @classmethod
     def add_token(cls, s: str, tokens: List[str], last_match: int, i: int):
         if i > last_match:
-            tokens.append(s[last_match:i])
+            token = intern(s[last_match:i])
+            tokens.append(token)
 
     @classmethod
     def to_tokens(cls, s: str) -> List[str]:
@@ -91,7 +99,7 @@ class WordTokenizer(SetTokenizer):
 
                 if NimiUCSUR.filter(s[i]):
                     cls.add_token(s, tokens, last_match, i)
-                    tokens.append(s[i])
+                    cls.add_token(s, tokens, i, i + 1)
                     i += 1
                     last_match = i
                     continue

sonatoki/constants.py

@@ -498,7 +498,10 @@ ALL_PUNCT_RANGES_STR = "".join(find_unicode_ranges(ALL_PUNCT))
 
 
 UNICODE_WHITESPACE_RANGES = [
-    "\\U00000020",
+    "\\U00000009",  # tab
+    "\\U0000000A",  # line feed
+    "\\U0000000D",  # carriage return
+    "\\U00000020",  # space
     "\\U000000a0",
     "\\U00001680",
     "\\U00002000-\\U0000200a",

sonatoki/utils.py

@@ -1,6 +1,6 @@
 # STL
 import itertools
-from typing import Set, List, TypeVar, Iterable
+from typing import Set, List, Tuple, TypeVar, Iterable
 
 # LOCAL
 from sonatoki.Cleaners import Lowercase, ConsecutiveDuplicates
@@ -72,12 +72,7 @@ def find_unicode_chars(ranges: List[str]) -> str:
     return "".join(result)
 
 
-def overlapping_pairs(iterable: Iterable[T]) -> Iterable[T]:
-    "s -> (s0,s1), (s1,s2), (s2, s3), ..."
-    return overlapping_ntuples(iterable, n=2)
-
-
-def overlapping_ntuples(iterable: Iterable[T], n: int) -> Iterable[T]:
+def overlapping_ntuples(iterable: Iterable[T], n: int) -> Iterable[Tuple[T, ...]]:
     teed = itertools.tee(iterable, n)
     for i in range(1, n):
         for j in range(i):

{sonatoki-0.11.0.dist-info → sonatoki-0.11.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sonatoki
-Version: 0.11.0
+Version: 0.11.2
 Summary: ilo li moku e toki li pana e sona ni: ni li toki ala toki pona?
 Author-Email: "jan Kekan San (@gregdan3)" <gregory.danielson3@gmail.com>
 License: AGPL-3.0-or-later

{sonatoki-0.11.0.dist-info → sonatoki-0.11.2.dist-info}/RECORD

@@ -1,22 +1,22 @@
-sonatoki-0.11.0.dist-info/METADATA,sha256=LXxje9dMJHy-2LGPS0nSRre5jWq4qvGU2bLEpJsUbpo,7303
-sonatoki-0.11.0.dist-info/WHEEL,sha256=thaaA2w1JzcGC48WYufAs8nrYZjJm8LqNfnXFOFyCC4,90
-sonatoki-0.11.0.dist-info/entry_points.txt,sha256=6OYgBcLyFCUgeqLgnvMyOJxPCWzgy7se4rLPKtNonMs,34
-sonatoki-0.11.0.dist-info/licenses/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
-sonatoki/Cleaners.py,sha256=x2dT3MpDUfbrHA0EP2D3n1sTiKFFi5jw9ha-1dX973o,1958
+sonatoki-0.11.2.dist-info/METADATA,sha256=DVseX_yIbP6VzT8tmmjU0Tual5Fu1Dg-Qr2y-fMptIg,7303
+sonatoki-0.11.2.dist-info/WHEEL,sha256=9P2ygRxDrTJz3gsagc0Z96ukrxjr-LFBGOgv3AuKlCA,90
+sonatoki-0.11.2.dist-info/entry_points.txt,sha256=6OYgBcLyFCUgeqLgnvMyOJxPCWzgy7se4rLPKtNonMs,34
+sonatoki-0.11.2.dist-info/licenses/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
+sonatoki/Cleaners.py,sha256=3qLpX2I2aqNxSmqtSXOvIBDaryEw9ooArvFHl2PflRE,2118
 sonatoki/Configs.py,sha256=cQizs-wqgtM9T9F4kkUsDHpIXN91p4FAgSMefDl333s,5114
 sonatoki/Filters.py,sha256=8HAtR6_Rk6GPboaS_MHwSjZBJxYnAA8kYbRPI0eR6sM,14823
 sonatoki/Preprocessors.py,sha256=RmzkvPVo6Kdx1rZ5HeR9cTtx6oxpp2iLKrOMCUEqIrM,7107
 sonatoki/Scorers.py,sha256=zkdWc0hbtCX1HPdhI2tu2mL4Z5_S5sv7T83MefE4Yik,7756
-sonatoki/Tokenizers.py,sha256=yAHqVF7G-bH5i7nsvYH-dMV2qjeKvLW2W2F-fgyUnR4,6783
+sonatoki/Tokenizers.py,sha256=BUs2jhnthP1WqknFiEYYjazvy-hcHQ-kyHkzQwWe7d0,6941
 sonatoki/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sonatoki/__main__.py,sha256=394ldEB4tFpw1UJLV4S4jJ55NfyLgH8rE7o3VWJoGik,6650
 sonatoki/alphabetic.txt,sha256=duyqAKilD2vLIr75RShCIAnktNJcGeEoQIk18V6czmg,11702
-sonatoki/constants.py,sha256=KhE385XkF-64bGBxkQNisu7vODsjpOfmrL8bnyQP_1k,20572
+sonatoki/constants.py,sha256=ifh9VIYb3cDjbcRFNSRc2x3P_Jxb5LVnWfiHlzsBAV0,20677
 sonatoki/ilo.py,sha256=MWoONZaYh8h92ZrMlG-MkNktFyqHX8Jb5zOD57800KI,6755
 sonatoki/linku.json,sha256=U5KVxFJSageQydXXDsQCT8X_QoNAK2OaZhJmbu0eoZo,299939
 sonatoki/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sonatoki/sandbox.json,sha256=QAviQZ7_nwstUr1ejKegxiIoYmBL2YJIoiZovDYNFL8,147485
 sonatoki/syllabic.txt,sha256=HnqY4TrZ3tPcHah3TsvG9F9gjMrnAGdJ8hHJNHyyUPc,1712
 sonatoki/types.py,sha256=VjYSGAzsbR_d3mg8n-VHg__7LyXpmGdEIMDsbPHyxFw,1265
-sonatoki/utils.py,sha256=sT5xLMEj0aLpy8GP92HKblJU1Wt1m8NUlMgCFWB32xQ,2265
-sonatoki-0.11.0.dist-info/RECORD,,
+sonatoki/utils.py,sha256=jFW2LcqLfeR_dlEEcPiBsXOp8i2Fxdh_t6LCwu-mxN0,2132
+sonatoki-0.11.2.dist-info/RECORD,,

{sonatoki-0.11.0.dist-info → sonatoki-0.11.2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: pdm-backend (2.4.3)
+Generator: pdm-backend (2.4.5)
 Root-Is-Purelib: true
 Tag: py3-none-any