sonatoki 0.11.0__tar.gz → 0.11.2__tar.gz

{sonatoki-0.11.0 → sonatoki-0.11.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sonatoki
-Version: 0.11.0
+Version: 0.11.2
 Summary: ilo li moku e toki li pana e sona ni: ni li toki ala toki pona?
 Author-Email: "jan Kekan San (@gregdan3)" <gregory.danielson3@gmail.com>
 License: AGPL-3.0-or-later

{sonatoki-0.11.0 → sonatoki-0.11.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sonatoki"
-version = "0.11.0"
+version = "0.11.2"
 description = "ilo li moku e toki li pana e sona ni: ni li toki ala toki pona?"
 authors = [
     { name = "jan Kekan San (@gregdan3)", email = "gregory.danielson3@gmail.com" },

{sonatoki-0.11.0 → sonatoki-0.11.2}/src/sonatoki/Cleaners.py

@@ -1,6 +1,7 @@
 # STL
 import re
 from abc import ABC, abstractmethod
+from sys import intern
 
 # PDM
 from typing_extensions import override
@@ -21,7 +22,7 @@ class RegexCleaner(Cleaner):
     @classmethod
     @override
     def clean(cls, token: str) -> str:
-        return re.sub(cls.pattern, cls.replace, token)
+        return intern(re.sub(cls.pattern, cls.replace, token))
 
 
 class ConsecutiveDuplicates(Cleaner):
@@ -44,29 +45,30 @@ class ConsecutiveDuplicates(Cleaner):
             return token
 
         output = token[0]
-
         last_output = output.lower()  # ignore case in comparison
         for i in range(1, len(token)):
-            cur_char = token[i].lower()
-            if cur_char == last_output:
+            cur_char = intern(token[i])
+            lower_cur_char = intern(cur_char.lower())
+            if lower_cur_char == last_output:
                 continue
-            output += token[i]  # preserve case of string
-            last_output = cur_char
+            output += cur_char  # preserve case of string
+            last_output = lower_cur_char
+        output = intern(output)
         return output
 
 
 class ConsecutiveDuplicatesRe(RegexCleaner):
     """Reference implementation for `ConsecutiveDuplicates`."""
 
-    pattern = re.compile(r"(.)\1+", flags=re.IGNORECASE)
-    replace = r"\1"
+    pattern: "re.Pattern[str]" = re.compile(r"(.)\1+", flags=re.IGNORECASE)
+    replace: str = r"\1"
 
 
 class Lowercase(Cleaner):
     @classmethod
     @override
     def clean(cls, token: str) -> str:
-        return token.lower()
+        return intern(token.lower())
 
 
 __all__ = [

{sonatoki-0.11.0 → sonatoki-0.11.2}/src/sonatoki/Tokenizers.py

@@ -1,6 +1,7 @@
 # STL
 import re
 from abc import ABC, abstractmethod
+from sys import intern
 from typing import Set, List
 
 # PDM
@@ -40,7 +41,11 @@ class RegexTokenizer(Tokenizer):
     @classmethod
     @override
     def tokenize(cls, s: str) -> List[str]:
-        return [clean for word in re.split(cls.pattern, s) if (clean := word.strip())]
+        return [
+            intern(clean)
+            for word in re.split(cls.pattern, s)
+            if (clean := word.strip())
+        ]
 
 
 class Regex1Tokenizer(Tokenizer):
@@ -50,7 +55,9 @@ class Regex1Tokenizer(Tokenizer):
     @override
     def tokenize(cls, s: str) -> List[str]:
         return [
-            clean for word in regex.split(cls.pattern, s) if (clean := word.strip())
+            intern(clean)
+            for word in regex.split(cls.pattern, s)
+            if (clean := word.strip())
         ]
 
 
@@ -65,7 +72,8 @@ class WordTokenizer(SetTokenizer):
     @classmethod
     def add_token(cls, s: str, tokens: List[str], last_match: int, i: int):
         if i > last_match:
-            tokens.append(s[last_match:i])
+            token = intern(s[last_match:i])
+            tokens.append(token)
 
     @classmethod
     def to_tokens(cls, s: str) -> List[str]:
@@ -91,7 +99,7 @@ class WordTokenizer(SetTokenizer):
 
                 if NimiUCSUR.filter(s[i]):
                     cls.add_token(s, tokens, last_match, i)
-                    tokens.append(s[i])
+                    cls.add_token(s, tokens, i, i + 1)
                     i += 1
                     last_match = i
                     continue

{sonatoki-0.11.0 → sonatoki-0.11.2}/src/sonatoki/constants.py

@@ -498,7 +498,10 @@ ALL_PUNCT_RANGES_STR = "".join(find_unicode_ranges(ALL_PUNCT))
 
 
 UNICODE_WHITESPACE_RANGES = [
-    "\\U00000020",
+    "\\U00000009",  # tab
+    "\\U0000000A",  # line feed
+    "\\U0000000D",  # carriage return
+    "\\U00000020",  # space
     "\\U000000a0",
     "\\U00001680",
     "\\U00002000-\\U0000200a",

{sonatoki-0.11.0 → sonatoki-0.11.2}/src/sonatoki/utils.py

@@ -1,6 +1,6 @@
 # STL
 import itertools
-from typing import Set, List, TypeVar, Iterable
+from typing import Set, List, Tuple, TypeVar, Iterable
 
 # LOCAL
 from sonatoki.Cleaners import Lowercase, ConsecutiveDuplicates
@@ -72,12 +72,7 @@ def find_unicode_chars(ranges: List[str]) -> str:
     return "".join(result)
 
 
-def overlapping_pairs(iterable: Iterable[T]) -> Iterable[T]:
-    "s -> (s0,s1), (s1,s2), (s2, s3), ..."
-    return overlapping_ntuples(iterable, n=2)
-
-
-def overlapping_ntuples(iterable: Iterable[T], n: int) -> Iterable[T]:
+def overlapping_ntuples(iterable: Iterable[T], n: int) -> Iterable[Tuple[T, ...]]:
     teed = itertools.tee(iterable, n)
     for i in range(1, n):
         for j in range(i):

{sonatoki-0.11.0 → sonatoki-0.11.2}/tests/test_cleaners.py

@@ -5,7 +5,7 @@ import hypothesis.strategies as st
 from hypothesis import given, assume, example
 
 # LOCAL
-from sonatoki.utils import overlapping_pairs
+from sonatoki.utils import overlapping_ntuples
 from sonatoki.Cleaners import Lowercase, ConsecutiveDuplicates, ConsecutiveDuplicatesRe
 
 # FILESYSTEM
@@ -21,7 +21,7 @@ from .test_utils import PROPER_NAME_RE
 def test_ConsecutiveDuplicatesRe(s: str):
     _ = assume("\n" not in s)
     res = ConsecutiveDuplicatesRe.clean(s)
-    for a, b in overlapping_pairs(res):
+    for a, b in overlapping_ntuples(res, 2):
         assert a.lower() != b.lower(), (s, res)
 
 

{sonatoki-0.11.0 → sonatoki-0.11.2}/tests/tokenize_cases/tokenize_sentences_tok.yml

@@ -94,6 +94,29 @@
   output:
     - "isn't that game-breaking?"
     - "i think so"
+- name: "intraword punctuation 3"
+  input: "e.g.\n- monsuta\n- monsi\n- ma"
+  output:
+    - "e.g."
+    - "-"
+    - "monsuta"
+    - "-"
+    - "monsi"
+    - "-"
+    - "ma"
+- name: "multiline with fake intraword"
+  input: >
+    toki!
+    sitelen pini ni li tu ala e toki.
+    ni kin.
+    taso ni li pini e toki anu seme:
+    pini la ni li toki sin.
+  output:
+    - "toki!"
+    - "sitelen pini ni li tu ala e toki."
+    - "ni kin."
+    - "taso ni li pini e toki anu seme:"
+    - "pini la ni li toki sin."
 - name: "fake intraword punct 1"
   input: "!.h"
   output: