sonatoki 0.11.1__tar.gz → 0.11.3__tar.gz

{sonatoki-0.11.1 → sonatoki-0.11.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sonatoki
-Version: 0.11.1
+Version: 0.11.3
 Summary: ilo li moku e toki li pana e sona ni: ni li toki ala toki pona?
 Author-Email: "jan Kekan San (@gregdan3)" <gregory.danielson3@gmail.com>
 License: AGPL-3.0-or-later

{sonatoki-0.11.1 → sonatoki-0.11.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sonatoki"
-version = "0.11.1"
+version = "0.11.3"
 description = "ilo li moku e toki li pana e sona ni: ni li toki ala toki pona?"
 authors = [
     { name = "jan Kekan San (@gregdan3)", email = "gregory.danielson3@gmail.com" },

{sonatoki-0.11.1 → sonatoki-0.11.3}/src/sonatoki/Cleaners.py

@@ -1,6 +1,7 @@
 # STL
 import re
 from abc import ABC, abstractmethod
+from sys import intern
 
 # PDM
 from typing_extensions import override
@@ -21,7 +22,7 @@ class RegexCleaner(Cleaner):
     @classmethod
     @override
     def clean(cls, token: str) -> str:
-        return re.sub(cls.pattern, cls.replace, token)
+        return intern(re.sub(cls.pattern, cls.replace, token))
 
 
 class ConsecutiveDuplicates(Cleaner):
@@ -44,29 +45,30 @@ class ConsecutiveDuplicates(Cleaner):
             return token
 
         output = token[0]
-
         last_output = output.lower()  # ignore case in comparison
         for i in range(1, len(token)):
-            cur_char = token[i].lower()
-            if cur_char == last_output:
+            cur_char = intern(token[i])
+            lower_cur_char = intern(cur_char.lower())
+            if lower_cur_char == last_output:
                 continue
-            output += token[i]  # preserve case of string
-            last_output = cur_char
+            output += cur_char  # preserve case of string
+            last_output = lower_cur_char
+        output = intern(output)
         return output
 
 
 class ConsecutiveDuplicatesRe(RegexCleaner):
     """Reference implementation for `ConsecutiveDuplicates`."""
 
-    pattern = re.compile(r"(.)\1+", flags=re.IGNORECASE)
-    replace = r"\1"
+    pattern: "re.Pattern[str]" = re.compile(r"(.)\1+", flags=re.IGNORECASE)
+    replace: str = r"\1"
 
 
 class Lowercase(Cleaner):
     @classmethod
     @override
     def clean(cls, token: str) -> str:
-        return token.lower()
+        return intern(token.lower())
 
 
 __all__ = [

{sonatoki-0.11.1 → sonatoki-0.11.3}/src/sonatoki/Tokenizers.py

@@ -1,6 +1,7 @@
 # STL
 import re
 from abc import ABC, abstractmethod
+from sys import intern
 from typing import Set, List
 
 # PDM
@@ -40,7 +41,11 @@ class RegexTokenizer(Tokenizer):
     @classmethod
     @override
     def tokenize(cls, s: str) -> List[str]:
-        return [clean for word in re.split(cls.pattern, s) if (clean := word.strip())]
+        return [
+            intern(clean)
+            for word in re.split(cls.pattern, s)
+            if (clean := word.strip())
+        ]
 
 
 class Regex1Tokenizer(Tokenizer):
@@ -50,7 +55,9 @@ class Regex1Tokenizer(Tokenizer):
     @override
     def tokenize(cls, s: str) -> List[str]:
         return [
-            clean for word in regex.split(cls.pattern, s) if (clean := word.strip())
+            intern(clean)
+            for word in regex.split(cls.pattern, s)
+            if (clean := word.strip())
         ]
 
 
@@ -65,7 +72,8 @@ class WordTokenizer(SetTokenizer):
     @classmethod
     def add_token(cls, s: str, tokens: List[str], last_match: int, i: int):
         if i > last_match:
-            tokens.append(s[last_match:i])
+            token = intern(s[last_match:i])
+            tokens.append(token)
 
     @classmethod
     def to_tokens(cls, s: str) -> List[str]:
@@ -91,7 +99,7 @@ class WordTokenizer(SetTokenizer):
 
                 if NimiUCSUR.filter(s[i]):
                     cls.add_token(s, tokens, last_match, i)
-                    tokens.append(s[i])
+                    cls.add_token(s, tokens, i, i + 1)
                     i += 1
                     last_match = i
                     continue

{sonatoki-0.11.1 → sonatoki-0.11.3}/src/sonatoki/constants.py

@@ -538,7 +538,7 @@ QUOTATIVE_PUNCT = """"«»‹›“”‟„⹂「」『』"""
 UCSUR_SENTENCE_PUNCT = """󱦜󱦝"""
 ALL_SENTENCE_PUNCT = BASIC_SENTENCE_PUNCT + UCSUR_SENTENCE_PUNCT
 
-INTRA_WORD_PUNCT = """-'’."""
+INTRA_WORD_PUNCT = """-'’._"""
 
 
 LINKU = Path(__file__).resolve().parent / Path("linku.json")

{sonatoki-0.11.1 → sonatoki-0.11.3}/src/sonatoki/utils.py

@@ -1,6 +1,6 @@
 # STL
 import itertools
-from typing import Set, List, TypeVar, Iterable
+from typing import Set, List, Tuple, TypeVar, Iterable
 
 # LOCAL
 from sonatoki.Cleaners import Lowercase, ConsecutiveDuplicates
@@ -72,12 +72,7 @@ def find_unicode_chars(ranges: List[str]) -> str:
     return "".join(result)
 
 
-def overlapping_pairs(iterable: Iterable[T]) -> Iterable[T]:
-    "s -> (s0,s1), (s1,s2), (s2, s3), ..."
-    return overlapping_ntuples(iterable, n=2)
-
-
-def overlapping_ntuples(iterable: Iterable[T], n: int) -> Iterable[T]:
+def overlapping_ntuples(iterable: Iterable[T], n: int) -> Iterable[Tuple[T, ...]]:
     teed = itertools.tee(iterable, n)
     for i in range(1, n):
         for j in range(i):

{sonatoki-0.11.1 → sonatoki-0.11.3}/tests/test_cleaners.py

@@ -5,7 +5,7 @@ import hypothesis.strategies as st
 from hypothesis import given, assume, example
 
 # LOCAL
-from sonatoki.utils import overlapping_pairs
+from sonatoki.utils import overlapping_ntuples
 from sonatoki.Cleaners import Lowercase, ConsecutiveDuplicates, ConsecutiveDuplicatesRe
 
 # FILESYSTEM
@@ -21,7 +21,7 @@ from .test_utils import PROPER_NAME_RE
 def test_ConsecutiveDuplicatesRe(s: str):
     _ = assume("\n" not in s)
     res = ConsecutiveDuplicatesRe.clean(s)
-    for a, b in overlapping_pairs(res):
+    for a, b in overlapping_ntuples(res, 2):
         assert a.lower() != b.lower(), (s, res)
 
 

{sonatoki-0.11.1 → sonatoki-0.11.3}/tests/tokenize_cases/tokenize_sentences_tok.yml

@@ -104,6 +104,11 @@
     - "monsi"
     - "-"
     - "ma"
+- name: "intraword punctuation 4"
+  input: "look at this variable: leaf_node_right"
+  output:
+    - "look at this variable:"
+    - "leaf_node_right"
 - name: "multiline with fake intraword"
   input: >
     toki!

{sonatoki-0.11.1 → sonatoki-0.11.3}/tests/tokenize_cases/tokenize_words_tok.yml

@@ -187,13 +187,7 @@
     - "e"
     - "sitelen"
     - "[_"
-    - "ike"
-    - "_"
-    - "nanpa"
-    - "_"
-    - "lete"
-    - "_"
-    - "ike"
+    - "ike_nanpa_lete_ike"
     - "]."
     - "ni"
     - "li"
@@ -345,6 +339,15 @@
   input: "whom's't'd've'n't"
   output:
     - "whom's't'd've'n't"
+- name: "underscore"
+  input: "look at this variable: leaf_node_right"
+  output:
+    - "look"
+    - "at"
+    - "this"
+    - "variable"
+    - ":"
+    - "leaf_node_right"
 - name: "just periods"
   input: "..."
   output: