sonatoki 0.3.1__py3-none-any.whl → 0.3.3__py3-none-any.whl

sonatoki/utils.py

@@ -1,12 +1,15 @@
 # STL
 import re
-from typing import Set, List, Iterable
+import itertools
+from typing import Set, List, TypeVar, Iterable
 
 # LOCAL
 from sonatoki.Cleaners import Lowercase, ConsecutiveDuplicates
 
 TO_ESCAPE = ["\\", "^", "[", "]", "-"]
 
+T = TypeVar("T")
+
 
 def prep_dictionary(words: Iterable[str]) -> Set[str]:
     out: Set[str] = set()
@@ -19,7 +22,8 @@ def prep_dictionary(words: Iterable[str]) -> Set[str]:
 
 
 def regex_escape(s: str) -> str:
-    """Escape all characters which must be escaped when embedded in a character class."""
+    """Escape all characters which must be escaped when embedded in a character
+    class."""
     for c in TO_ESCAPE:
         s = s.replace(c, rf"\{c}")  # one backslash
     return s
@@ -69,10 +73,24 @@ def find_unicode_chars(ranges: List[str]) -> str:
     return "".join(result)
 
 
+def overlapping_pairs(iterable: Iterable[T]) -> Iterable[T]:
+    "s -> (s0,s1), (s1,s2), (s2, s3), ..."
+    return overlapping_ntuples(iterable, n=2)
+
+
+def overlapping_ntuples(iterable: Iterable[T], n: int) -> Iterable[T]:
+    teed = itertools.tee(iterable, n)
+    for i in range(1, n):
+        for j in range(i):
+            _ = next(teed[i], None)
+            # offset start by position
+
+    # ends when any iter is empty; all groups will be same size
+    return zip(*teed)
+
+
 if __name__ == "__main__":
-    """
-    Helper script to fetch UNICODE_PUNCT in constants.py
-    """
+    """Helper script to fetch UNICODE_PUNCT in constants.py."""
 
     PUNCT_CATEGORIES = {
         "Pc",

{sonatoki-0.3.1.dist-info → sonatoki-0.3.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sonatoki
-Version: 0.3.1
+Version: 0.3.3
 Summary: ilo li moku e toki li pana e sona ni: ni li toki ala toki pona?
 Author-Email: "jan Kekan San (@gregdan3)" <gregory.danielson3@gmail.com>
 License: AGPL-3.0-or-later

sonatoki-0.3.3.dist-info/RECORD

@@ -0,0 +1,18 @@
+sonatoki-0.3.3.dist-info/METADATA,sha256=b78h2-lsc4aBzkEWrzXTslfvJb-ZVbFjCpgPOF_kYrg,6341
+sonatoki-0.3.3.dist-info/WHEEL,sha256=SOP-4bEE0jbVaCHQGVvF08uWxk5rcSsfEybvoQVHlD8,90
+sonatoki-0.3.3.dist-info/licenses/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
+sonatoki/Cleaners.py,sha256=x2dT3MpDUfbrHA0EP2D3n1sTiKFFi5jw9ha-1dX973o,1958
+sonatoki/Configs.py,sha256=ZNKJGeAxgolAxqNSeS7iADvQEcN4X3tY0iaoiAaL95U,4160
+sonatoki/Filters.py,sha256=mpJBl-YPMF-Yl6mKFXf0D6DwkPR6H424RlvrkSeh4Dc,10714
+sonatoki/Preprocessors.py,sha256=nvAzxpWP9WwT6gOCKcuiz5F8xYDdKIt9bOVUvy9o-G0,4459
+sonatoki/Scorers.py,sha256=LRQLgXKTU2VqhkMHFPVxyVt83DXf85_zrpDGk4ThU24,3811
+sonatoki/Tokenizers.py,sha256=So5_Tu6J98MD3yVcwB_X3lw2uMG0TN6XHcTbQjFCu5Q,4254
+sonatoki/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sonatoki/__main__.py,sha256=6xc-wIrrFo9wTyn4zRQNAmqwmJBtVvCMwV-CrM-hueA,82
+sonatoki/constants.py,sha256=wH3iR32-Ic7vSkrMjAZIvmIysTtkJ-KBVU5zv3Oamqs,12656
+sonatoki/ilo.py,sha256=yyLgNPI0Hmb4f1BzX6IRHr11FPChfL2xDR_9odlr8_8,3849
+sonatoki/linku.json,sha256=fm4-dks5s9x1bs7q82GNngAedVCWilMPCQ_o-j35QL0,270950
+sonatoki/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sonatoki/sandbox.json,sha256=zPtZgJ_CpJa-2Den0gTNlk52f-YEwFVcjMarQXeeu5U,77563
+sonatoki/utils.py,sha256=L984aXxvzfJaZ6GSWRKs7LweOGZYTLK11CdAhpLQr0g,4067
+sonatoki-0.3.3.dist-info/RECORD,,

{sonatoki-0.3.1.dist-info → sonatoki-0.3.3.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: pdm-backend (2.3.0)
+Generator: pdm-backend (2.3.1)
 Root-Is-Purelib: true
 Tag: py3-none-any

sonatoki-0.3.1.dist-info/RECORD

@@ -1,18 +0,0 @@
-sonatoki-0.3.1.dist-info/METADATA,sha256=nWomuM-AeE98VwnWen7qffNclw8emxAf-oFtXwba8wI,6341
-sonatoki-0.3.1.dist-info/WHEEL,sha256=vnE8JVcI2Wz7GRKorsPArnBdnW2SWKWGow5gu5tHlRU,90
-sonatoki-0.3.1.dist-info/licenses/LICENSE,sha256=DZak_2itbUtvHzD3E7GNUYSRK6jdOJ-GqncQ2weavLA,34523
-sonatoki/Cleaners.py,sha256=m0j1a1vs9Mdqp724r9Xfh1Y_tyP6GYCkihv8rH8m7lA,1871
-sonatoki/Configs.py,sha256=NS1_esoDNna8LyH_9bPMkxbo2sMSilYhG1PwYLdq6L8,3402
-sonatoki/Filters.py,sha256=-j5xSZ8URjqalQVGMBabMvJ5ofZWazfN7YPfXkM_4uQ,9429
-sonatoki/Preprocessors.py,sha256=aMXXuFBDlJudvzvukvCa7BixuROXXEb62un7I-TGOGs,4441
-sonatoki/Scorers.py,sha256=W-1uYiqjsDejJzoe592ixs7wHazjJXPhuo-41zuJ26U,3643
-sonatoki/Tokenizers.py,sha256=So5_Tu6J98MD3yVcwB_X3lw2uMG0TN6XHcTbQjFCu5Q,4254
-sonatoki/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sonatoki/__main__.py,sha256=6xc-wIrrFo9wTyn4zRQNAmqwmJBtVvCMwV-CrM-hueA,82
-sonatoki/constants.py,sha256=XTFmEcnLBXwdYXjTq_EuW9e_TWLtnNLz2vFCf8m-sz0,12844
-sonatoki/ilo.py,sha256=yyLgNPI0Hmb4f1BzX6IRHr11FPChfL2xDR_9odlr8_8,3849
-sonatoki/linku.json,sha256=B5KNdhyM5UEfMciROgh1ECHr3i-ASBeMvwrkzNJX47c,271013
-sonatoki/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sonatoki/sandbox.json,sha256=hx6LRsfvmmTtqXcXIyCsfSaGK3DZ-GCdbM8xhZQBHoA,77650
-sonatoki/utils.py,sha256=OMaRyoNvKGKYQCBDjQyaCI58-wMpQ0wrrNjTJKsEZ9Y,3550
-sonatoki-0.3.1.dist-info/RECORD,,