sonatoki 0.9.1__tar.gz → 0.9.2__tar.gz

{sonatoki-0.9.1 → sonatoki-0.9.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sonatoki
-Version: 0.9.1
+Version: 0.9.2
 Summary: ilo li moku e toki li pana e sona ni: ni li toki ala toki pona?
 Author-Email: "jan Kekan San (@gregdan3)" <gregory.danielson3@gmail.com>
 License: AGPL-3.0-or-later

{sonatoki-0.9.1 → sonatoki-0.9.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sonatoki"
-version = "0.9.1"
+version = "0.9.2"
 description = "ilo li moku e toki li pana e sona ni: ni li toki ala toki pona?"
 authors = [
     { name = "jan Kekan San (@gregdan3)", email = "gregory.danielson3@gmail.com" },

{sonatoki-0.9.1 → sonatoki-0.9.2}/src/sonatoki/Tokenizers.py

@@ -104,6 +104,10 @@ class WordTokenizer(SetTokenizer):
                 # we skipped, but there wasn't another writing character
                 cls.add_token(s, tokens, last_match, i - 1)
                 last_match = i - 1
+                # there may be punctuation though
+                # TODO: this is duplicated
+                while i < slen and cls.is_delimiter(s[i]):
+                    i += 1
 
             cls.add_token(s, tokens, last_match, i)
 

{sonatoki-0.9.1 → sonatoki-0.9.2}/src/sonatoki/constants.py

@@ -507,7 +507,7 @@ SENTENCE_PUNCT = """.?!:;"()[-]«»‹›“”‟„⹂‽·•…「」『』"
 # single quotes are word boundaries if not intra-word, but double quotes are sentence
 # boundaries
 
-INTRA_WORD_PUNCT = """-'’"""
+INTRA_WORD_PUNCT = """-'’."""
 
 
 LINKU = Path(__file__).resolve().parent / Path("linku.json")

{sonatoki-0.9.1 → sonatoki-0.9.2}/tests/test_ilo.py

@@ -165,6 +165,7 @@ EXCESSIVE_ENGLISH = [
     "I wanna see",  # same down to here
     "i'm online all the time",
     "How to Cut a Kiwi",
+    "ni li make e sense",
     "21st",  # previous false positive; fixed by ProperName change
     "a e i o u",  # voting brings this back to false positive zone...
 ]

{sonatoki-0.9.1 → sonatoki-0.9.2}/tests/tokenize_cases/tokenize_words_tok.yml

@@ -53,9 +53,7 @@
   output:
     - "i'm"
     - "an"
-    - "m"
-    - "."
-    - "d"
+    - "m.d"
     - "."
 - name: "english 4"
   input: "it's mind-numbing honestly"
@@ -142,15 +140,7 @@
 - name: periods every word
   input: "mi.unpa.e.mama.sina"
   output:
-    - "mi"
-    - "."
-    - "unpa"
-    - "."
-    - "e"
-    - "."
-    - "mama"
-    - "."
-    - "sina"
+    - "mi.unpa.e.mama.sina"
 - name: simple bold
   input: "**mi unpa e mama sina**"
   output:
@@ -313,6 +303,11 @@
   input: "isn't"
   output:
     - "isn't"
+- name: "simple intrapunct with punct"
+  input: "isn't."
+  output:
+    - "isn't"
+    - "."
 - name: "quoted with intrapunct"
   input: "'bother'"
   output:
@@ -337,3 +332,47 @@
   input: "whom's't'd've'n't"
   output:
     - "whom's't'd've'n't"
+- name: "just periods"
+  input: "..."
+  output:
+    - "..."
+- name: "just periods 2"
+  input: "... ..."
+  output:
+    - "..."
+    - "..."
+- name: "mixed periods spoilers"
+  input: "||...||"
+  output:
+    - "||...||"
+- name: "trailing periods"
+  input: "h.."
+  output:
+    - "h"
+    - ".."
+- name: "trailing periods"
+  input: "h.!"
+  output:
+    - "h"
+    - ".!"
+- name: "trailing period"
+  input: "h."
+  output:
+    - "h"
+    - "."
+- name: "trailing interpunctuation"
+  input: "h-.'"
+  output:
+    - "h"
+    - "-.'"
+- name: "trailing period 2"
+  input: "h. h."
+  output:
+    - "h"
+    - "."
+    - "h"
+    - "."
+- name: "sad face"
+  input: "q.q"
+  output:
+    - "q.q"