stream2sentence 0.2.7__tar.gz → 0.2.8__tar.gz

{stream2sentence-0.2.7 → stream2sentence-0.2.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: stream2sentence
-Version: 0.2.7
+Version: 0.2.8
 Summary: Real-time processing and delivery of sentences from a continuous stream of characters or text chunks.
 Home-page: https://github.com/KoljaB/stream2sentence
 Author: Kolja Beigel

{stream2sentence-0.2.7 → stream2sentence-0.2.8}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="stream2sentence", 
-    version="0.2.7",
+    version="0.2.8",
     author="Kolja Beigel",
     author_email="kolja.beigel@web.de",
     description="Real-time processing and delivery of sentences from a continuous stream of characters or text chunks.",

{stream2sentence-0.2.7 → stream2sentence-0.2.8}/stream2sentence/stream2sentence.py

@@ -213,7 +213,7 @@ async def generate_sentences_async(
     log_characters: bool = False,
     sentence_fragment_delimiters: str = ".?!;:,\n…)]}。-",
     full_sentence_delimiters: str = ".?!\n…。",
-    force_first_fragment_after_words=15,
+    force_first_fragment_after_words=30,
     debug=False,
 ) -> AsyncIterator[str]:
     """
@@ -267,7 +267,7 @@ async def generate_sentences_async(
           considered full sentence delimiters. Default is ".?!\n…。".
         force_first_fragment_after_words (int): The number of words after
           which the first sentence fragment is forced to be yielded.
-          Default is 15 words.
+          Default is 30 words.
         debug (bool): If True, enables debug mode for logging.
 
     Yields:
@@ -322,7 +322,7 @@ async def generate_sentences_async(
 
                 if (
                     buffer[-1] in sentence_fragment_delimiters
-                    or word_count >= force_first_fragment_after_words
+                    or char.isspace() and word_count >= force_first_fragment_after_words
                 ):
 
                     yield_text = _clean_text(
@@ -330,11 +330,15 @@ async def generate_sentences_async(
                         cleanup_text_links,
                         cleanup_text_emojis)
                     if debug:
-                        print("\033[36mDebug: Yielding first sentence fragment: \"{}\"\033[0m".format(yield_text))                    
+                        if buffer[-1] in sentence_fragment_delimiters:
+                            print("\033[36mDebug: Yielding first sentence fragment: \"{}\" because buffer[-1] {} is sentence frag \033[0m".format(yield_text, buffer[-1]))
+                        else:
+                            print("\033[36mDebug: Yielding first sentence fragment: \"{}\" because word_count {} is >= force_first_fragment_after_words \033[0m".format(yield_text, word_count))
 
                     yield yield_text
 
                     buffer = ""
+                    word_count = 0
                     if not quick_yield_every_fragment:
                         is_first_sentence = False
 
@@ -389,6 +393,7 @@ async def generate_sentences_async(
                                 print("\033[36mDebug: Yielding sentence: \"{}\"\033[0m".format(yield_text))
 
                             yield yield_text
+                            word_count = 0
 
                         if quick_yield_for_all_sentences:
                             is_first_sentence = True

{stream2sentence-0.2.7 → stream2sentence-0.2.8}/stream2sentence.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: stream2sentence
-Version: 0.2.7
+Version: 0.2.8
 Summary: Real-time processing and delivery of sentences from a continuous stream of characters or text chunks.
 Home-page: https://github.com/KoljaB/stream2sentence
 Author: Kolja Beigel

{stream2sentence-0.2.7 → stream2sentence-0.2.8}/tests/test_stream2sentence.py

@@ -1,11 +1,12 @@
 import unittest
-from stream2sentence import generate_sentences
+from stream2sentence import generate_sentences, generate_sentences_async
 
 class TestSentenceGenerator(unittest.TestCase):
 
     def test_chinese(self):
         text = "我喜欢读书。天气很好。我们去公园吧。今天是星期五。早上好。这是我的朋友。请帮我。吃饭了吗？我在学中文。晚安。"
-        expected = ["我喜欢读书。", "天气很好。", "我们去公园吧。", "今天是星期五。", "早上好。", "这是我的朋友。", "请帮我。吃饭了吗？", "我在学中文。", "晚安。"]
+        #expected = ["我喜欢读书。", "天气很好。", "我们去公园吧。", "今天是星期五。", "早上好。", "这是我的朋友。", "请帮我。吃饭了吗？", "我在学中文。", "晚安。"]
+        expected = ["我喜欢读书。", "天气很好。", "我们去公园吧。", "今天是星期五。", "早上好。", "这是我的朋友。", "请帮我。", "吃饭了吗？", "我在学中文。", "晚安。"]
         sentences = list(generate_sentences(text, minimum_sentence_length=2, context_size=2, tokenizer="stanza", language="zh"))
         self.assertEqual(sentences, expected)