stream2sentence 0.2.7__tar.gz → 0.2.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {stream2sentence-0.2.7 → stream2sentence-0.2.9}/PKG-INFO +1 -1
- {stream2sentence-0.2.7 → stream2sentence-0.2.9}/setup.py +1 -1
- {stream2sentence-0.2.7 → stream2sentence-0.2.9}/stream2sentence/stream2sentence.py +13 -4
- {stream2sentence-0.2.7 → stream2sentence-0.2.9}/stream2sentence.egg-info/PKG-INFO +1 -1
- {stream2sentence-0.2.7 → stream2sentence-0.2.9}/tests/test_stream2sentence.py +3 -2
- {stream2sentence-0.2.7 → stream2sentence-0.2.9}/README.md +0 -0
- {stream2sentence-0.2.7 → stream2sentence-0.2.9}/setup.cfg +0 -0
- {stream2sentence-0.2.7 → stream2sentence-0.2.9}/stream2sentence/__init__.py +0 -0
- {stream2sentence-0.2.7 → stream2sentence-0.2.9}/stream2sentence.egg-info/SOURCES.txt +0 -0
- {stream2sentence-0.2.7 → stream2sentence-0.2.9}/stream2sentence.egg-info/dependency_links.txt +0 -0
- {stream2sentence-0.2.7 → stream2sentence-0.2.9}/stream2sentence.egg-info/requires.txt +0 -0
- {stream2sentence-0.2.7 → stream2sentence-0.2.9}/stream2sentence.egg-info/top_level.txt +0 -0
- {stream2sentence-0.2.7 → stream2sentence-0.2.9}/tests/test_stream_from_llm.py +0 -0
- {stream2sentence-0.2.7 → stream2sentence-0.2.9}/tests/test_stream_from_llm_old_api.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: stream2sentence
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.9
|
|
4
4
|
Summary: Real-time processing and delivery of sentences from a continuous stream of characters or text chunks.
|
|
5
5
|
Home-page: https://github.com/KoljaB/stream2sentence
|
|
6
6
|
Author: Kolja Beigel
|
|
@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
|
|
|
5
5
|
|
|
6
6
|
setuptools.setup(
|
|
7
7
|
name="stream2sentence",
|
|
8
|
-
version="0.2.
|
|
8
|
+
version="0.2.9",
|
|
9
9
|
author="Kolja Beigel",
|
|
10
10
|
author_email="kolja.beigel@web.de",
|
|
11
11
|
description="Real-time processing and delivery of sentences from a continuous stream of characters or text chunks.",
|
|
@@ -213,7 +213,7 @@ async def generate_sentences_async(
|
|
|
213
213
|
log_characters: bool = False,
|
|
214
214
|
sentence_fragment_delimiters: str = ".?!;:,\n…)]}。-",
|
|
215
215
|
full_sentence_delimiters: str = ".?!\n…。",
|
|
216
|
-
force_first_fragment_after_words=
|
|
216
|
+
force_first_fragment_after_words=30,
|
|
217
217
|
debug=False,
|
|
218
218
|
) -> AsyncIterator[str]:
|
|
219
219
|
"""
|
|
@@ -267,7 +267,7 @@ async def generate_sentences_async(
|
|
|
267
267
|
considered full sentence delimiters. Default is ".?!\n…。".
|
|
268
268
|
force_first_fragment_after_words (int): The number of words after
|
|
269
269
|
which the first sentence fragment is forced to be yielded.
|
|
270
|
-
Default is
|
|
270
|
+
Default is 30 words.
|
|
271
271
|
debug (bool): If True, enables debug mode for logging.
|
|
272
272
|
|
|
273
273
|
Yields:
|
|
@@ -303,6 +303,10 @@ async def generate_sentences_async(
|
|
|
303
303
|
async for char in _generate_characters(generator, log_characters):
|
|
304
304
|
|
|
305
305
|
if char:
|
|
306
|
+
if len(buffer) == 0:
|
|
307
|
+
if not char.isalnum():
|
|
308
|
+
continue
|
|
309
|
+
|
|
306
310
|
buffer += char
|
|
307
311
|
buffer = buffer.lstrip()
|
|
308
312
|
|
|
@@ -322,7 +326,7 @@ async def generate_sentences_async(
|
|
|
322
326
|
|
|
323
327
|
if (
|
|
324
328
|
buffer[-1] in sentence_fragment_delimiters
|
|
325
|
-
or word_count >= force_first_fragment_after_words
|
|
329
|
+
or char.isspace() and word_count >= force_first_fragment_after_words
|
|
326
330
|
):
|
|
327
331
|
|
|
328
332
|
yield_text = _clean_text(
|
|
@@ -330,11 +334,15 @@ async def generate_sentences_async(
|
|
|
330
334
|
cleanup_text_links,
|
|
331
335
|
cleanup_text_emojis)
|
|
332
336
|
if debug:
|
|
333
|
-
|
|
337
|
+
if buffer[-1] in sentence_fragment_delimiters:
|
|
338
|
+
print("\033[36mDebug: Yielding first sentence fragment: \"{}\" because buffer[-1] {} is sentence frag \033[0m".format(yield_text, buffer[-1]))
|
|
339
|
+
else:
|
|
340
|
+
print("\033[36mDebug: Yielding first sentence fragment: \"{}\" because word_count {} is >= force_first_fragment_after_words \033[0m".format(yield_text, word_count))
|
|
334
341
|
|
|
335
342
|
yield yield_text
|
|
336
343
|
|
|
337
344
|
buffer = ""
|
|
345
|
+
word_count = 0
|
|
338
346
|
if not quick_yield_every_fragment:
|
|
339
347
|
is_first_sentence = False
|
|
340
348
|
|
|
@@ -389,6 +397,7 @@ async def generate_sentences_async(
|
|
|
389
397
|
print("\033[36mDebug: Yielding sentence: \"{}\"\033[0m".format(yield_text))
|
|
390
398
|
|
|
391
399
|
yield yield_text
|
|
400
|
+
word_count = 0
|
|
392
401
|
|
|
393
402
|
if quick_yield_for_all_sentences:
|
|
394
403
|
is_first_sentence = True
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: stream2sentence
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.9
|
|
4
4
|
Summary: Real-time processing and delivery of sentences from a continuous stream of characters or text chunks.
|
|
5
5
|
Home-page: https://github.com/KoljaB/stream2sentence
|
|
6
6
|
Author: Kolja Beigel
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import unittest
|
|
2
|
-
from stream2sentence import generate_sentences
|
|
2
|
+
from stream2sentence import generate_sentences, generate_sentences_async
|
|
3
3
|
|
|
4
4
|
class TestSentenceGenerator(unittest.TestCase):
|
|
5
5
|
|
|
6
6
|
def test_chinese(self):
|
|
7
7
|
text = "我喜欢读书。天气很好。我们去公园吧。今天是星期五。早上好。这是我的朋友。请帮我。吃饭了吗?我在学中文。晚安。"
|
|
8
|
-
expected = ["我喜欢读书。", "天气很好。", "我们去公园吧。", "今天是星期五。", "早上好。", "这是我的朋友。", "请帮我。吃饭了吗?", "我在学中文。", "晚安。"]
|
|
8
|
+
#expected = ["我喜欢读书。", "天气很好。", "我们去公园吧。", "今天是星期五。", "早上好。", "这是我的朋友。", "请帮我。吃饭了吗?", "我在学中文。", "晚安。"]
|
|
9
|
+
expected = ["我喜欢读书。", "天气很好。", "我们去公园吧。", "今天是星期五。", "早上好。", "这是我的朋友。", "请帮我。", "吃饭了吗?", "我在学中文。", "晚安。"]
|
|
9
10
|
sentences = list(generate_sentences(text, minimum_sentence_length=2, context_size=2, tokenizer="stanza", language="zh"))
|
|
10
11
|
self.assertEqual(sentences, expected)
|
|
11
12
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{stream2sentence-0.2.7 → stream2sentence-0.2.9}/stream2sentence.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|