stream2sentence 0.2.7__tar.gz → 0.2.9__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: stream2sentence
3
- Version: 0.2.7
3
+ Version: 0.2.9
4
4
  Summary: Real-time processing and delivery of sentences from a continuous stream of characters or text chunks.
5
5
  Home-page: https://github.com/KoljaB/stream2sentence
6
6
  Author: Kolja Beigel
@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
5
5
 
6
6
  setuptools.setup(
7
7
  name="stream2sentence",
8
- version="0.2.7",
8
+ version="0.2.9",
9
9
  author="Kolja Beigel",
10
10
  author_email="kolja.beigel@web.de",
11
11
  description="Real-time processing and delivery of sentences from a continuous stream of characters or text chunks.",
@@ -213,7 +213,7 @@ async def generate_sentences_async(
213
213
  log_characters: bool = False,
214
214
  sentence_fragment_delimiters: str = ".?!;:,\n…)]}。-",
215
215
  full_sentence_delimiters: str = ".?!\n…。",
216
- force_first_fragment_after_words=15,
216
+ force_first_fragment_after_words=30,
217
217
  debug=False,
218
218
  ) -> AsyncIterator[str]:
219
219
  """
@@ -267,7 +267,7 @@ async def generate_sentences_async(
267
267
  considered full sentence delimiters. Default is ".?!\n…。".
268
268
  force_first_fragment_after_words (int): The number of words after
269
269
  which the first sentence fragment is forced to be yielded.
270
- Default is 15 words.
270
+ Default is 30 words.
271
271
  debug (bool): If True, enables debug mode for logging.
272
272
 
273
273
  Yields:
@@ -303,6 +303,10 @@ async def generate_sentences_async(
303
303
  async for char in _generate_characters(generator, log_characters):
304
304
 
305
305
  if char:
306
+ if len(buffer) == 0:
307
+ if not char.isalnum():
308
+ continue
309
+
306
310
  buffer += char
307
311
  buffer = buffer.lstrip()
308
312
 
@@ -322,7 +326,7 @@ async def generate_sentences_async(
322
326
 
323
327
  if (
324
328
  buffer[-1] in sentence_fragment_delimiters
325
- or word_count >= force_first_fragment_after_words
329
+ or char.isspace() and word_count >= force_first_fragment_after_words
326
330
  ):
327
331
 
328
332
  yield_text = _clean_text(
@@ -330,11 +334,15 @@ async def generate_sentences_async(
330
334
  cleanup_text_links,
331
335
  cleanup_text_emojis)
332
336
  if debug:
333
- print("\033[36mDebug: Yielding first sentence fragment: \"{}\"\033[0m".format(yield_text))
337
+ if buffer[-1] in sentence_fragment_delimiters:
338
+ print("\033[36mDebug: Yielding first sentence fragment: \"{}\" because buffer[-1] {} is sentence frag \033[0m".format(yield_text, buffer[-1]))
339
+ else:
340
+ print("\033[36mDebug: Yielding first sentence fragment: \"{}\" because word_count {} is >= force_first_fragment_after_words \033[0m".format(yield_text, word_count))
334
341
 
335
342
  yield yield_text
336
343
 
337
344
  buffer = ""
345
+ word_count = 0
338
346
  if not quick_yield_every_fragment:
339
347
  is_first_sentence = False
340
348
 
@@ -389,6 +397,7 @@ async def generate_sentences_async(
389
397
  print("\033[36mDebug: Yielding sentence: \"{}\"\033[0m".format(yield_text))
390
398
 
391
399
  yield yield_text
400
+ word_count = 0
392
401
 
393
402
  if quick_yield_for_all_sentences:
394
403
  is_first_sentence = True
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: stream2sentence
3
- Version: 0.2.7
3
+ Version: 0.2.9
4
4
  Summary: Real-time processing and delivery of sentences from a continuous stream of characters or text chunks.
5
5
  Home-page: https://github.com/KoljaB/stream2sentence
6
6
  Author: Kolja Beigel
@@ -1,11 +1,12 @@
1
1
  import unittest
2
- from stream2sentence import generate_sentences
2
+ from stream2sentence import generate_sentences, generate_sentences_async
3
3
 
4
4
  class TestSentenceGenerator(unittest.TestCase):
5
5
 
6
6
  def test_chinese(self):
7
7
  text = "我喜欢读书。天气很好。我们去公园吧。今天是星期五。早上好。这是我的朋友。请帮我。吃饭了吗?我在学中文。晚安。"
8
- expected = ["我喜欢读书。", "天气很好。", "我们去公园吧。", "今天是星期五。", "早上好。", "这是我的朋友。", "请帮我。吃饭了吗?", "我在学中文。", "晚安。"]
8
+ #expected = ["我喜欢读书。", "天气很好。", "我们去公园吧。", "今天是星期五。", "早上好。", "这是我的朋友。", "请帮我。吃饭了吗?", "我在学中文。", "晚安。"]
9
+ expected = ["我喜欢读书。", "天气很好。", "我们去公园吧。", "今天是星期五。", "早上好。", "这是我的朋友。", "请帮我。", "吃饭了吗?", "我在学中文。", "晚安。"]
9
10
  sentences = list(generate_sentences(text, minimum_sentence_length=2, context_size=2, tokenizer="stanza", language="zh"))
10
11
  self.assertEqual(sentences, expected)
11
12