stream2sentence 0.2.6__tar.gz → 0.2.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: stream2sentence
3
- Version: 0.2.6
3
+ Version: 0.2.7
4
4
  Summary: Real-time processing and delivery of sentences from a continuous stream of characters or text chunks.
5
5
  Home-page: https://github.com/KoljaB/stream2sentence
6
6
  Author: Kolja Beigel
@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
5
5
 
6
6
  setuptools.setup(
7
7
  name="stream2sentence",
8
- version="0.2.6",
8
+ version="0.2.7",
9
9
  author="Kolja Beigel",
10
10
  author_email="kolja.beigel@web.de",
11
11
  description="Real-time processing and delivery of sentences from a continuous stream of characters or text chunks.",
@@ -0,0 +1,5 @@
1
+ from .stream2sentence import (
2
+ generate_sentences,
3
+ generate_sentences_async,
4
+ init_tokenizer,
5
+ )
@@ -3,11 +3,22 @@ Real-time processing and delivery of sentences
3
3
  from a continuous stream of characters or text chunks
4
4
  """
5
5
 
6
- from typing import Iterator
6
+ import functools
7
7
  import logging
8
- import emoji
9
- import time
10
8
  import re
9
+ import time
10
+ from typing import (
11
+ AsyncIterable,
12
+ AsyncIterator,
13
+ Awaitable,
14
+ Callable,
15
+ Concatenate,
16
+ Iterable,
17
+ Iterator,
18
+ ParamSpec,
19
+ )
20
+
21
+ import emoji
11
22
 
12
23
  current_tokenizer = "nltk"
13
24
  stanza_initialized = False
@@ -15,7 +26,7 @@ nltk_initialized = False
15
26
  nlp = None
16
27
 
17
28
 
18
- def initialize_nltk():
29
+ def initialize_nltk(debug=False):
19
30
  """
20
31
  Initializes NLTK by downloading required data for sentence tokenization.
21
32
  """
@@ -23,17 +34,19 @@ def initialize_nltk():
23
34
  if nltk_initialized:
24
35
  return
25
36
 
26
- logging.info("Initializing NLTK")
37
+ logging.info("Initializing NLTK Tokenizer")
27
38
 
28
- import nltk
29
39
  try:
30
- _ = nltk.data.find('tokenizers/punkt')
31
- except LookupError:
32
- nltk.download('punkt')
33
- nltk_initialized = True
40
+ import nltk
41
+
42
+ nltk.download("punkt_tab", quiet=not debug)
43
+ nltk_initialized = True
44
+ except Exception as e:
45
+ print(f"Error initializing nltk tokenizer: {e}")
46
+ nltk_initialized = False
34
47
 
35
48
 
36
- def initialize_stanza(language: str = "en"):
49
+ def initialize_stanza(language: str = "en", offline=False):
37
50
  """
38
51
  Initializes Stanza by downloading required data for sentence tokenization.
39
52
  """
@@ -41,12 +54,19 @@ def initialize_stanza(language: str = "en"):
41
54
  if stanza_initialized:
42
55
  return
43
56
 
44
- logging.info("Initializing Stanza")
57
+ logging.info("Initializing Stanza Tokenizer")
45
58
 
46
- import stanza
47
- stanza.download(language)
48
- nlp = stanza.Pipeline(language)
49
- stanza_initialized = True
59
+ try:
60
+ import stanza
61
+
62
+ if not offline:
63
+ stanza.download(language)
64
+
65
+ nlp = stanza.Pipeline(language, download_method=None)
66
+ stanza_initialized = True
67
+ except Exception as e:
68
+ print(f"Error initializing stanza tokenizer: {e}")
69
+ stanza_initialized = False
50
70
 
51
71
 
52
72
  def _remove_links(text: str) -> str:
@@ -60,11 +80,11 @@ def _remove_links(text: str) -> str:
60
80
  str: Text with links removed
61
81
  """
62
82
  pattern = (
63
- r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|'
64
- r'[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
83
+ r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|"
84
+ r"[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+"
65
85
  )
66
86
 
67
- return re.sub(pattern, '', text)
87
+ return re.sub(pattern, "", text)
68
88
 
69
89
 
70
90
  def _remove_emojis(text: str) -> str:
@@ -77,11 +97,12 @@ def _remove_emojis(text: str) -> str:
77
97
  Returns:
78
98
  str: Text with emojis removed
79
99
  """
80
- return emoji.replace_emoji(text, u'')
100
+ return emoji.replace_emoji(text, "")
81
101
 
82
102
 
83
- def _generate_characters(generator: Iterator[str],
84
- log_characters: bool = False) -> Iterator[str]:
103
+ async def _generate_characters(
104
+ generator: AsyncIterable[str], log_characters: bool = False
105
+ ) -> AsyncIterator[str]:
85
106
  """
86
107
  Generates individual characters from a text generator.
87
108
 
@@ -94,7 +115,7 @@ def _generate_characters(generator: Iterator[str],
94
115
  """
95
116
  if log_characters:
96
117
  print("Stream: ", end="", flush=True)
97
- for chunk in generator:
118
+ async for chunk in generator:
98
119
  for char in chunk:
99
120
  if log_characters:
100
121
  print(char, end="", flush=True)
@@ -103,10 +124,12 @@ def _generate_characters(generator: Iterator[str],
103
124
  print()
104
125
 
105
126
 
106
- def _clean_text(text: str,
107
- cleanup_text_links: bool = False,
108
- cleanup_text_emojis: bool = False,
109
- strip_text: bool = True) -> str:
127
+ def _clean_text(
128
+ text: str,
129
+ cleanup_text_links: bool = False,
130
+ cleanup_text_emojis: bool = False,
131
+ strip_text: bool = True,
132
+ ) -> str:
110
133
  """
111
134
  Cleans the text by removing links and emojis.
112
135
 
@@ -129,7 +152,7 @@ def _clean_text(text: str,
129
152
  return text
130
153
 
131
154
 
132
- def _tokenize_sentences(text: str, tokenize_sentences=None) -> Iterator[str]:
155
+ def _tokenize_sentences(text: str, tokenize_sentences=None) -> list[str]:
133
156
  """
134
157
  Tokenizes sentences from the input text.
135
158
 
@@ -147,51 +170,52 @@ def _tokenize_sentences(text: str, tokenize_sentences=None) -> Iterator[str]:
147
170
  nlp_start_time = time.time()
148
171
  if current_tokenizer == "nltk":
149
172
  import nltk
173
+
150
174
  sentences = nltk.tokenize.sent_tokenize(text)
151
175
  elif current_tokenizer == "stanza":
152
176
  import stanza
177
+
153
178
  global nlp
154
179
  doc = nlp(text)
155
180
  sentences = [sentence.text for sentence in doc.sentences]
156
181
  else:
157
182
  raise ValueError(f"Unknown tokenizer: {current_tokenizer}")
158
183
  nlp_end_time = time.time()
159
- logging.info("Time to split sentences: "
160
- f"{nlp_end_time - nlp_start_time}")
184
+ logging.info("Time to split sentences: " f"{nlp_end_time - nlp_start_time}")
161
185
  return sentences
162
186
 
163
187
 
164
- def init_tokenizer(tokenizer: str, language: str = "en"):
188
+ def init_tokenizer(tokenizer: str, language: str = "en", offline=False, debug=False):
165
189
  """
166
190
  Initializes the sentence tokenizer.
167
191
  """
168
192
  if tokenizer == "nltk":
169
- initialize_nltk()
193
+ initialize_nltk(debug)
170
194
  elif tokenizer == "stanza":
171
- initialize_stanza(language)
195
+ initialize_stanza(language,offline=offline)
172
196
  else:
173
197
  logging.warning(f"Unknown tokenizer: {tokenizer}")
174
198
 
175
-
176
- def generate_sentences(generator: Iterator[str],
177
- context_size: int = 12,
178
- context_size_look_overhead: int = 12,
179
- minimum_sentence_length: int = 10,
180
- minimum_first_fragment_length: int = 10,
181
- quick_yield_single_sentence_fragment: bool = False,
182
- quick_yield_for_all_sentences: bool = False,
183
- quick_yield_every_fragment : bool = False,
184
- cleanup_text_links: bool = False,
185
- cleanup_text_emojis: bool = False,
186
- tokenize_sentences=None,
187
- tokenizer: str = "nltk",
188
- language: str = "en",
189
- log_characters: bool = False,
190
- sentence_fragment_delimiters: str = ".?!;:,\n…)]}。-",
191
- full_sentence_delimiters: str = ".?!\n…。",
192
- force_first_fragment_after_words=15,
193
- debug_mode=False
194
- ) -> Iterator[str]:
199
+ async def generate_sentences_async(
200
+ generator: AsyncIterable[str],
201
+ context_size: int = 12,
202
+ context_size_look_overhead: int = 12,
203
+ minimum_sentence_length: int = 10,
204
+ minimum_first_fragment_length=10,
205
+ quick_yield_single_sentence_fragment: bool = False,
206
+ quick_yield_for_all_sentences: bool = False,
207
+ quick_yield_every_fragment: bool = False,
208
+ cleanup_text_links: bool = False,
209
+ cleanup_text_emojis: bool = False,
210
+ tokenize_sentences=None,
211
+ tokenizer: str = "nltk",
212
+ language: str = "en",
213
+ log_characters: bool = False,
214
+ sentence_fragment_delimiters: str = ".?!;:,\n…)]}。-",
215
+ full_sentence_delimiters: str = ".?!\n…。",
216
+ force_first_fragment_after_words=15,
217
+ debug=False,
218
+ ) -> AsyncIterator[str]:
195
219
  """
196
220
  Generates well-formed sentences from a stream of characters or text chunks
197
221
  provided by an input generator.
@@ -244,7 +268,7 @@ def generate_sentences(generator: Iterator[str],
244
268
  force_first_fragment_after_words (int): The number of words after
245
269
  which the first sentence fragment is forced to be yielded.
246
270
  Default is 15 words.
247
- debug_mode (bool): If True, enables debug mode for logging.
271
+ debug (bool): If True, enables debug mode for logging.
248
272
 
249
273
  Yields:
250
274
  Iterator[str]: An iterator of complete sentences constructed from the
@@ -262,10 +286,10 @@ def generate_sentences(generator: Iterator[str],
262
286
  # Initialize the tokenizer based on the specified tokenizer and language
263
287
  global current_tokenizer
264
288
  current_tokenizer = tokenizer
265
- init_tokenizer(current_tokenizer, language)
289
+ init_tokenizer(current_tokenizer, language, debug)
266
290
 
267
- buffer = '' # Buffer to hold incoming text for processing
268
- is_first_sentence = True # Flag to identify the first sentence
291
+ buffer = ""
292
+ is_first_sentence = True
269
293
  word_count = 0 # Initialize word count
270
294
  last_delimiter_position = -1 # Position of last full sentence delimiter
271
295
 
@@ -276,8 +300,7 @@ def generate_sentences(generator: Iterator[str],
276
300
  if quick_yield_for_all_sentences:
277
301
  quick_yield_single_sentence_fragment = True
278
302
 
279
- # Process each character from the generator
280
- for char in _generate_characters(generator, log_characters):
303
+ async for char in _generate_characters(generator, log_characters):
281
304
 
282
305
  if char:
283
306
  buffer += char
@@ -287,31 +310,39 @@ def generate_sentences(generator: Iterator[str],
287
310
  if char.isspace() or char in sentence_fragment_delimiters:
288
311
  word_count += 1
289
312
 
290
- if debug_mode:
313
+ if debug:
291
314
  print("\033[36mDebug: Added char, buffer size: \"{}\"\033[0m".format(len(buffer)))
292
315
 
293
316
  # Check conditions to yield first sentence fragment quickly
294
- if (is_first_sentence
295
- and len(buffer) > minimum_first_fragment_length
296
- and quick_yield_single_sentence_fragment):
297
-
298
- if (buffer[-1] in sentence_fragment_delimiters
299
- or word_count >= force_first_fragment_after_words):
317
+ if (
318
+ is_first_sentence
319
+ and len(buffer) > minimum_first_fragment_length
320
+ and quick_yield_single_sentence_fragment
321
+ ):
322
+
323
+ if (
324
+ buffer[-1] in sentence_fragment_delimiters
325
+ or word_count >= force_first_fragment_after_words
326
+ ):
300
327
 
301
328
  yield_text = _clean_text(
302
329
  buffer,
303
330
  cleanup_text_links,
304
331
  cleanup_text_emojis)
305
- if debug_mode:
332
+ if debug:
306
333
  print("\033[36mDebug: Yielding first sentence fragment: \"{}\"\033[0m".format(yield_text))
334
+
307
335
  yield yield_text
336
+
308
337
  buffer = ""
309
338
  if not quick_yield_every_fragment:
310
339
  is_first_sentence = False
340
+
311
341
  continue
312
342
 
313
343
  # Continue accumulating characters if buffer is under minimum sentence length
314
344
  if len(buffer) <= minimum_sentence_length + context_size:
345
+
315
346
  continue
316
347
 
317
348
  # Update last delimiter position if a new delimiter is found
@@ -320,14 +351,16 @@ def generate_sentences(generator: Iterator[str],
320
351
 
321
352
  # Define context window for checking potential sentence boundaries
322
353
  context_window_end_pos = len(buffer) - context_size - 1
323
- context_window_start_pos = context_window_end_pos - context_size_look_overhead
354
+ context_window_start_pos = (
355
+ context_window_end_pos - context_size_look_overhead
356
+ )
324
357
  if context_window_start_pos < 0:
325
358
  context_window_start_pos = 0
326
359
 
327
360
  # Tokenize sentences from buffer
328
361
  sentences = _tokenize_sentences(buffer, tokenize_sentences)
329
362
 
330
- if debug_mode:
363
+ if debug:
331
364
  print("\033[36mbuffer: \"{}\"\033[0m".format(buffer))
332
365
  print("\033[36mlast_delimiter_position: {}\033[0m".format(last_delimiter_position))
333
366
  print("\033[36mlen(sentences) > 2: {}\033[0m".format(len(sentences) > 2))
@@ -335,22 +368,28 @@ def generate_sentences(generator: Iterator[str],
335
368
  print("\033[36mcontext_window_end_pos: {}\033[0m".format(context_window_end_pos))
336
369
 
337
370
  # Process and yield sentences based on conditions
338
- if (len(sentences) > 2 or
339
- (last_delimiter_position >= 0 and
340
- context_window_start_pos <= last_delimiter_position <= context_window_end_pos)
341
- ):
371
+ if len(sentences) > 2 or (
372
+ last_delimiter_position >= 0
373
+ and context_window_start_pos
374
+ <= last_delimiter_position
375
+ <= context_window_end_pos
376
+ ):
342
377
 
343
378
  if len(sentences) > 1:
344
- total_length_except_last = sum(len(sentence) for sentence in sentences[:-1])
379
+ total_length_except_last = sum(
380
+ len(sentence) for sentence in sentences[:-1]
381
+ )
345
382
  if total_length_except_last >= minimum_sentence_length:
346
383
  for sentence in sentences[:-1]:
347
384
  yield_text = _clean_text(
348
385
  sentence,
349
386
  cleanup_text_links,
350
387
  cleanup_text_emojis)
351
- if debug_mode:
388
+ if debug:
352
389
  print("\033[36mDebug: Yielding sentence: \"{}\"\033[0m".format(yield_text))
390
+
353
391
  yield yield_text
392
+
354
393
  if quick_yield_for_all_sentences:
355
394
  is_first_sentence = True
356
395
 
@@ -380,13 +419,15 @@ def generate_sentences(generator: Iterator[str],
380
419
  sentence_buffer += sentence
381
420
  if len(sentence_buffer) < minimum_sentence_length:
382
421
  sentence_buffer += " "
422
+
383
423
  continue
384
- if debug_mode:
424
+
425
+ if debug:
385
426
  print("\033[36mDebug: Yielding final sentence(s): \"{}\"\033[0m".format(yield_text))
386
427
  yield_text = _clean_text(
387
- sentence_buffer,
388
- cleanup_text_links,
389
- cleanup_text_emojis)
428
+ sentence_buffer, cleanup_text_links, cleanup_text_emojis
429
+ )
430
+
390
431
  yield yield_text
391
432
 
392
433
  sentence_buffer = ""
@@ -396,6 +437,46 @@ def generate_sentences(generator: Iterator[str],
396
437
  sentence_buffer,
397
438
  cleanup_text_links,
398
439
  cleanup_text_emojis)
399
- if debug_mode:
440
+ if debug:
400
441
  print("\033[36mDebug: Yielding remaining text: \"{}\"\033[0m".format(yield_text))
442
+
401
443
  yield yield_text
444
+
445
+
446
+ def _await_sync(f: Awaitable[str]) -> str:
447
+ gen = f.__await__()
448
+ try:
449
+ next(gen)
450
+ raise RuntimeError(f"{f} failed to be synchronous")
451
+ except StopIteration as e:
452
+ return e.value
453
+
454
+
455
+ def _async_iter_to_sync(f: AsyncIterator[str]) -> Iterator[str]:
456
+ try:
457
+ while True:
458
+ yield _await_sync(f.__anext__())
459
+ except StopAsyncIteration:
460
+ return
461
+
462
+
463
+ P = ParamSpec("P")
464
+
465
+
466
+ def _dowrap(
467
+ f: Callable[Concatenate[AsyncIterable[str], P], AsyncIterator[str]]
468
+ ) -> Callable[Concatenate[Iterable[str], P], Iterator[str]]:
469
+ @functools.wraps(f)
470
+ def inner(generator: Iterable[str], *args: P.args, **kwargs: P.kwargs):
471
+ async def gen_wrap():
472
+ for x in generator:
473
+ yield x
474
+
475
+ return _async_iter_to_sync(f(gen_wrap(), *args, **kwargs))
476
+
477
+ return inner
478
+
479
+
480
+ generate_sentences = _dowrap(generate_sentences_async)
481
+ generate_sentences.__name__ = "generate_sentences"
482
+ generate_sentences.__qualname__ = "generate_sentences"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: stream2sentence
3
- Version: 0.2.6
3
+ Version: 0.2.7
4
4
  Summary: Real-time processing and delivery of sentences from a continuous stream of characters or text chunks.
5
5
  Home-page: https://github.com/KoljaB/stream2sentence
6
6
  Author: Kolja Beigel
@@ -1 +0,0 @@
1
- from .stream2sentence import generate_sentences, init_tokenizer