stream2sentence 0.1.1__py3-none-any.whl

stream2sentence/__init__.py

@@ -0,0 +1 @@
+from .stream2sentence import generate_sentences

stream2sentence/stream2sentence.py

@@ -0,0 +1,161 @@
+"""
+Real-time processing and delivery of sentences from a continuous stream of characters or text chunks
+"""
+
+import re
+import nltk
+from typing import Iterator
+
+def initialize_nltk():
+    """
+    Initializes NLTK by downloading required data for sentence tokenization.
+    """
+    try:
+        _ = nltk.data.find('tokenizers/punkt') 
+    except LookupError:
+        nltk.download('punkt')
+
+initialize_nltk()
+
+def _remove_links(text: str) -> str:
+    """
+    Removes any links from the input text.
+
+    Args:
+        text (str): Input text
+
+    Returns:
+        str: Text with links removed
+    """
+    pattern = r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
+    return re.sub(pattern, '', text)
+
+def _remove_emojis(text: str) -> str:
+    """
+    Removes emojis from the input text.
+
+    Args:
+        text (str): Input text
+    
+    Returns:
+        str: Text with emojis removed
+    """
+    emoji_pattern = re.compile("["
+                               u"\U0001F600-\U0001F64F" 
+                               u"\U0001F300-\U0001F5FF"
+                               u"\U0001F680-\U0001F6FF" 
+                               u"\U0001F1E0-\U0001F1FF"
+                               u"\U00002702-\U000027B0"  
+                               u"\U000024C2-\U0001F251"
+                               "]+", flags=re.UNICODE)
+    return emoji_pattern.sub(r'', text)
+
+def _generate_characters(generator: Iterator[str], 
+                         log_characters: bool = False) -> Iterator[str]:
+    """
+    Generates individual characters from a text generator.
+
+    Args:
+        generator (Iterator[str]): Input text generator
+        log_characters (bool): Whether to log the characters to the console
+
+    Yields:
+        Individual characters from the generator
+    """
+    for chunk in generator:
+        for char in chunk:
+            if log_characters:
+                print(char, end="", flush=True)
+            yield char
+            
+def _clean_text(text: str, 
+               cleanup_text_links: bool = False, 
+               cleanup_text_emojis: bool = False) -> str:
+    """
+    Cleans the text by removing links and emojis.
+
+    Args:
+        text (str): Input text
+        cleanup_text_links (boolean, optional): Remove non-desired links from the stream.
+        cleanup_text_emojis (boolean, optional): Remove non-desired emojis from the stream. 
+    
+    Returns:
+        str: Cleaned text
+    """
+    if cleanup_text_links:
+        text = _remove_links(text)
+    if cleanup_text_emojis:
+        text = _remove_emojis(text)
+
+    text = text.strip()
+    return text
+
+def generate_sentences(generator: Iterator[str],  
+                       context_size: int = 20,
+                       minimum_sentence_length: int = 1,
+                       quick_yield_single_sentence_fragment: bool = False,
+                       cleanup_text_links: bool = False,
+                       cleanup_text_emojis: bool = False, 
+                       log_characters: bool = False) -> Iterator[str]:
+    """
+    Generates sentences from a stream of characters or input chunks.
+
+    Args:
+        generator (Iterator[str]): Input character generator
+        context_size (int): The character context size for sentence detection. A larger context ensures more reliable sentence detection. Default value is 10 characters.
+        minimum_sentence_length (int): Minimum character length for a sentence. Short sentences will combine with the subsequent sentence to enhance synthesis quality. Default value is 8 characters.
+        quick_yield_single_sentence_fragment (bool): Whether to return a sentence fragment as fast as possible (for realtime speech synthesis)
+        cleanup_text_links (boolean, optional): Remove non-desired links from the stream.
+        cleanup_text_emojis (boolean, optional): Remove non-desired emojis from the stream. 
+        log_characters (bool): Whether to log the characters to the console
+
+    Yields:
+        Iterator[str]: Sentences based on input characters  
+    """
+    
+    buffer = ''
+    is_first_sentence = True
+
+    sentence_delimiters = '.?!;:-,\n…)]}'
+
+    for char in _generate_characters(generator, log_characters):
+
+        if char:
+            buffer += char
+
+            if is_first_sentence and len(buffer) > 1 and quick_yield_single_sentence_fragment:
+
+                if buffer[-1] in sentence_delimiters:
+                    yield _clean_text(buffer, cleanup_text_links, cleanup_text_emojis)
+                    buffer = ""
+                    is_first_sentence = False
+                    continue
+
+            # Check if minimum length reached
+            if len(buffer) <= minimum_sentence_length + context_size:
+                continue
+
+            # Potential delimiter character has to be a bit away from the end of the buffer
+            # For reliable sentence detection the engine needs enough context to work with
+            delimiter_char = buffer[-context_size]
+
+            if delimiter_char in sentence_delimiters:
+                
+                sentences = nltk.tokenize.sent_tokenize(buffer)
+                if len(sentences) > 1:
+                    if len(sentences[0]) == len(buffer) - context_size + 1:
+                        yield _clean_text(buffer[:-context_size + 1], cleanup_text_links, cleanup_text_emojis)
+                        buffer = buffer[-context_size + 1:]
+                        is_first_sentence = False            
+
+    # Yield remaining buffer
+    if buffer:
+        sentences = nltk.tokenize.sent_tokenize(buffer)
+        sentence_buffer = ""
+        for sentence in sentences:
+            sentence_buffer += sentence
+            if len(sentence_buffer) < minimum_sentence_length:
+                sentence_buffer += " "
+                continue
+            yield _clean_text(sentence_buffer, cleanup_text_links, cleanup_text_emojis)
+            sentence_buffer = ""

stream2sentence-0.1.1.dist-info/METADATA

@@ -0,0 +1,121 @@
+Metadata-Version: 2.1
+Name: stream2sentence
+Version: 0.1.1
+Summary: Real-time processing and delivery of sentences from a continuous stream of characters or text chunks.
+Home-page: https://github.com/KoljaB/stream2sentence
+Author: Kolja Beigel
+Author-email: kolja.beigel@web.de
+Keywords: realtime,text streaming,stream,sentence,sentence detection,sentence generation,tts,speech synthesis,nltk,text analysis,audio processing,boundary detection,sentence boundary detection
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+Requires-Dist: nltk >=3.6
+
+# Real-Time Sentence Detection
+
+Real-time processing and delivery of sentences from a continuous stream of characters or text chunks.
+
+## Table of Contents
+
+- [Features](#features)
+- [Installation](#installation)
+- [Usage](#usage)
+- [Configuration](#configuration)
+- [Contributing](#contributing)
+- [License](#license)
+
+## Features
+
+- Generates sentences from a stream of text in real-time.
+- Customizable to finetune/balance speed vs reliability.
+- Option to clean the output by removing links and emojis from the detected sentences.
+- Easy to configure and integrate.
+
+## Installation
+
+```bash
+pip install stream2sentence
+```
+
+## Usage
+
+Pass a generator of characters or text chunks to `generate_sentences()` to get a generator of sentences in return.
+
+Here's a basic example:
+
+```python
+from stream2sentence import generate_sentences
+
+# Dummy generator for demonstration
+def dummy_generator():
+    yield "This is a sentence. And here's another! Yet, "
+    yield "there's more. This ends now."
+
+for sentence in generate_sentences(dummy_generator()):
+    print(sentence)
+```
+
+This will output:
+```
+This is a sentence.
+And here's another!
+Yet, there's more.
+This ends now.
+```
+
+One main use case of this library is enable fast text to speech synthesis in the context of character feeds generated from large language models: this library enables fastest possible access to a complete sentence or sentence fragment (using the quick_yield_single_sentence_fragment flag) that then can be synthesized in realtime. The usage of this is demonstrated in the test_stream_from_llm.py file in the tests directory.
+
+## Configuration
+
+The `generate_sentences()` function has the following parameters:
+
+- `generator`: Input character generator.
+  Iterator that emits chunks of text. These chunks can be of any size, and they'll be processed one by one to extract sentences from them. It forms the primary source from which the function reads and generates sentences.
+
+- `quick_yield_single_sentence_fragment`: Whether to return a sentence fragment as fast as possible.  
+  This is a feature for realtime speech synthesis. In some use cases you want to audio stream a minimal chunk of text as fast as possible, even when it means to synthesize mid-sentence. In this case you set this flag to True which will yield a synthesizable sentence fragment as early as possible.
+
+- `context_size`: Context size for sentence detection.  
+  This controls how much context is looked at to detect sentence boundaries. It determines the number of characters around a potential delimiter (like a period) that are considered when detecting sentence boundaries. A larger context size allows more reliable sentence boundary detection, but requires buffering more characters before emitting a sentence.  
+  Default is 10 characters. Increasing this can help detect sentences more accurately, at the cost of added latency.
+
+- `minimum_sentence_length`: Minimum length of a sentence to be detected.  
+  Specifies the minimum number of characters a chunk of text should have before it's considered a potential sentence. This ensures that very short sequences of characters are not mistakenly identified as sentences.Shorter fragments are ignored and kept in the buffer.  
+  Default is 8 characters. Increasing this avoids emitting very short sentence fragments, at the cost of potentially missing some sentences.
+
+- `quick_yield_single_sentence_fragment`: Yield a sentence fragment quickly for real-time applications.
+  When set to True, this option allows the function to quickly yield a sentence fragment as soon as it identifies a potential sentence delimiter, without waiting for further context. This is useful for applications like real-time speech synthesis where there's a need for immediate feedback even if the entire sentence isn't complete. 
+  Default is False. Set to True for faster but potentially less accurate sentence yields.
+
+- `cleanup_text_links`: Option to remove links from the output sentences.  
+  When set to True, this option enables the function to identify and remove HTTP/HTTPS hyperlinks from the emitted output sentences. This helps clean up the output by avoiding unnecessary links.  
+  Default is False. Set to True if links are not required in the output.
+
+- `cleanup_text_emojis`: Option to remove emojis from the output sentences.  
+  If True, any Unicode emoji characters are identified and removed from the emitted output sentences. This can help to clean up the output.  
+  Default is False. Set to True if emojis are not required in the output.
+
+- `log_characters`: Option to log characters to the console.
+  When enabled, each character processed by the function is printed to the console. This is mainly for debugging purposes to observe the flow of characters through the function.
+  Default is False. Set to True for a visual representation of characters being processed. Example: allows printing llm output to console when using stream2sentence to prepare input generation for text to speech synthesis.
+
+
+## Contributing
+
+Any Contributions you make are welcome and **greatly appreciated**.
+
+1. **Fork** the Project.
+2. **Create** your Feature Branch (`git checkout -b feature/AmazingFeature`).
+3. **Commit** your Changes (`git commit -m 'Add some AmazingFeature'`).
+4. **Push** to the Branch (`git push origin feature/AmazingFeature`).
+5. **Open** a Pull Request.
+
+## License
+
+This project is licensed under the MIT License. For more details, see the [`LICENSE`](LICENSE) file.
+
+---
+
+Project created and maintained by [Kolja Beigel](https://github.com/KoljaB).

stream2sentence-0.1.1.dist-info/RECORD

@@ -0,0 +1,6 @@
+stream2sentence/__init__.py,sha256=_-8GjderFI-gHmS4qyG3hafZ7K_XXa5OqjC6XVTyqy4,47
+stream2sentence/stream2sentence.py,sha256=TYTASkFRTdtLsZe5dX4eJlBQ47qnQ7J-SO-4JOht6eg,6052
+stream2sentence-0.1.1.dist-info/METADATA,sha256=6OtBSMHfwZLP5q59lq3UTKF6fpdd_pIyxdGlc_uFnMA,6398
+stream2sentence-0.1.1.dist-info/WHEEL,sha256=5sUXSg9e4bi7lTLOHcm6QEYwO5TIF1TNbTSVFVjcJcc,92
+stream2sentence-0.1.1.dist-info/top_level.txt,sha256=7RtS4EfUkpV3tMMyUqRgSGDvRm2rPBAYdazTN2z8m1A,16
+stream2sentence-0.1.1.dist-info/RECORD,,

stream2sentence-0.1.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.41.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

stream2sentence-0.1.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+stream2sentence