stream2sentence 0.3.0__tar.gz → 0.3.2__tar.gz

{stream2sentence-0.3.0 → stream2sentence-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: stream2sentence
-Version: 0.3.0
+Version: 0.3.2
 Summary: Real-time processing and delivery of sentences from a continuous stream of characters or text chunks.
 Home-page: https://github.com/KoljaB/stream2sentence
 Author: Kolja Beigel
@@ -11,9 +11,19 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
-Requires-Dist: nltk==3.9.1
-Requires-Dist: emoji==2.14.0
-Requires-Dist: stanza==1.9.2
+Requires-Dist: nltk==3.9.4
+Requires-Dist: emoji==2.15.0
+Requires-Dist: stanza==1.11.1
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 # Real-Time Sentence Detection
 
@@ -170,6 +180,51 @@ These parameters control how quickly and frequently the generator yields sentenc
   - Ensures timely output even with long opening sentences.
   - Default: 15 words
 
+
+## Time based strategy
+Instead of a purely lexigraphical strategy, a time based strategy is available.
+A target tokens per second (tps) is input, and generate_sentences will yield the best
+available output (full sentence, longest fragment, or any available buffer, in that order) if it is approaching a "deadline"
+where what has been output would be slower than the input tps target. If LLM is more than
+two full sentences ahead of the target it will output a sentence even if it's ahead of the "deadline"
+
+`from stream2sentence.stream2sentence_time_based import generate_sentences_time_based`
+
+### Parameters
+- `generator (Iterator[str])`
+  - A generator that yields chunks of text as a stream of characters.`
+- `target_tps: float = 4`
+  - the rate in tokens per second you want to use to calculate deadlines for output.
+  - Default is 4. (approximately the speed of human speech)
+- `lead_time: float = 1`
+  - amount of time in seconds to wait for the buffer to build for before returning values.
+- `max_wait_for_fragments = [3, 2]`
+  - Max amount of time in seconds that the Nth sentence will wait beyond the "deadline" for a "fragment" (text preceeding a fragment delimiter), which is preferred over a piece of buffer.
+  - The last value in the array is used for all subsequent checks.
+- `min_output_lengths: int[] = [2, 3, 3, 4]`
+  - An array that corresponds to the minimum output size in words for the corresponding output sentence, the last value in the array is used for all remaining output.
+  - For example [4,5,6] would mean the first piece of output must have 4 words, the second 5 words, and all subsequent 6.
+- `preferred_sentence_fragment_delimiters: str[] = ['. ', '? ', '! ', '\n']`
+  - Array of strings that deliniate a sentence fragment. "Preferred" are checked first and always used if the fragment meets the length requirement over the other fragment delimiters.
+  - Note the trailing spaces, added to differentiate between values like $3.5 and a proper sentence end
+- `sentence_fragment_delimiters: str[] = ['; ', ': ', ', ', '* ', '**', '– ']`
+  - Array of strings that are checked after "preferred" delimiters
+- `delimiter_ignore_prefixes: str[]` 
+  - Array of strings that will not be considered "delimiters" if preceeded by a delimiter.
+  - Used to ignore common abbreviations for things like Mr. Dr. and Mrs. where we don't want to split
+  - Default is a long list documented in delimiter_ignore_prefixes
+- `wait_for_if_non_fragment: str[]`
+  - Array of strings that the algorithm will not use as the last value if the whole buffer is being output (not a fragment or sentence).
+  - Avoids awkward pauses on common words that are unnatural to pause at.
+  - Default is a long list of common words documented in avoid_pause_words.py
+- `deadline_offsets_static: float[] = [1]`
+  - Constant amount of time in seconds to subtract from the deadline for first n sentences. 
+  - Last value applied to all subsequent sentences
+- `deadline_offsets_dynamic: float[] = [0]`: 
+  - Added to account for the time it takes a TTS engine to generate output. 
+  - For example, if it takes your TTS engine around 1 second to generate 10 words, you can use a value of 0.1 so that the TTS generation time is included in the deadline. 
+  - Applied to first n sentences, last value applied to all subsequent
+
 ## Contributing
 
 Any Contributions you make are welcome and **greatly appreciated**.

{stream2sentence-0.3.0 → stream2sentence-0.3.2}/README.md

@@ -153,6 +153,51 @@ These parameters control how quickly and frequently the generator yields sentenc
   - Ensures timely output even with long opening sentences.
   - Default: 15 words
 
+
+## Time based strategy
+Instead of a purely lexigraphical strategy, a time based strategy is available.
+A target tokens per second (tps) is input, and generate_sentences will yield the best
+available output (full sentence, longest fragment, or any available buffer, in that order) if it is approaching a "deadline"
+where what has been output would be slower than the input tps target. If LLM is more than
+two full sentences ahead of the target it will output a sentence even if it's ahead of the "deadline"
+
+`from stream2sentence.stream2sentence_time_based import generate_sentences_time_based`
+
+### Parameters
+- `generator (Iterator[str])`
+  - A generator that yields chunks of text as a stream of characters.`
+- `target_tps: float = 4`
+  - the rate in tokens per second you want to use to calculate deadlines for output.
+  - Default is 4. (approximately the speed of human speech)
+- `lead_time: float = 1`
+  - amount of time in seconds to wait for the buffer to build for before returning values.
+- `max_wait_for_fragments = [3, 2]`
+  - Max amount of time in seconds that the Nth sentence will wait beyond the "deadline" for a "fragment" (text preceeding a fragment delimiter), which is preferred over a piece of buffer.
+  - The last value in the array is used for all subsequent checks.
+- `min_output_lengths: int[] = [2, 3, 3, 4]`
+  - An array that corresponds to the minimum output size in words for the corresponding output sentence, the last value in the array is used for all remaining output.
+  - For example [4,5,6] would mean the first piece of output must have 4 words, the second 5 words, and all subsequent 6.
+- `preferred_sentence_fragment_delimiters: str[] = ['. ', '? ', '! ', '\n']`
+  - Array of strings that deliniate a sentence fragment. "Preferred" are checked first and always used if the fragment meets the length requirement over the other fragment delimiters.
+  - Note the trailing spaces, added to differentiate between values like $3.5 and a proper sentence end
+- `sentence_fragment_delimiters: str[] = ['; ', ': ', ', ', '* ', '**', '– ']`
+  - Array of strings that are checked after "preferred" delimiters
+- `delimiter_ignore_prefixes: str[]` 
+  - Array of strings that will not be considered "delimiters" if preceeded by a delimiter.
+  - Used to ignore common abbreviations for things like Mr. Dr. and Mrs. where we don't want to split
+  - Default is a long list documented in delimiter_ignore_prefixes
+- `wait_for_if_non_fragment: str[]`
+  - Array of strings that the algorithm will not use as the last value if the whole buffer is being output (not a fragment or sentence).
+  - Avoids awkward pauses on common words that are unnatural to pause at.
+  - Default is a long list of common words documented in avoid_pause_words.py
+- `deadline_offsets_static: float[] = [1]`
+  - Constant amount of time in seconds to subtract from the deadline for first n sentences. 
+  - Last value applied to all subsequent sentences
+- `deadline_offsets_dynamic: float[] = [0]`: 
+  - Added to account for the time it takes a TTS engine to generate output. 
+  - For example, if it takes your TTS engine around 1 second to generate 10 words, you can use a value of 0.1 so that the TTS generation time is included in the deadline. 
+  - Applied to first n sentences, last value applied to all subsequent
+
 ## Contributing
 
 Any Contributions you make are welcome and **greatly appreciated**.

{stream2sentence-0.3.0 → stream2sentence-0.3.2}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="stream2sentence", 
-    version="0.3.0",
+    version="0.3.2",
     author="Kolja Beigel",
     author_email="kolja.beigel@web.de",
     description="Real-time processing and delivery of sentences from a continuous stream of characters or text chunks.",
@@ -20,9 +20,9 @@ setuptools.setup(
     ],
     python_requires='>=3.6',
     install_requires=[
-        'nltk==3.9.1',
-        'emoji==2.14.0',
-        'stanza==1.9.2'
+        'nltk==3.9.4',
+        'emoji==2.15.0',
+        'stanza==1.11.1'
     ],
     keywords='realtime, text streaming, stream, sentence, sentence detection, sentence generation, tts, speech synthesis, nltk, text analysis, audio processing, boundary detection, sentence boundary detection'
 )

stream2sentence-0.3.2/stream2sentence/__init__.py

@@ -0,0 +1,14 @@
+from .stream2sentence import (
+    generate_sentences,
+    generate_sentences_async,
+    init_tokenizer,
+    SentenceSplitter,
+)
+
+from .stream2sentence_time_based import (
+    generate_sentences_time_based,
+)
+
+from .avoid_pause_words import (
+    AVOID_PAUSE_WORDS,
+)

stream2sentence-0.3.2/stream2sentence/avoid_pause_words.py

@@ -0,0 +1,65 @@
+
+
+# Conjunctions
+conjunctions = ["and", "or", "but", "so", "for", "nor", "yet"]
+
+# Prepositions
+prepositions = [
+    "in", "on", "at", "by", "with", "about", "of", "to", "for", 
+    "from", "as", "over", "under", "through", "between", "during", "there"
+]
+
+# Articles
+articles = ["a", "an", "the"]
+
+# Possessives and Demonstratives
+possessives_and_demonstratives = [
+    "my", "your", "his", "her", "its", "our", "their", 
+    "this", "that", "these", "those"
+]
+
+# Auxiliary/Helping Verbs
+auxiliary_verbs = [
+    "is", "are", "was", "were", "am", "be", "been", "being",
+    "do", "does", "did", "have", "has", "had", 
+    "can", "could", "shall", "should", 
+    "will", "would", "may", "might", "must"
+]
+
+# Pronouns
+pronouns = [
+    "I", "we", "you", "he", "she", "it", "they", 
+    "who", "whom", "whose", "which", "that"
+]
+
+# Quantifiers
+quantifiers = ["some", "many", "few", "all", "any", "most", "much", "none", "several"]
+
+# Adverbs that Modify Flow
+adverbs = ["very", "too", "just", "quite", "almost", "nearly", "only"]
+
+# Interrogatives
+interrogatives = ["what", "where", "when", "why", "how"]
+
+# Relative Pronouns
+relative_pronouns = ["who", "which", "that"]
+
+# Subordinating Conjunctions
+subordinating_conjunctions = [
+    "although", "because", "if", "since", 
+    "though", "while", "until", "unless"
+]
+
+AVOID_PAUSE_WORDS = set(
+  conjunctions + 
+  prepositions + 
+  articles + 
+  possessives_and_demonstratives + 
+  auxiliary_verbs + 
+  pronouns + 
+  quantifiers + 
+  adverbs + 
+  interrogatives + 
+  subordinating_conjunctions
+)
+

stream2sentence-0.3.2/stream2sentence/delimiter_ignore_prefixes.py

@@ -0,0 +1,34 @@
+
+titles_and_abbreviations = [
+    "Mr.", "Mrs.", "Ms.", "Dr.", "Prof.", "Rev.", "St.",
+    "Ph.D.", "Phd.", "PhD.", "M.D.", "B.A.", "M.A.", "D.D.S.", "J.D.",
+    "Co.", "Corp.", "Ave.", "Blvd.", "Rd.", "Mt.",
+    "a.m.", "p.m.", "Jr.", "Sr.",
+    "Gov.", "Gen.", "Capt.", "Lt.", "Maj.", "Col.", "Adm.", "Cmdr.",
+    "Sgt.", "Cpl.", "Pvt.", "U.S.", "U.K.", "vs.", "i.e.", "e.g.",
+    "Vol.", "Art.", "Sec.", "Chap.", "Fig.", "Ref.", "Dept."
+]
+
+dates_and_times = [
+    "Jan.", "Feb.", "Mar.", "Apr.", "Jun.", "Jul.", "Aug.",
+    "Sep.", "Oct.", "Nov.", "Dec.",
+    "Mon.", "Tue.", "Wed.", "Thu.", "Fri.", "Sat.", "Sun.",
+]
+
+financial_abbreviations = [
+    "Inc.", "Ltd.", "Corp.", "PLC.", "LLC.", "LLP.",
+    "P/E.", "EPS.", "NAV.", "ROI.", "ROA.", "ROE.",
+]
+
+country_abbreviations = [
+    "U.S.A.", "U.K.", "U.A.E.", "P.R.C.", "D.R.C.", "R.O.C.", 
+    "E.U.", "U.N.", "A.U.",
+    "U.S.", "U.K.", "E.U.", "P.R.C.", "D.R.C.", "R.O.C.",
+]
+
+DELIMITER_IGNORE_PREFIXES = set(
+    titles_and_abbreviations + dates_and_times +
+    financial_abbreviations + country_abbreviations
+)
+
+