livekit-plugins-nltk 0.5.dev1__py3-none-any.whl → 0.6.0__py3-none-any.whl

livekit/plugins/nltk/__init__.py

@@ -13,12 +13,11 @@
 # limitations under the License.
 
 
-from .sentence_tokenizer import SentenceStream, SentenceTokenizer
+from .sentence_tokenizer import SentenceTokenizer
 from .version import __version__
 
 __all__ = [
     "SentenceTokenizer",
-    "SentenceStream",
     "__version__",
 ]
 

livekit/plugins/nltk/sentence_tokenizer.py

@@ -1,16 +1,13 @@
 from __future__ import annotations
 
-import asyncio
 import dataclasses
+import functools
 from dataclasses import dataclass
-from typing import List, Optional
 
 from livekit import agents
 
 import nltk  # type: ignore
 
-from .log import logger
-
 # nltk is using the punkt tokenizer
 # https://www.nltk.org/_modules/nltk/tokenize/punkt.html
 # this code is using a whitespace to concatenate small sentences together
@@ -18,7 +15,7 @@ from .log import logger
 
 
 @dataclass
-class TokenizerOptions:
+class _TokenizerOptions:
     language: str
     min_sentence_len: int
     stream_context_len: int
@@ -27,26 +24,25 @@ class TokenizerOptions:
 class SentenceTokenizer(agents.tokenize.SentenceTokenizer):
     def __init__(
         self,
+        *,
         language: str = "english",
         min_sentence_len: int = 20,
         stream_context_len: int = 10,
     ) -> None:
         super().__init__()
-        self._config = TokenizerOptions(
+        self._config = _TokenizerOptions(
             language=language,
             min_sentence_len=min_sentence_len,
             stream_context_len=stream_context_len,
         )
 
-    def _sanitize_options(self, language: Optional[str] = None) -> TokenizerOptions:
+    def _sanitize_options(self, language: str | None = None) -> _TokenizerOptions:
         config = dataclasses.replace(self._config)
         if language:
             config.language = language
         return config
 
-    def tokenize(
-        self, *, text: str, language: Optional[str] = None
-    ) -> List[agents.tokenize.SegmentedSentence]:
+    def tokenize(self, *, text: str, language: str | None = None) -> list[str]:
         config = self._sanitize_options(language=language)
         sentences = nltk.tokenize.sent_tokenize(text, config.language)
         new_sentences = []
@@ -60,84 +56,19 @@ class SentenceTokenizer(agents.tokenize.SentenceTokenizer):
         if buff:
             new_sentences.append(buff.rstrip())
 
-        return [agents.tokenize.SegmentedSentence(text=text) for text in new_sentences]
+        return new_sentences
 
     def stream(
         self,
         *,
-        language: Optional[str] = None,
+        language: str | None = None,
     ) -> agents.tokenize.SentenceStream:
         config = self._sanitize_options(language=language)
-        return SentenceStream(
-            language=config.language,
-            min_sentence_len=config.min_sentence_len,
-            context_len=config.stream_context_len,
+        return agents.tokenize.BufferedTokenStream(
+            tokenizer=functools.partial(
+                nltk.tokenize.sent_tokenize,
+                language=config.language,
+            ),
+            min_token_len=self._config.min_sentence_len,
+            ctx_len=self._config.stream_context_len,
         )
-
-
-class SentenceStream(agents.tokenize.SentenceStream):
-    def __init__(
-        self, *, language: str, min_sentence_len: int, context_len: int
-    ) -> None:
-        self._language = language
-        self._context_len = context_len
-        self._min_sentence_len = min_sentence_len
-        self._event_queue = asyncio.Queue[agents.tokenize.SegmentedSentence | None]()
-        self._closed = False
-
-        self._incomplete_sentences: List[str] = []  # <= min_sentence_len
-        self._buffer = ""
-
-    def push_text(self, text: str) -> None:
-        if self._closed:
-            logger.error("Cannot push text to closed stream")
-            return
-
-        for char in text:
-            self._buffer += char
-
-            if len(self._buffer) < self._context_len:
-                continue
-
-            sentences = nltk.tokenize.sent_tokenize(self._buffer, self._language)
-            if len(sentences) < 2:
-                continue
-
-            new_sentence = sentences[0]
-            self._incomplete_sentences.append(new_sentence)
-            s = " ".join(self._incomplete_sentences)
-
-            if len(s) >= self._min_sentence_len:
-                self._event_queue.put_nowait(agents.tokenize.SegmentedSentence(text=s))
-                self._incomplete_sentences = []
-
-            self._buffer = self._buffer[len(new_sentence) :].lstrip()
-
-    async def flush(self) -> None:
-        # try to segment the remaining data inside self._text_buffer
-        buff = " ".join(self._incomplete_sentences)
-        sentences = nltk.tokenize.sent_tokenize(self._buffer, self._language)
-        for sentence in sentences:
-            buff += " " + sentence
-            if len(buff) >= self._min_sentence_len:
-                await self._event_queue.put(
-                    agents.tokenize.SegmentedSentence(text=buff)
-                )
-                buff = ""
-
-        if buff:
-            await self._event_queue.put(agents.tokenize.SegmentedSentence(text=buff))
-
-    async def aclose(self) -> None:
-        self._closed = True
-        self._event_queue.put_nowait(None)
-
-    async def __anext__(self) -> agents.tokenize.SegmentedSentence:
-        event = await self._event_queue.get()
-        if event is None:
-            raise StopAsyncIteration
-
-        return event
-
-    def __aiter__(self) -> "SentenceStream":
-        return self

livekit/plugins/nltk/version.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "0.5.dev1"
+__version__ = "0.6.0"

{livekit_plugins_nltk-0.5.dev1.dist-info → livekit_plugins_nltk-0.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: livekit-plugins-nltk
-Version: 0.5.dev1
+Version: 0.6.0
 Summary: Agent Framework plugin for NLTK-based text processing.
 Home-page: https://github.com/livekit/agents
 License: Apache-2.0
@@ -20,7 +20,7 @@ Requires-Python: >=3.9.0
 Description-Content-Type: text/markdown
 Requires-Dist: livekit ~=0.11
 Requires-Dist: nltk <4,>=3
-Requires-Dist: livekit-agents ~=0.6.dev1
+Requires-Dist: livekit-agents ~=0.7.0
 
 # LiveKit Plugins NLTK
 

livekit_plugins_nltk-0.6.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+livekit/plugins/nltk/__init__.py,sha256=deM101uCRxPAD7m2xmPkxspW4ozp83jCJpFxoc9bT5U,1096
+livekit/plugins/nltk/log.py,sha256=12Ta0eunqKM3KyIlyzp2QFyO_Y-PBLAtyjEWDiuJdKk,67
+livekit/plugins/nltk/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+livekit/plugins/nltk/sentence_tokenizer.py,sha256=Y2wXJwqqUxYMqSqPrAXn_OIeMQ1Q-18-Nvd-vog8Z40,2211
+livekit/plugins/nltk/version.py,sha256=Z62pORgDetwUvtfZOgPeIzXJugcrpDAOzC876rjCR0o,600
+livekit_plugins_nltk-0.6.0.dist-info/METADATA,sha256=kJNhhnFxOF4rtFIdaWwkpiClxaPRTgjGQURcNLIU-PE,1182
+livekit_plugins_nltk-0.6.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+livekit_plugins_nltk-0.6.0.dist-info/top_level.txt,sha256=OoDok3xUmXbZRvOrfvvXB-Juu4DX79dlq188E19YHoo,8
+livekit_plugins_nltk-0.6.0.dist-info/RECORD,,

livekit_plugins_nltk-0.5.dev1.dist-info/RECORD

@@ -1,9 +0,0 @@
-livekit/plugins/nltk/__init__.py,sha256=HGbaUwK-0cU-SbvbEl2WSQKwNrkHn8XuO-86Hiy0cy4,1134
-livekit/plugins/nltk/log.py,sha256=12Ta0eunqKM3KyIlyzp2QFyO_Y-PBLAtyjEWDiuJdKk,67
-livekit/plugins/nltk/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-livekit/plugins/nltk/sentence_tokenizer.py,sha256=vhdWqMmwJSiqJsK9WibNHkWChiXyn13Yk4jOmfcLEM8,4570
-livekit/plugins/nltk/version.py,sha256=-_fka_7I45q1K43PbmujkpMuAINzB0LjfWFVnQ3gwZg,603
-livekit_plugins_nltk-0.5.dev1.dist-info/METADATA,sha256=1auBYd4W2v8opS_WiMQp-sWNNc8XDw7QVhQL3uPZ6Bc,1188
-livekit_plugins_nltk-0.5.dev1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-livekit_plugins_nltk-0.5.dev1.dist-info/top_level.txt,sha256=OoDok3xUmXbZRvOrfvvXB-Juu4DX79dlq188E19YHoo,8
-livekit_plugins_nltk-0.5.dev1.dist-info/RECORD,,