livekit-plugins-nltk 0.5.dev0__tar.gz → 0.6.0__tar.gz

{livekit_plugins_nltk-0.5.dev0 → livekit_plugins_nltk-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: livekit-plugins-nltk
-Version: 0.5.dev0
+Version: 0.6.0
 Summary: Agent Framework plugin for NLTK-based text processing.
 Home-page: https://github.com/livekit/agents
 License: Apache-2.0
@@ -20,7 +20,7 @@ Requires-Python: >=3.9.0
 Description-Content-Type: text/markdown
 Requires-Dist: livekit~=0.11
 Requires-Dist: nltk<4,>=3
-Requires-Dist: livekit-agents~=0.6.dev0
+Requires-Dist: livekit-agents~=0.7.0
 
 # LiveKit Plugins NLTK
 

{livekit_plugins_nltk-0.5.dev0 → livekit_plugins_nltk-0.6.0}/livekit/plugins/nltk/__init__.py

@@ -13,12 +13,11 @@
 # limitations under the License.
 
 
-from .sentence_tokenizer import SentenceStream, SentenceTokenizer
+from .sentence_tokenizer import SentenceTokenizer
 from .version import __version__
 
 __all__ = [
     "SentenceTokenizer",
-    "SentenceStream",
     "__version__",
 ]
 

livekit_plugins_nltk-0.6.0/livekit/plugins/nltk/sentence_tokenizer.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import dataclasses
+import functools
+from dataclasses import dataclass
+
+from livekit import agents
+
+import nltk  # type: ignore
+
+# nltk is using the punkt tokenizer
+# https://www.nltk.org/_modules/nltk/tokenize/punkt.html
+# this code is using a whitespace to concatenate small sentences together
+# (languages such as Chinese and Japanese are not yet supported)
+
+
+@dataclass
+class _TokenizerOptions:
+    language: str
+    min_sentence_len: int
+    stream_context_len: int
+
+
+class SentenceTokenizer(agents.tokenize.SentenceTokenizer):
+    def __init__(
+        self,
+        *,
+        language: str = "english",
+        min_sentence_len: int = 20,
+        stream_context_len: int = 10,
+    ) -> None:
+        super().__init__()
+        self._config = _TokenizerOptions(
+            language=language,
+            min_sentence_len=min_sentence_len,
+            stream_context_len=stream_context_len,
+        )
+
+    def _sanitize_options(self, language: str | None = None) -> _TokenizerOptions:
+        config = dataclasses.replace(self._config)
+        if language:
+            config.language = language
+        return config
+
+    def tokenize(self, *, text: str, language: str | None = None) -> list[str]:
+        config = self._sanitize_options(language=language)
+        sentences = nltk.tokenize.sent_tokenize(text, config.language)
+        new_sentences = []
+        buff = ""
+        for sentence in sentences:
+            buff += sentence + " "
+            if len(buff) - 1 >= config.min_sentence_len:
+                new_sentences.append(buff.rstrip())
+                buff = ""
+
+        if buff:
+            new_sentences.append(buff.rstrip())
+
+        return new_sentences
+
+    def stream(
+        self,
+        *,
+        language: str | None = None,
+    ) -> agents.tokenize.SentenceStream:
+        config = self._sanitize_options(language=language)
+        return agents.tokenize.BufferedTokenStream(
+            tokenizer=functools.partial(
+                nltk.tokenize.sent_tokenize,
+                language=config.language,
+            ),
+            min_token_len=self._config.min_sentence_len,
+            ctx_len=self._config.stream_context_len,
+        )

{livekit_plugins_nltk-0.5.dev0 → livekit_plugins_nltk-0.6.0}/livekit/plugins/nltk/version.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "0.5.dev0"
+__version__ = "0.6.0"

{livekit_plugins_nltk-0.5.dev0 → livekit_plugins_nltk-0.6.0}/livekit_plugins_nltk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: livekit-plugins-nltk
-Version: 0.5.dev0
+Version: 0.6.0
 Summary: Agent Framework plugin for NLTK-based text processing.
 Home-page: https://github.com/livekit/agents
 License: Apache-2.0
@@ -20,7 +20,7 @@ Requires-Python: >=3.9.0
 Description-Content-Type: text/markdown
 Requires-Dist: livekit~=0.11
 Requires-Dist: nltk<4,>=3
-Requires-Dist: livekit-agents~=0.6.dev0
+Requires-Dist: livekit-agents~=0.7.0
 
 # LiveKit Plugins NLTK
 

{livekit_plugins_nltk-0.5.dev0 → livekit_plugins_nltk-0.6.0}/livekit_plugins_nltk.egg-info/requires.txt

@@ -1,3 +1,3 @@
 livekit~=0.11
 nltk<4,>=3
-livekit-agents~=0.6.dev0
+livekit-agents~=0.7.0

{livekit_plugins_nltk-0.5.dev0 → livekit_plugins_nltk-0.6.0}/setup.py

@@ -49,7 +49,7 @@ setuptools.setup(
     install_requires=[
         "livekit~=0.11",
         "nltk >= 3, < 4",
-        "livekit-agents~=0.6.dev0",
+        "livekit-agents~=0.7.0",
     ],
     package_data={
         "livekit.plugins.nltk": ["py.typed"],

livekit_plugins_nltk-0.5.dev0/livekit/plugins/nltk/sentence_tokenizer.py

@@ -1,143 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import dataclasses
-from dataclasses import dataclass
-from typing import List, Optional
-
-from livekit import agents
-
-import nltk  # type: ignore
-
-from .log import logger
-
-# nltk is using the punkt tokenizer
-# https://www.nltk.org/_modules/nltk/tokenize/punkt.html
-# this code is using a whitespace to concatenate small sentences together
-# (languages such as Chinese and Japanese are not yet supported)
-
-
-@dataclass
-class TokenizerOptions:
-    language: str
-    min_sentence_len: int
-    stream_context_len: int
-
-
-class SentenceTokenizer(agents.tokenize.SentenceTokenizer):
-    def __init__(
-        self,
-        language: str = "english",
-        min_sentence_len: int = 20,
-        stream_context_len: int = 10,
-    ) -> None:
-        super().__init__()
-        self._config = TokenizerOptions(
-            language=language,
-            min_sentence_len=min_sentence_len,
-            stream_context_len=stream_context_len,
-        )
-
-    def _sanitize_options(self, language: Optional[str] = None) -> TokenizerOptions:
-        config = dataclasses.replace(self._config)
-        if language:
-            config.language = language
-        return config
-
-    def tokenize(
-        self, *, text: str, language: Optional[str] = None
-    ) -> List[agents.tokenize.SegmentedSentence]:
-        config = self._sanitize_options(language=language)
-        sentences = nltk.tokenize.sent_tokenize(text, config.language)
-        new_sentences = []
-        buff = ""
-        for sentence in sentences:
-            buff += sentence + " "
-            if len(buff) - 1 >= config.min_sentence_len:
-                new_sentences.append(buff.rstrip())
-                buff = ""
-
-        if buff:
-            new_sentences.append(buff.rstrip())
-
-        return [agents.tokenize.SegmentedSentence(text=text) for text in new_sentences]
-
-    def stream(
-        self,
-        *,
-        language: Optional[str] = None,
-    ) -> agents.tokenize.SentenceStream:
-        config = self._sanitize_options(language=language)
-        return SentenceStream(
-            language=config.language,
-            min_sentence_len=config.min_sentence_len,
-            context_len=config.stream_context_len,
-        )
-
-
-class SentenceStream(agents.tokenize.SentenceStream):
-    def __init__(
-        self, *, language: str, min_sentence_len: int, context_len: int
-    ) -> None:
-        self._language = language
-        self._context_len = context_len
-        self._min_sentence_len = min_sentence_len
-        self._event_queue = asyncio.Queue[agents.tokenize.SegmentedSentence | None]()
-        self._closed = False
-
-        self._incomplete_sentences: List[str] = []  # <= min_sentence_len
-        self._buffer = ""
-
-    def push_text(self, text: str) -> None:
-        if self._closed:
-            logger.error("Cannot push text to closed stream")
-            return
-
-        for char in text:
-            self._buffer += char
-
-            if len(self._buffer) < self._context_len:
-                continue
-
-            sentences = nltk.tokenize.sent_tokenize(self._buffer, self._language)
-            if len(sentences) < 2:
-                continue
-
-            new_sentence = sentences[0]
-            self._incomplete_sentences.append(new_sentence)
-            s = " ".join(self._incomplete_sentences)
-
-            if len(s) >= self._min_sentence_len:
-                self._event_queue.put_nowait(agents.tokenize.SegmentedSentence(text=s))
-                self._incomplete_sentences = []
-
-            self._buffer = self._buffer[len(new_sentence) :].lstrip()
-
-    async def flush(self) -> None:
-        # try to segment the remaining data inside self._text_buffer
-        buff = " ".join(self._incomplete_sentences)
-        sentences = nltk.tokenize.sent_tokenize(self._buffer, self._language)
-        for sentence in sentences:
-            buff += " " + sentence
-            if len(buff) >= self._min_sentence_len:
-                await self._event_queue.put(
-                    agents.tokenize.SegmentedSentence(text=buff)
-                )
-                buff = ""
-
-        if buff:
-            await self._event_queue.put(agents.tokenize.SegmentedSentence(text=buff))
-
-    async def aclose(self) -> None:
-        self._closed = True
-        self._event_queue.put_nowait(None)
-
-    async def __anext__(self) -> agents.tokenize.SegmentedSentence:
-        event = await self._event_queue.get()
-        if event is None:
-            raise StopAsyncIteration
-
-        return event
-
-    def __aiter__(self) -> "SentenceStream":
-        return self