dv-pipecat-ai 0.0.74.dev770__py3-none-any.whl → 0.0.82.dev776__py3-none-any.whl

pipecat/utils/string.py

@@ -4,44 +4,92 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
-import re
-from typing import Optional, Sequence, Tuple
-
-ENDOFSENTENCE_PATTERN_STR = r"""
-    (?<![A-Z])       # Negative lookbehind: not preceded by an uppercase letter (e.g., "U.S.A.")
-    (?<!\d\.\d)      # Not preceded by a decimal number (e.g., "3.14159")
-    (?<!^\d\.)       # Not preceded by a numbered list item (e.g., "1. Let's start")
-    (?<!\d\s[ap])    # Negative lookbehind: not preceded by time (e.g., "3:00 a.m.")
-    (?<!Mr|Ms|Dr)    # Negative lookbehind: not preceded by Mr, Ms, Dr (combined bc. length is the same)
-    (?<!Mrs)         # Negative lookbehind: not preceded by "Mrs"
-    (?<!Prof)        # Negative lookbehind: not preceded by "Prof"
-    (\.\s*\.\s*\.|[\.\?\!;])|   # Match a period, question mark, exclamation point, or semicolon
-    (\。\s*\。\s*\。|[。？！；।])  # the full-width version (mainly used in East Asian languages such as Chinese, Hindi)
-    $                # End of string
-"""
+"""Text processing utilities for sentence boundary detection and tag parsing.
 
-ENDOFSENTENCE_PATTERN = re.compile(ENDOFSENTENCE_PATTERN_STR, re.VERBOSE)
+This module provides utilities for natural language text processing including
+sentence boundary detection, email and number pattern handling, and XML-style
+tag parsing for structured text content.
 
-EMAIL_PATTERN = re.compile(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}")
+Dependencies:
+    This module uses NLTK (Natural Language Toolkit) for robust sentence
+    tokenization. NLTK is licensed under the Apache License 2.0.
+    See: https://www.nltk.org/
+    Source: https://www.nltk.org/api/nltk.tokenize.punkt.html
+"""
 
-NUMBER_PATTERN = re.compile(r"[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?")
+import re
+from typing import FrozenSet, Optional, Sequence, Tuple
+
+import nltk
+from nltk.tokenize import sent_tokenize
+
+# Ensure punkt_tab tokenizer data is available
+try:
+    nltk.data.find("tokenizers/punkt_tab")
+except LookupError:
+    nltk.download("punkt_tab", quiet=True)
+
+SENTENCE_ENDING_PUNCTUATION: FrozenSet[str] = frozenset(
+    {
+        # Latin script punctuation (most European languages, Filipino, etc.)
+        ".",
+        "!",
+        "?",
+        ";",
+        # East Asian punctuation (Chinese (Traditional & Simplified), Japanese, Korean)
+        "。",  # Ideographic full stop
+        "？",  # Full-width question mark
+        "！",  # Full-width exclamation mark
+        "；",  # Full-width semicolon
+        "．",  # Full-width period
+        "｡",  # Halfwidth ideographic period
+        # Indic scripts punctuation (Hindi, Sanskrit, Marathi, Nepali, Bengali, Tamil, Telugu, Kannada, Malayalam, Gujarati, Punjabi, Oriya, Assamese)
+        "।",  # Devanagari danda (single vertical bar)
+        "॥",  # Devanagari double danda (double vertical bar)
+        # Arabic script punctuation (Arabic, Persian, Urdu, Pashto)
+        "؟",  # Arabic question mark
+        "؛",  # Arabic semicolon
+        "۔",  # Urdu full stop
+        "؏",  # Arabic sign misra (classical texts)
+        # Thai
+        "।",  # Thai uses Devanagari-style punctuation in some contexts
+        # Myanmar/Burmese
+        "၊",  # Myanmar sign little section
+        "။",  # Myanmar sign section
+        # Khmer
+        "។",  # Khmer sign khan
+        "៕",  # Khmer sign bariyoosan
+        # Lao
+        "໌",  # Lao cancellation mark (used as period)
+        "༎",  # Tibetan mark delimiter tsheg bstar (also used in Lao contexts)
+        # Tibetan
+        "།",  # Tibetan mark intersyllabic tsheg
+        "༎",  # Tibetan mark delimiter tsheg bstar
+        # Armenian
+        "։",  # Armenian full stop
+        "՜",  # Armenian exclamation mark
+        "՞",  # Armenian question mark
+        # Ethiopic script (Amharic)
+        "።",  # Ethiopic full stop
+        "፧",  # Ethiopic question mark
+        "፨",  # Ethiopic paragraph separator
+    }
+)
 
 StartEndTags = Tuple[str, str]
 
 
 def replace_match(text: str, match: re.Match, old: str, new: str) -> str:
-    """Replace occurrences of a substring within a matched section of a given
-    text.
+    """Replace occurrences of a substring within a matched section of text.
 
     Args:
-        text (str): The input text in which replacements will be made.
-        match (re.Match): A regex match object representing the section of text to modify.
-        old (str): The substring to be replaced.
-        new (str): The substring to replace `old` with.
+        text: The input text in which replacements will be made.
+        match: A regex match object representing the section of text to modify.
+        old: The substring to be replaced.
+        new: The substring to replace `old` with.
 
     Returns:
-        str: The modified text with the specified replacements made within the matched section.
-
+        The modified text with the specified replacements made within the matched section.
     """
     start = match.start()
     end = match.end()
@@ -51,37 +99,47 @@ def replace_match(text: str, match: re.Match, old: str, new: str) -> str:
 
 
 def match_endofsentence(text: str) -> int:
-    """Finds the position of the end of a sentence in the provided text string.
+    """Find the position of the end of a sentence in the provided text.
 
-    This function processes the input text by replacing periods in email
-    addresses and numbers with ampersands to prevent them from being
-    misidentified as sentence terminals. It then searches for the end of a
-    sentence using a specified regex pattern.
+    This function uses NLTK's sentence tokenizer to detect sentence boundaries
+    in the input text, combined with punctuation verification to ensure that
+    single tokens without proper sentence endings aren't considered complete sentences.
 
     Args:
-        text (str): The input text in which to find the end of the sentence.
+        text: The input text in which to find the end of the sentence.
 
     Returns:
-        int: The position of the end of the sentence if found, otherwise 0.
-
+        The position of the end of the sentence if found, otherwise 0.
     """
     text = text.rstrip()
 
-    # Replace email dots by ampersands so we can find the end of sentence. For
-    # example, first.last@email.com becomes first&last@email&com.
-    emails = list(EMAIL_PATTERN.finditer(text))
-    for email_match in emails:
-        text = replace_match(text, email_match, ".", "&")
+    if not text:
+        return 0
 
-    # Replace number dots by ampersands so we can find the end of sentence.
-    numbers = list(NUMBER_PATTERN.finditer(text))
-    for number_match in numbers:
-        text = replace_match(text, number_match, ".", "&")
+    # Use NLTK's sentence tokenizer to find sentence boundaries
+    sentences = sent_tokenize(text)
 
-    # Match against the new text.
-    match = ENDOFSENTENCE_PATTERN.search(text)
+    if not sentences:
+        return 0
 
-    return match.end() if match else 0
+    first_sentence = sentences[0]
+
+    # If there's only one sentence that equals the entire text,
+    # verify it actually ends with sentence-ending punctuation.
+    # This is required as NLTK may return a single sentence for
+    # text that's a single word. In the case of LLM tokens, it's
+    # common for text to be single words, so we need to ensure
+    # sentence-ending punctuation is present.
+    if len(sentences) == 1 and first_sentence == text:
+        return len(text) if text and text[-1] in SENTENCE_ENDING_PUNCTUATION else 0
+
+    # If there are multiple sentences, the first one is complete by definition
+    # (NLTK found a boundary, so there must be proper punctuation)
+    if len(sentences) > 1:
+        return len(first_sentence)
+
+    # Single sentence that doesn't equal the full text means incomplete
+    return 0
 
 
 def parse_start_end_tags(
@@ -90,24 +148,22 @@ def parse_start_end_tags(
     current_tag: Optional[StartEndTags],
     current_tag_index: int,
 ) -> Tuple[Optional[StartEndTags], int]:
-    """Parses the given text to identify a pair of start/end tags.
+    """Parse text to identify start and end tag pairs.
 
-    If a start tag was previously found (i.e. current_tags is valid), wait for
-    the corresponding end tag.  Otherwise, wait for a start tag.
+    If a start tag was previously found (i.e., current_tag is valid), wait for
+    the corresponding end tag. Otherwise, wait for a start tag.
 
-    This function will return the index in the text that we should start parsing
+    This function returns the index in the text where parsing should continue
     in the next call and the current or new tags.
 
-    Parameters:
-    - text (str): The text to be parsed.
-    - tags (Sequence[StartEndTags]): List of tuples containing start and end tags.
-    - current_tags (Optional[StartEndTags]): The currently active tags, if any.
-    - current_tags_index (int): The current index in the text.
+    Args:
+        text: The text to be parsed.
+        tags: List of tuples containing start and end tags.
+        current_tag: The currently active tags, if any.
+        current_tag_index: The current index in the text.
 
     Returns:
-    Tuple[Optional[StartEndTags], int]: A tuple containing None or the current
-    tag and the index of the text.
-
+        A tuple containing None or the current tag and the index of the text.
     """
     # If we are already inside a tag, check if the end tag is in the text.
     if current_tag:

pipecat/utils/text/base_text_aggregator.py

@@ -4,54 +4,85 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
+"""Base text aggregator interface for Pipecat text processing.
+
+This module defines the abstract base class for text aggregators that accumulate
+and process text tokens, typically used by TTS services to determine when
+aggregated text should be sent for speech synthesis.
+"""
+
 from abc import ABC, abstractmethod
 from typing import Optional
 
 
 class BaseTextAggregator(ABC):
-    """This is the base class for text aggregators. Text aggregators are usually
-    used by the TTS service to aggregate LLM tokens and decide when the
-    aggregated text should be pushed to the TTS service.
+    """Base class for text aggregators in the Pipecat framework.
+
+    Text aggregators are usually used by the TTS service to aggregate LLM tokens
+    and decide when the aggregated text should be pushed to the TTS service.
 
     Text aggregators can also be used to manipulate text while it's being
     aggregated (e.g. reasoning blocks can be removed).
 
+    Subclasses must implement all abstract methods to define specific aggregation
+    logic, text manipulation behavior, and state management for interruptions.
     """
 
     @property
     @abstractmethod
     def text(self) -> str:
-        """Returns the currently aggregated text."""
+        """Get the currently aggregated text.
+
+        Subclasses must implement this property to return the text that has
+        been accumulated so far in their internal buffer or storage.
+
+        Returns:
+            The text that has been accumulated so far.
+        """
         pass
 
     @abstractmethod
     async def aggregate(self, text: str) -> Optional[str]:
-        """Aggregates the specified text with the currently accumulated text.
+        """Aggregate the specified text with the currently accumulated text.
 
         This method should be implemented to define how the new text contributes
         to the aggregation process. It returns the updated aggregated text if
         it's ready to be processed, or None otherwise.
 
+        Subclasses should implement their specific logic for:
+
+        - How to combine new text with existing accumulated text
+        - When to consider the aggregated text ready for processing
+        - What criteria determine text completion (e.g., sentence boundaries)
+
         Args:
-            text (str): The text to be aggregated.
+            text: The text to be aggregated.
 
         Returns:
-            Optional[str]: The updated aggregated text or None if aggregated
-            text is not ready.
-
+            The updated aggregated text if ready for processing, or None if more
+            text is needed before the aggregated content is ready.
         """
         pass
 
     @abstractmethod
     async def handle_interruption(self):
-        """Handles interruptions. When an interruption occurs it is possible
-        that we might want to discard the aggregated text or do some internal
-        modifications to the aggregated text.
+        """Handle interruptions in the text aggregation process.
 
+        When an interruption occurs it is possible that we might want to discard
+        the aggregated text or do some internal modifications to the aggregated text.
+
+        Subclasses should implement this method to define how they respond to
+        interruptions, such as clearing buffers, resetting state, or preserving
+        partial content.
         """
         pass
 
     @abstractmethod
     async def reset(self):
-        """Clears the internally aggregated text."""
+        """Clear the internally aggregated text and reset to initial state.
+
+        Subclasses should implement this method to return the aggregator to its
+        initial state, discarding any previously accumulated text content and
+        resetting any internal tracking variables.
+        """
         pass

pipecat/utils/text/base_text_filter.py

@@ -4,23 +4,69 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
+"""Base text filter interface for Pipecat text processing.
+
+This module defines the abstract base class for text filters that can modify
+text content in the processing pipeline, including support for settings updates
+and interruption handling.
+"""
+
 from abc import ABC, abstractmethod
 from typing import Any, Mapping
 
 
 class BaseTextFilter(ABC):
+    """Abstract base class for text filters in the Pipecat framework.
+
+    Text filters are responsible for modifying text content as it flows through
+    the processing pipeline. They support dynamic settings updates and can handle
+    interruptions to reset their internal state.
+
+    Subclasses must implement all abstract methods to define specific filtering
+    behavior, settings management, and interruption handling logic.
+    """
+
     @abstractmethod
     async def update_settings(self, settings: Mapping[str, Any]):
+        """Update the filter's configuration settings.
+
+        Subclasses should implement this method to handle dynamic configuration
+        updates during runtime, updating internal state as needed.
+
+        Args:
+            settings: Dictionary of setting names to values for configuration.
+        """
         pass
 
     @abstractmethod
     async def filter(self, text: str) -> str:
+        """Apply filtering transformations to the input text.
+
+        Subclasses must implement this method to define the specific text
+        transformations that should be applied to the input.
+
+        Args:
+            text: The input text to be filtered.
+
+        Returns:
+            The filtered text after applying transformations.
+        """
         pass
 
     @abstractmethod
     async def handle_interruption(self):
+        """Handle interruption events in the processing pipeline.
+
+        Subclasses should implement this method to reset internal state,
+        clear buffers, or perform other cleanup when an interruption occurs.
+        """
         pass
 
     @abstractmethod
     async def reset_interruption(self):
+        """Reset the filter state after an interruption has been handled.
+
+        Subclasses should implement this method to restore the filter to normal
+        operation after an interruption has been processed and resolved.
+        """
         pass

pipecat/utils/text/markdown_text_filter.py

@@ -4,6 +4,12 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
+"""Markdown text filter for removing Markdown formatting from text.
+
+This module provides a text filter that converts Markdown content to plain text
+while preserving structure and handling special cases like code blocks and tables.
+"""
+
 import re
 from typing import Any, Mapping, Optional
 
@@ -14,19 +20,34 @@ from pipecat.utils.text.base_text_filter import BaseTextFilter
 
 
 class MarkdownTextFilter(BaseTextFilter):
-    """Removes Markdown formatting from text in TextFrames.
+    """Text filter that removes Markdown formatting from text content.
 
     Converts Markdown to plain text while preserving the overall structure,
     including leading and trailing spaces. Handles special cases like
-    asterisks and table formatting.
+    asterisks and table formatting. Supports selective filtering of code
+    blocks and tables based on configuration.
     """
 
     class InputParams(BaseModel):
+        """Configuration parameters for Markdown text filtering.
+
+        Parameters:
+            enable_text_filter: Whether to apply Markdown filtering. Defaults to True.
+            filter_code: Whether to remove code blocks from the text. Defaults to False.
+            filter_tables: Whether to remove table content from the text. Defaults to False.
+        """
+
         enable_text_filter: Optional[bool] = True
         filter_code: Optional[bool] = False
         filter_tables: Optional[bool] = False
 
     def __init__(self, params: Optional[InputParams] = None, **kwargs):
+        """Initialize the Markdown text filter.
+
+        Args:
+            params: Configuration parameters for filtering behavior.
+            **kwargs: Additional keyword arguments passed to parent class.
+        """
         super().__init__(**kwargs)
         self._settings = params or MarkdownTextFilter.InputParams()
         self._in_code_block = False
@@ -34,11 +55,24 @@ class MarkdownTextFilter(BaseTextFilter):
         self._interrupted = False
 
     async def update_settings(self, settings: Mapping[str, Any]):
+        """Update the filter's configuration settings.
+
+        Args:
+            settings: Dictionary of setting names to values for configuration.
+        """
         for key, value in settings.items():
             if hasattr(self._settings, key):
                 setattr(self._settings, key, value)
 
     async def filter(self, text: str) -> str:
+        """Apply Markdown filtering transformations to the input text.
+
+        Args:
+            text: The input text containing Markdown formatting to be filtered.
+
+        Returns:
+            The filtered text with Markdown formatting removed or converted.
+        """
         if self._settings.enable_text_filter:
             # Remove newlines and replace with a space only when there's no text before or after
             filtered_text = re.sub(r"^\s*\n", " ", text, flags=re.MULTILINE)
@@ -108,11 +142,20 @@ class MarkdownTextFilter(BaseTextFilter):
             return text
 
     async def handle_interruption(self):
+        """Handle interruption events in the processing pipeline.
+
+        Resets the filter state and clears any tracking variables for
+        code blocks and tables.
+        """
         self._interrupted = True
         self._in_code_block = False
         self._in_table = False
 
     async def reset_interruption(self):
+        """Reset the filter state after an interruption has been handled.
+
+        Clears the interrupted flag to restore normal operation.
+        """
         self._interrupted = False
 
     #
@@ -120,8 +163,10 @@ class MarkdownTextFilter(BaseTextFilter):
     #
 
     def _remove_code_blocks(self, text: str) -> str:
-        """Main method to remove code blocks from the input text.
-        Handles interruptions and delegates to specific methods based on the current state.
+        """Remove code blocks from the input text.
+
+        Handles interruptions and delegates to specific methods based on the
+        current state.
         """
         if self._interrupted:
             self._in_code_block = False
@@ -137,8 +182,10 @@ class MarkdownTextFilter(BaseTextFilter):
         return self._handle_not_in_code_block(match, text, code_block_pattern)
 
     def _handle_in_code_block(self, match, text):
-        """Handle text when we're currently inside a code block.
-        If we find the end of the block, return text after it. Otherwise, skip the content.
+        """Handle text when not currently inside a code block.
+
+        If we find the end of the block, return text after it. Otherwise, skip
+        the content.
         """
         if match:
             self._in_code_block = False
@@ -147,9 +194,7 @@ class MarkdownTextFilter(BaseTextFilter):
         return ""  # Skip content inside code block
 
     def _handle_not_in_code_block(self, match, text, code_block_pattern):
-        """Handle text when we're not currently inside a code block.
-        Delegate to specific methods based on whether we find a code block delimiter.
-        """
+        """Handle text when not currently inside a code block."""
         if not match:
             return text  # No code block found, return original text
 
@@ -159,14 +204,17 @@ class MarkdownTextFilter(BaseTextFilter):
         return self._handle_code_block_within_text(text, code_block_pattern)
 
     def _handle_start_of_code_block(self, text, start_index):
-        """Handle the case where we find the start of a code block.
-        Return any text before the code block and set the state to inside a code block.
+        """Handle the case where a code block starts.
+
+        Return any text before the code block and set the state to inside a
+        code block.
         """
         self._in_code_block = True
         return text[:start_index].strip()
 
     def _handle_code_block_within_text(self, text, code_block_pattern):
-        """Handle the case where we find a code block within the text.
+        """Handle code blocks found within text content.
+
         If it's a complete code block, remove it and return surrounding text.
         If it's the start of a code block, return text before it and set state.
         """
@@ -180,8 +228,16 @@ class MarkdownTextFilter(BaseTextFilter):
     # Filter tables
     #
     def remove_tables(self, text: str) -> str:
-        """Remove tables from the input text, handling cases where
-        both start and end tags are in the same input.
+        """Remove HTML tables from the input text.
+
+        Handles cases where both start and end tags are in the same input,
+        as well as tables that span multiple text chunks.
+
+        Args:
+            text: The text containing HTML tables to remove.
+
+        Returns:
+            The text with tables removed.
         """
         if self._interrupted:
             self._in_table = False

pipecat/utils/text/pattern_pair_aggregator.py

@@ -4,6 +4,13 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
+"""Pattern pair aggregator for processing structured content in streaming text.
+
+This module provides an aggregator that identifies and processes content between
+pattern pairs (like XML tags or custom delimiters) in streaming text, with
+support for custom handlers and configurable pattern removal.
+"""
+
 import re
 from typing import Awaitable, Callable, Optional, Tuple
 
@@ -20,20 +27,15 @@ class PatternMatch:
     in the text. It contains information about which pattern was matched,
     the full matched text (including start and end patterns), and the
     content between the patterns.
-
-    Attributes:
-        pattern_id: The identifier of the matched pattern pair.
-        full_match: The complete text including start and end patterns.
-        content: The text content between the start and end patterns.
     """
 
     def __init__(self, pattern_id: str, full_match: str, content: str):
         """Initialize a pattern match.
 
         Args:
-            pattern_id: ID of the pattern pair.
-            full_match: Complete matched text including start and end patterns.
-            content: Content between the start and end patterns.
+            pattern_id: The identifier of the matched pattern pair.
+            full_match: The complete text including start and end patterns.
+            content: The text content between the start and end patterns.
         """
         self.pattern_id = pattern_id
         self.full_match = full_match
@@ -43,7 +45,7 @@ class PatternMatch:
         """Return a string representation of the pattern match.
 
         Returns:
-            A string describing the pattern match.
+            A descriptive string showing the pattern ID and content.
         """
         return f"PatternMatch(id={self.pattern_id}, content={self.content})"
 
@@ -66,6 +68,7 @@ class PatternPairAggregator(BaseTextAggregator):
         """Initialize the pattern pair aggregator.
 
         Creates an empty aggregator with no patterns or handlers registered.
+        Text buffering and pattern detection will begin when text is aggregated.
         """
         self._text = ""
         self._patterns = {}
@@ -76,7 +79,7 @@ class PatternPairAggregator(BaseTextAggregator):
         """Get the currently buffered text.
 
         Returns:
-            The current text buffer content.
+            The current text buffer content that hasn't been processed yet.
         """
         return self._text
 
@@ -115,7 +118,7 @@ class PatternPairAggregator(BaseTextAggregator):
 
         Args:
             pattern_id: ID of the pattern pair to match.
-            handler: Function to call when pattern is matched.
+            handler: Async function to call when pattern is matched.
                      The function should accept a PatternMatch object.
 
         Returns:
@@ -131,10 +134,11 @@ class PatternPairAggregator(BaseTextAggregator):
         appropriate handlers, and optionally removes the matches.
 
         Args:
-            text: The text to process.
+            text: The text to process for pattern matches.
 
         Returns:
             Tuple of (processed_text, was_modified) where:
+
             - processed_text is the text after processing patterns
             - was_modified indicates whether any changes were made
         """
@@ -185,7 +189,7 @@ class PatternPairAggregator(BaseTextAggregator):
         matching end patterns, which would indicate incomplete content.
 
         Args:
-            text: The text to check.
+            text: The text to check for incomplete patterns.
 
         Returns:
             True if there are incomplete patterns, False otherwise.
@@ -257,6 +261,6 @@ class PatternPairAggregator(BaseTextAggregator):
         """Clear the internally aggregated text.
 
         Resets the aggregator to its initial state, discarding any
-        buffered text.
+        buffered text and clearing pattern tracking state.
         """
         self._text = ""