arbtok 0.0.0a2__tar.gz

arbtok-0.0.0a2/PKG-INFO

@@ -0,0 +1,93 @@
+Metadata-Version: 2.4
+Name: arbtok
+Version: 0.0.0a2
+Summary: Rule-based Arabic (MSA) text-to-IPA with tashkeel diacritization — an orthography2ipa G2P plugin
+License: Apache-2.0
+Project-URL: Homepage, https://github.com/TigreGotico/arbtok
+Project-URL: Issues, https://github.com/TigreGotico/arbtok/issues
+Keywords: arabic,ipa,g2p,phonemizer,tashkeel,diacritization,tts
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: numpy
+Requires-Dist: quebra-frases
+Requires-Dist: langcodes
+Requires-Dist: ovos-number-parser>=0.4.0
+Requires-Dist: ovos-date-parser>=0.6.4a1
+Requires-Dist: orthography2ipa>=0.3.0a1
+Requires-Dist: text2tashkeel>=0.1.0
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-timeout; extra == "test"
+
+# arbtok
+
+Rule-based **Arabic (MSA) text→IPA** with **tashkeel** diacritization — a
+downstream Arabic engine built on
+[orthography2ipa](https://github.com/TigreGotico/orthography2ipa).
+
+A context-sensitive token tree (sentence → words → characters) implements the
+morpho-phonological rules that table lookups cannot express: sun-letter
+assimilation, hamzat al-waṣl elision, tanwīn pausal forms, tāʾ marbūṭa,
+mater-lectionis vowel lengthening, definite-article waṣl, and idgham/iqlab
+nasal assimilation. Bare (undiacritized) text is diacritized first via
+[text2tashkeel](https://github.com/TigreGotico/text2tashkeel) — a model
+picker over bundled ONNX diacritization models.
+
+> Honesty note: the gold IPA reference set was LLM-generated and has not been
+> validated by a native MSA speaker. If you speak MSA, pull requests are very
+> welcome.
+
+## Installation
+
+```bash
+pip install arbtok
+```
+
+## Usage
+
+arbtok is built **on** [orthography2ipa](https://github.com/TigreGotico/orthography2ipa)
+(spec data and the shared `G2PPlugin`/`WordContext` base types) and owns the
+Arabic pipeline — orthography2ipa stays the language-agnostic base library.
+
+### Engine class
+
+```python
+from arbtok.tokenizer import Sentence
+
+Sentence("اَلسَّلَامُ عَلَيْكُمْ").ipa
+```
+
+Bare text is handled by diacritizing first:
+
+```python
+from arbtok.plugin import ArbtokG2PPlugin
+
+plugin = ArbtokG2PPlugin()
+plugin.transcribe("كتاب جميل")    # auto-tashkeel + IPA
+```
+
+### Diacritization only
+
+```python
+from arbtok.tashkeel import TashkeelDiacritizer   # wraps text2tashkeel
+
+TashkeelDiacritizer().diacritize("كتاب جميل")
+```
+
+## Quality benchmarks
+
+The test suite pins a gold sentence set (CER target ≤ 5% against the
+reference transcriptions) and benchmarks against espeak-ng. See
+`tests/test_ipa_fuzzy.py` and `docs/` for details.

arbtok-0.0.0a2/README.md

@@ -0,0 +1,60 @@
+# arbtok
+
+Rule-based **Arabic (MSA) text→IPA** with **tashkeel** diacritization — a
+downstream Arabic engine built on
+[orthography2ipa](https://github.com/TigreGotico/orthography2ipa).
+
+A context-sensitive token tree (sentence → words → characters) implements the
+morpho-phonological rules that table lookups cannot express: sun-letter
+assimilation, hamzat al-waṣl elision, tanwīn pausal forms, tāʾ marbūṭa,
+mater-lectionis vowel lengthening, definite-article waṣl, and idgham/iqlab
+nasal assimilation. Bare (undiacritized) text is diacritized first via
+[text2tashkeel](https://github.com/TigreGotico/text2tashkeel) — a model
+picker over bundled ONNX diacritization models.
+
+> Honesty note: the gold IPA reference set was LLM-generated and has not been
+> validated by a native MSA speaker. If you speak MSA, pull requests are very
+> welcome.
+
+## Installation
+
+```bash
+pip install arbtok
+```
+
+## Usage
+
+arbtok is built **on** [orthography2ipa](https://github.com/TigreGotico/orthography2ipa)
+(spec data and the shared `G2PPlugin`/`WordContext` base types) and owns the
+Arabic pipeline — orthography2ipa stays the language-agnostic base library.
+
+### Engine class
+
+```python
+from arbtok.tokenizer import Sentence
+
+Sentence("اَلسَّلَامُ عَلَيْكُمْ").ipa
+```
+
+Bare text is handled by diacritizing first:
+
+```python
+from arbtok.plugin import ArbtokG2PPlugin
+
+plugin = ArbtokG2PPlugin()
+plugin.transcribe("كتاب جميل")    # auto-tashkeel + IPA
+```
+
+### Diacritization only
+
+```python
+from arbtok.tashkeel import TashkeelDiacritizer   # wraps text2tashkeel
+
+TashkeelDiacritizer().diacritize("كتاب جميل")
+```
+
+## Quality benchmarks
+
+The test suite pins a gold sentence set (CER target ≤ 5% against the
+reference transcriptions) and benchmarks against espeak-ng. See
+`tests/test_ipa_fuzzy.py` and `docs/` for details.

arbtok-0.0.0a2/arbtok/__init__.py

@@ -0,0 +1,13 @@
+"""
+I'm just a dude trying to create TTS voices to help the visually impaired arabic community.
+
+I am not an arabic speaker, I never had any previous interaction with the arabic script, this might be a pointless experiment powered by LLM hallucinations.
+
+You have been warned.
+
+if you speak MSA pull requests welcome!
+
+- test.py contains a reference dataset to benchmark the code against, IPA transcriptions have been LLM generated
+- main logic is in tokenizer.py
+- see tests folder for some benchmarks and comparison against pre existing (low quality) datasets
+"""

arbtok-0.0.0a2/arbtok/constants.py

@@ -0,0 +1,87 @@
+import string
+from typing import Set
+
+# ==============================================================================
+# ARABIC CONSTANTS
+# ==============================================================================
+
+# --- Diacritics (Tashkeel) ---
+FATHA = '\u064E'  # َ (short a)
+DAMMA = '\u064F'  # ُ (short u)
+KASRA = '\u0650'  # ِ (short i)
+SHADDA = '\u0651'  # ّ (consonant doubling/gemination)
+SUKUN = '\u0652'  # ْ (no vowel/silence)
+TANWIN_FATH = '\u064B'  # ً (an)
+TANWIN_DAMM = '\u064C'  # ٌ (un)
+TANWIN_KASR = '\u064D'  # ٍ (in)
+DAGGER_ALIF = '\u0670'  # ٰ (superscript alef, long a)
+MADD = '\u0653'  # ٓ (madda, vowel elongation)
+
+# --- Letters & Variants ---
+HAMZA = '\u0621'  # ء (standalone hamza)
+ALEF_MADDA = '\u0622'  # آ (alef with madda)
+ALEF_HAMZA_ABOVE = '\u0623'  # أ (alef with hamza above)
+WAW_HAMZA = '\u0624'  # ؤ (waw with hamza)
+ALEF_HAMZA_BELOW = '\u0625'  # إ (alef with hamza below)
+YA_HAMZA = '\u0626'  # ئ (ya with hamza)
+ALIF = '\u0627'  # ا (bare alef)
+TA_MARBUTA = '\u0629'  # ة (tied ta, fem. marker)
+WAW = '\u0648'  # و
+ALIF_MAKSURA = '\u0649'  # ى (broken alef)
+YA = '\u064A'  # ي
+LAM = '\u0644'  # ل
+HAMZAT_AL_WASL = '\u0671'  # ٱ (wasl, connecting hamza)
+
+ALEF_VARIANTS = {ALIF, ALEF_MADDA, ALEF_HAMZA_ABOVE, ALEF_HAMZA_BELOW, HAMZAT_AL_WASL}
+
+# --- Punctuation ---
+ARABIC_PUNCT = "،؛؟ـ…"  # Comma, Semicolon, Question mark, Tatweel, Ellipsis
+PUNCT = string.punctuation + ARABIC_PUNCT
+
+# --- Sun Letters (Shamsiyya) ---
+# These consonants assimilate the 'l' of the definite article 'al-'.
+# e.g., al-shams -> ash-shams.
+SUN_LETTERS: Set[str] = {
+    '\u062A',  # ت (t)
+    '\u062B',  # ث (th)
+    '\u062F',  # د (d)
+    '\u0630',  # ذ (dh)
+    '\u0631',  # ر (r)
+    '\u0632',  # ز (z)
+    '\u0633',  # س (s)
+    '\u0634',  # ش (sh)
+    '\u0635',  # ص (sad)
+    '\u0636',  # ض (dad)
+    '\u0637',  # ط (ta)
+    '\u0638',  # ظ (dha)
+    '\u0644',  # ل (l)
+    '\u0646',  # ن (n)
+}
+
+# --- Proclitics ---
+# Short prepositions/conjunctions that attach to the following word in writing.
+CLITIC_BASES = {
+    WAW,  # و (and)
+    '\u0641',  # ف (so/then)
+    '\u0628',  # ب (by/with)
+    '\u0643',  # ك (like/as)
+    LAM,  # ل (for/to)
+    '\u0633'  # س (future marker 'sa-')
+}
+
+# --- IPA Mappings ---
+# for easy unambiguous phoneme reference in the code when the unicode maps to a specific IPA sound
+B = '\u0628'
+R = '\u0631'
+Z = '\u0632'
+S = '\u0633'
+F = '\u0641'
+Q = '\u0642'
+K = '\u0643'
+M = '\u0645'
+N = '\u0646'
+H = '\u0647'
+T = '\u062A'
+DJ = '\u062C'
+X = '\u062E'
+D = '\u062F'

arbtok-0.0.0a2/arbtok/dialects.py

@@ -0,0 +1,111 @@
+"""
+TODO: to be extended with different phoneme realizations per dialect.
+logic in tokenizer might use ArabicDialect enum if needed for further contextual rules
+
+currently only MSA is targeted
+"""
+from enum import Enum
+from arbtok.constants import (B, T, DJ, X, D, R, Z, S, F, Q, K, M, N, H, LAM, WAW, YA,
+                              FATHA, DAMMA, KASRA, DAGGER_ALIF, MADD,
+                              HAMZA, ALEF_HAMZA_ABOVE, ALEF_HAMZA_BELOW, WAW_HAMZA, YA_HAMZA,
+                              TANWIN_FATH, TANWIN_KASR, TANWIN_DAMM,
+                              ALIF_MAKSURA)
+
+
+class ArabicDialect(str, Enum):
+    CLA = "CLA"
+    MSA = "MSA"
+
+
+ARABIC_TO_IPA_CONSONANTS = {
+    HAMZA: "ʔ", ALEF_HAMZA_ABOVE: "ʔ", ALEF_HAMZA_BELOW: "ʔ", WAW_HAMZA: "ʔ", YA_HAMZA: "ʔ",
+    B: 'b',
+    T: 't',
+    '\u062B': 'θ',
+    DJ: 'dʒ',  # DJ: 'ʤ',
+    '\u062D': 'ħ',
+    X: 'x',
+    D: 'd',
+    '\u0630': 'ð',
+    R: 'r',
+    Z: 'z',
+    S: 's',
+    '\u0634': 'ʃ',
+    '\u0635': 'sˤ',
+    '\u0636': 'dˤ',
+    '\u0637': 'tˤ',
+    '\u0638': 'ðˤ',
+    '\u0639': 'ʕ',
+    '\u063A': 'ɣ',
+    F: 'f',
+    Q: 'q',
+    K: 'k',
+    M: 'm',
+    N: 'n',
+    H: 'h',
+    LAM: 'l',
+    WAW: 'w',
+    YA: 'j',
+    # Other unicode variations
+    "ﻻ": "laː"
+}
+
+DIACRITIC_TO_IPA = {
+    FATHA: 'a',
+    DAMMA: 'u',
+    KASRA: 'i',
+    DAGGER_ALIF: 'aː',
+    MADD: 'aː'
+}
+TANWIN_TO_IPA = {
+    TANWIN_FATH: "an",
+    TANWIN_DAMM: "un",
+    TANWIN_KASR: "in"
+}
+
+VOWEL_MAP = {**DIACRITIC_TO_IPA,
+             **TANWIN_TO_IPA,
+             MADD: ":",
+             ALIF_MAKSURA: 'aː'}
+
+# --- Word Exceptions ---
+# TODO - LLM generated, needs validation from native speaker
+# Dictionary for words with irregular orthography vs pronunciation.
+# These words have "deficient" or "historical" spelling where vowels
+# are pronounced but not written, or written letters are silent.
+WORD_EXCEPTIONS = {
+    # === Unicode / orthographic variants ===
+
+    # Demonstratives with "Dagger Alif" (pronounced long /a:/ but written short or omitted)
+    "هَٰذَا": "haːðaː",  # ha-dha (this, m.)
+    "هٰذَا": "haːðaː",  # variant
+    "هَذَا": "haːðaː",  # common deficient spelling
+    "هَٰذِهِ": "haːðihi",  # ha-dhi-hi (this, f.)
+    "هٰذِهِ": "haːðihi",  # variant
+    "ذَٰلِكَ": "ðaːlika",  # dha-li-ka (that)
+    "ذٰلِكَ": "ðaːlika",  # variant
+    "أُولَٰئِكَ": "ʔulaːʔika",  # u-la-i-ka (those) - note medial hamza logic is complex, hardcoded here
+
+    # Particles
+    "لَٰكِن": "laːkin",  # la-kin (but) - unwritten medial alif
+    "لَكِن": "laːkin",  # deficient spelling
+    "لَٰكِنَّ": "laːkinna",  # la-kin-na (but...)
+
+    # Divine Names
+    "ﷲ": "allaːh",
+    "اللّٰه": "allaːh",  # Allah - heavy L, unwritten alif
+    "اللَّه": "allaːh",  # variant
+    "إِلَٰه": "ʔilaːh",  # ilah (god)
+    "الرَّحْمَٰن": "arraħmaːn",  # Ar-Rahman
+
+    # Irregular pronunciations
+    "مِائَة": "miʔa",  # mi'a (hundred) - silent extra Alif in spelling
+    "عَمْرٌو": "ʕamrun",  # 'Amr - final Waw is silent (differentiates from 'Umar)
+    "أُولِي": "ʔuliː",  # 'uli (owners of) - silent Waw
+    "أُولُو": "ʔuluː",  # 'ulu - silent Waw
+
+    # TODO - why are these irregular?
+    "مُسْتَشْفَى": "mustashfaː",
+    "مَدْرَسَة": "madrasah"
+}
+

arbtok-0.0.0a2/arbtok/espeak_wrapper.py

@@ -0,0 +1,237 @@
+import os
+import re
+import string
+import subprocess
+from typing import List, Optional
+from typing import Tuple
+
+import numpy as np
+import onnxruntime
+import requests
+from quebra_frases import sentence_tokenize
+
+from arbtok.tashkeel import TashkeelDiacritizer
+from arbtok.util import normalize, match_lang
+
+# list of (substring, terminator, end_of_sentence) tuples.
+TextChunks = List[Tuple[str, str, bool]]
+# list of (phonemes, terminator, end_of_sentence) tuples.
+RawPhonemizedChunks = List[Tuple[str, str, bool]]
+
+PhonemizedChunks = list[list[str]]
+
+
+class EspeakError(Exception):
+    """Custom exception for espeak-ng related errors."""
+    pass
+
+
+class EspeakPhonemizer:
+    """
+    A phonemizer class that uses the espeak-ng command-line tool to convert text into phonemes.
+    It segments the input text heuristically based on punctuation to mimic clause-by-clause processing.
+    """
+    ESPEAK_LANGS = ['es-419', 'ca', 'qya', 'ga', 'et', 'ky', 'io', 'fa-latn', 'en-gb', 'fo', 'haw', 'kl',
+                    'ta', 'ml', 'gd', 'sd', 'es', 'hy', 'ur', 'ro', 'hi', 'or', 'ti', 'ca-va', 'om', 'tr', 'pa',
+                    'smj', 'mk', 'bg', 'cv', "fr", 'fi', 'en-gb-x-rp', 'ru', 'mt', 'an', 'mr', 'pap', 'vi', 'id',
+                    'fr-be', 'ltg', 'my', 'nl', 'shn', 'ba', 'az', 'cmn', 'da', 'as', 'sw',
+                    'piqd', 'en-us', 'hr', 'it', 'ug', 'th', 'mi', 'cy', 'ru-lv', 'ia', 'tt', 'hu', 'xex', 'te', 'ne',
+                    'eu', 'ja', 'bpy', 'hak', 'cs', 'en-gb-scotland', 'hyw', 'uk', 'pt', 'bn', 'mto', 'yue',
+                    'be', 'gu', 'sv', 'sl', 'cmn-latn-pinyin', 'lfn', 'lv', 'fa', 'sjn', 'nog', 'ms',
+                    'vi-vn-x-central', 'lt', 'kn', 'he', 'qu', 'ca-ba', 'quc', 'nb', 'sk', 'tn', 'py', 'si', 'de',
+                    'ar', 'en-gb-x-gbcwmd', 'bs', 'qdb', 'sq', 'sr', 'tk', 'en-029', 'ht', 'ru-cl', 'af', 'pt-br',
+                    'fr-ch', 'ka', 'en-gb-x-gbclan', 'ko', 'is', 'ca-nw', 'gn', 'kok', 'la', 'lb', 'am', 'kk', 'ku',
+                    'kaa', 'jbo', 'eo', 'uz', 'nci', 'vi-vn-x-south', 'el', 'pl', 'grc', ]
+
+    def __init__(self, pausal: bool = True):
+        self.pausal = pausal  # arabic only
+        self._tashkeel: Optional[TashkeelDiacritizer] = None
+
+    @classmethod
+    def get_lang(cls, target_lang: str) -> str:
+        """
+        Validates and returns the closest supported language code.
+
+        Args:
+            target_lang (str): The language code to validate.
+
+        Returns:
+            str: The validated language code.
+
+        Raises:
+            ValueError: If the language code is unsupported.
+        """
+        if target_lang.lower() == "en-gb":
+            return "en-gb-x-rp"
+        if target_lang in cls.ESPEAK_LANGS:
+            return target_lang
+        if target_lang.lower().split("-")[0] in cls.ESPEAK_LANGS:
+            return target_lang.lower().split("-")[0]
+        return cls.match_lang(target_lang, cls.ESPEAK_LANGS)
+
+    @property
+    def tashkeel(self) -> TashkeelDiacritizer:
+        if self._tashkeel is None:
+            self._tashkeel = TashkeelDiacritizer()
+        return self._tashkeel
+
+    @staticmethod
+    def _run_espeak_command(args: List[str], input_text: str = None, check: bool = True) -> str:
+        """
+        Helper function to run espeak-ng commands via subprocess.
+        Executes 'espeak-ng' with the given arguments and input text.
+        Captures stdout and stderr, and raises EspeakError on failure.
+
+        Args:
+            args (List[str]): A list of command-line arguments for espeak-ng.
+            input_text (str, optional): The text to pass to espeak-ng's stdin. Defaults to None.
+            check (bool, optional): If True, raises a CalledProcessError if the command returns a non-zero exit code. Defaults to True.
+
+        Returns:
+            str: The stripped standard output from the espeak-ng command.
+
+        Raises:
+            EspeakError: If espeak-ng command is not found, or if the subprocess call fails.
+        """
+        command: List[str] = ['espeak-ng'] + args
+
+        # Standard arguments for subprocess.run
+        subprocess_args = {
+            'input': input_text,
+            'capture_output': True,
+            'text': True,
+            'check': check,
+            'encoding': 'utf-8',
+            'errors': 'replace'  # Replaces unencodable characters with a placeholder
+        }
+
+        # Add 'creationflags' to hide the terminal window on Windows
+        if os.name == 'nt':
+            subprocess_args['creationflags'] = 0x08000000
+
+        try:
+            process: subprocess.CompletedProcess = subprocess.run(
+                command,
+                **subprocess_args  # Use the dynamic arguments
+            )
+            return process.stdout.strip()
+        except FileNotFoundError:
+            raise EspeakError(
+                "espeak-ng command not found. Please ensure espeak-ng is installed "
+                "and available in your system's PATH."
+            )
+        except subprocess.CalledProcessError as e:
+            raise EspeakError(
+                f"espeak-ng command failed with error code {e.returncode}:\n"
+                f"STDOUT: {e.stdout}\n"
+                f"STDERR: {e.stderr}"
+            )
+        except Exception as e:
+            raise EspeakError(f"An unexpected error occurred while running espeak-ng: {e}")
+
+    def phonemize_string(self, text: str, lang: str) -> str:
+        lang = self.get_lang(lang)
+        return self._run_espeak_command(
+            ['-q', '-x', '--ipa', '-v', lang],
+            input_text=text
+        )
+
+    def phonemize_to_list(self, text: str, lang: str = "ar") -> List[str]:
+        return list(self.phonemize_string(text, lang))
+
+    def add_diacritics(self, text: str, lang: str= "ar") -> str:
+        if lang.startswith("ar"):
+            return self.tashkeel.diacritize(text, pausal=self.pausal)
+        return text
+
+    def phonemize(self, text: str, lang: str= "ar") -> PhonemizedChunks:
+        if not text:
+            return [('', '', True)]
+        results: RawPhonemizedChunks = []
+        text = normalize(text, lang)
+        for chunk, punct, eos in self.chunk_text(text):
+            phoneme_str = self.phonemize_string(self.remove_punctuation(chunk), lang)
+            results += [(phoneme_str, punct, True)]
+        return self._process_phones(results)
+
+    @staticmethod
+    def _process_phones(raw_phones: RawPhonemizedChunks) -> PhonemizedChunks:
+        """Text to phonemes grouped by sentence."""
+        all_phonemes: list[list[str]] = []
+        sentence_phonemes: list[str] = []
+        for phonemes_str, terminator_str, end_of_sentence in raw_phones:
+            # Filter out (lang) switch (flags).
+            # These surround words from languages other than the current voice.
+            phonemes_str = re.sub(r"\([^)]+\)", "", phonemes_str)
+            sentence_phonemes.extend(list(phonemes_str))
+            if end_of_sentence:
+                all_phonemes.append(sentence_phonemes)
+                sentence_phonemes = []
+        if sentence_phonemes:
+            all_phonemes.append(sentence_phonemes)
+        return all_phonemes
+
+    @staticmethod
+    def match_lang(target_lang: str, valid_langs: List[str]) -> str:
+        """
+        Validates and returns the closest supported language code.
+
+        Args:
+            target_lang (str): The language code to validate.
+
+        Returns:
+            str: The validated language code.
+
+        Raises:
+            ValueError: If the language code is unsupported.
+        """
+        lang, score = match_lang(target_lang, valid_langs)
+        if score > 10:
+            # raise an error for unsupported language
+            raise ValueError(f"unsupported language code: {target_lang}")
+        return lang
+
+    @staticmethod
+    def remove_punctuation(text):
+        """
+        Removes all punctuation characters from a string.
+        Punctuation characters are defined by string.punctuation.
+        """
+        # Create a regex pattern that matches any character in string.punctuation
+        punctuation_pattern = r"[" + re.escape(string.punctuation) + r"]"
+        return re.sub(punctuation_pattern, '', text).strip()
+
+    @staticmethod
+    def chunk_text(text: str, delimiters: Optional[List[str]] = None) -> TextChunks:
+        if not text:
+            return [('', '', True)]
+
+        results: TextChunks = []
+        delimiters = delimiters or [":", ";", "...", "|"]
+
+        # Create a regex pattern that matches any of the delimiters
+        delimiter_pattern = re.escape(delimiters[0])
+        for delimiter in delimiters[1:]:
+            delimiter_pattern += f"|{re.escape(delimiter)}"
+
+        for sentence in sentence_tokenize(text):
+            # Default punctuation if no specific punctuation found
+            default_punc = sentence[-1] if sentence and sentence[-1] in string.punctuation else "."
+
+            # Use regex to split the sentence by any of the delimiters
+            parts = re.split(f'({delimiter_pattern})', sentence)
+
+            # Group parts into chunks (text + delimiter)
+            chunks = []
+            for i in range(0, len(parts), 2):
+                # If there's a delimiter after the text, use it
+                delimiter = parts[i + 1] if i + 1 < len(parts) else default_punc
+
+                # Last chunk is marked as complete
+                is_last = (i + 2 >= len(parts))
+
+                chunks.append((parts[i].strip(), delimiter.strip(), is_last))
+
+            results.extend(chunks)
+
+        return results