yasbd-lib 0.1.0__py3-none-any.whl

yasbd/__init__.py

@@ -0,0 +1,3 @@
+from yasbd.boundary_detector import BoundaryDetector, ParagraphEOF
+
+__all__ = ["BoundaryDetector", "ParagraphEOF"]

yasbd/boundary_detector.py

@@ -0,0 +1,187 @@
+from collections.abc import Generator, Iterable
+from importlib import import_module
+from io import TextIOBase
+from itertools import tee
+
+from loguru import logger
+
+from yasbd.utils.cleaner_stub import StreamCleanerStub
+from yasbd.utils.input_validator import validate_input
+from yasbd.utils.paragraph_stream import ParagraphStream
+
+# Signals transition between paragraphs in relative mode
+# during boundary detection
+ParagraphEOF = type("_ParagraphEOF", (), {"__repr__": lambda self: "ParagraphEOF"})()
+
+
+class BoundaryDetector:
+    @validate_input
+    def __init__(
+        self,
+        lang: str = "en",
+        *,
+        preserve_quote_and_paren: bool = True,
+        verbose: bool = False,
+    ):
+        """Initialize the segmenter.
+
+        Args:
+            lang: Two chars ISO language code (e.g. en, fr, ...).
+            preserve_quote_and_paren: Do not split on terminators inside
+                quoted or parenthesised text.
+            verbose: Enable verbose logging.
+        """
+        self.preserve_quote_and_paren = preserve_quote_and_paren
+        self.verbose = verbose
+        self.lang = lang.lower()
+        if self.verbose:  # pragma: no cover
+            logger.info(
+                "Initialized with lang={!r}, preserve_quote_and_paren={}, verbose={}",
+                self._lang,
+                self.preserve_quote_and_paren,
+                self.verbose,
+            )
+
+    @property
+    def lang(self) -> str:
+        """ISO language code of the active rule set."""
+        return self._lang
+
+    @lang.setter
+    def lang(self, lang: str) -> None:
+        lang = lang.lower()
+        old_lang = getattr(self, "_lang", None)
+        if lang == old_lang:
+            return
+
+        self._load_rule(lang)
+        self._lang = lang
+        if self.verbose:  # pragma: no cover
+            logger.info("Language switched from {} to {}", old_lang, self._lang)
+
+    def _load_rule(self, lang: str) -> None:
+        """Dynamically import and instantiate the rule module for *lang*."""
+        if self.verbose:  # pragma: no cover
+            logger.info("Trying to load rule module for {}", lang)
+
+        try:
+            rule_module = import_module(f"yasbd.rules.{lang}")
+        except ModuleNotFoundError:
+            raise ValueError(f"Unsupported language: {lang!r}") from None
+
+        self._rule = getattr(rule_module, f"{lang.capitalize()}Rules")()
+
+    def _detect_relative_spans(
+        self,
+        para_iter: Iterable[str],
+    ) -> Generator[tuple[int, int], None, None]:
+        """Yield per-paragraph sentence spans."""
+        for para in para_iter:
+            if not para or para.isspace():
+                boundaries = [0, len(para)]
+            else:
+                boundaries = self._rule.apply(para, self.preserve_quote_and_paren)
+
+            for i in range(len(boundaries) - 1):
+                start = boundaries[i]
+                end = boundaries[i + 1]
+                yield (start, end)
+
+    @validate_input
+    def detect(
+        self,
+        source: str | TextIOBase | StreamCleanerStub,
+        *,
+        relative: bool = False,
+    ) -> Generator[int, None, None]:
+        """Detect sentence boundaries in the source text.
+
+        Args:
+            source: Plain text string, an open text stream (e.g., ``StringIO``),
+               or a ``StreamCleaner`` instance.
+            relative: If ``False`` (default), yields absolute character
+                offsets from the beginning of the entire stream. If ``True``,
+                offsets reset at each paragraph break, yielding indices relative
+                to the start of the current paragraph.
+
+        Note:
+            When ``relative=True``, a ``ParagraphEOF`` sentinel is yielded
+            between distinct paragraphs to signal the boundary of the local
+            coordinate system. Import via: ``from yasbd import ParagraphEOF``.
+
+        Yields:
+            Integer boundary offsets or ``ParagraphEOF`` sentinels.
+        """
+
+        if self.verbose:  # pragma: no cover
+            logger.info(
+                "Called with type={}, relative={}", type(source).__name__, relative
+            )
+
+        para_iter = (
+            ParagraphStream(source) if isinstance(source, (str, TextIOBase)) else source
+        )
+
+        offset = 0
+        is_first_pos = True
+        for para in para_iter:
+            if not para or para.isspace():
+                if not relative:
+                    offset += len(para)
+                else:
+                    yield ParagraphEOF
+                continue
+
+            if relative and not is_first_pos:
+                yield ParagraphEOF
+            is_first_pos = False
+
+            boundaries = self._rule.apply(para.rstrip(), self.preserve_quote_and_paren)
+
+            for pos in boundaries[1:]:
+                yield offset + pos if not relative else pos
+
+            if not relative:
+                offset += len(para)
+
+    @validate_input
+    def segment(
+        self,
+        source: str | TextIOBase | StreamCleanerStub,
+        *,
+        preserve_whitespace: bool = False,
+    ) -> Generator[str, None, None]:
+        """Split text into sentences.
+
+        Args:
+            source: Plain text string or ``TextIOBase`` stream (e.g., ``StringIO``, opened file).
+            preserve_whitespace: If ``False`` (default), strip leading and
+                trailing whitespace from each sentence.
+
+        Yields:
+            Individual sentences as strings.
+        """
+        if self.verbose:  # pragma: no cover
+            logger.info("Called with preserve_whitespace={}", preserve_whitespace)
+
+        para_iter = (
+            ParagraphStream(source, skip_empty_lines=not preserve_whitespace)
+            if isinstance(source, (str, TextIOBase))
+            else source
+        )
+
+        input_for_detection, input_for_slicing = tee(para_iter)
+        curr_para = None
+        for start, end in self._detect_relative_spans(input_for_detection):
+            if start == 0:
+                curr_para = next(input_for_slicing, None)
+
+            if curr_para is None:
+                break
+
+            sent = curr_para[start:end]
+            if not preserve_whitespace:
+                sent = sent.strip()
+                if not sent:
+                    continue
+            yield sent

yasbd/rules/_template.py

@@ -0,0 +1,57 @@
+from yasbd.rules.base import Rules
+
+
+# Don't remove the fmt guards as they keep compact set formatting
+# fmt: off
+class LangRules(Rules):
+    """Template for adding new language rule modules.
+
+    Copy this file and rename it to ``<lang>.py`` (e.g.
+    ``fr.py``), rename the class to ``<Lang>Rules`` (e.g.
+    ``FrRules``) and override only the sets your language needs
+    (please, not all of them).
+    """
+
+    # Extra sentence terminators used by the language.
+    TERMINATORS = Rules.TERMINATORS | {...}
+
+    # Honorifics and professional abbreviations that should not split sentences.
+    TITLE_ABBRVS = Rules.TITLE_ABBRVS | {...}
+
+    # Country and regional abbreviations written with periods (U.S., E.U., etc.).
+    GEOPOLITICAL_ABBRVS = Rules.GEOPOLITICAL_ABBRVS | {...}
+
+    # Citation and reference abbreviations commonly used mid-text.
+    REFERENCE_ABBRVS = Rules.REFERENCE_ABBRVS | {...}
+
+    # Street and address abbreviations (Ave., Blvd., Rd., etc.).
+    STREET_ABBRVS = Rules.STREET_ABBRVS | {...}
+
+    # Common inline abbreviations that should not end a sentence.
+    MID_SENTENCE_ABBRVS = Rules.MID_SENTENCE_ABBRVS | {...}
+
+    # Names or titles containing "!" that should not trigger sentence breaks.
+    NAMES_WITH_EXCLAMATION = Rules.NAMES_WITH_EXCLAMATION | {...}
+
+    # Month, weekday, and calendar abbreviations.
+    DATE_ABBRVS = Rules.DATE_ABBRVS | {...}
+
+    # Common nouns appearing inside organization or institution names.
+    COMMON_ORG_NOUNS = {...}
+
+    # Frequently occurring sentence starters used as weak boundary hints.
+    COMMON_SENT_STARTERS = {...}
+
+    # -- Mostly useful for unicase or weakly-cased languages --
+
+    # Quotative particles used after speech, thoughts, or labels.
+    QUOTATIVE_PARTICLES = {...}
+
+    # Postpositional case markers that tightly bind an abbreviation to the clause
+    # (Japanese の, Chinese 的, etc.).
+    CASE_MARKERS = {...}
+
+    # Verbs commonly used for dialogue attribution or reported speech.
+    REPORTING_WORDS = {...}
+
+# fmt: on

yasbd/rules/base.py

@@ -0,0 +1,366 @@
+import re  # For simpler patterns
+
+import regex as re2
+
+
+def _build_abbr_pattern(options: set[str]) -> str:
+    """Build a safe escaped regex alternation pattern.
+
+    Returns a never-match pattern if no valid options exist.
+    Ref: https://stackoverflow.com/questions/1723182/a-regex-that-will-never-be-matched-by-anything?
+    """
+    cleaned = [
+        re2.escape(opt.strip())
+        for opt in sorted(options, key=len, reverse=True)
+        if opt.strip()
+    ]
+
+    return "|".join(cleaned) if cleaned else r"(?!)"
+
+
+# fmt: off
+class Rules:
+    TERMINATORS = {"。", "．", ".", "！", "!", "？", "?", "‼", "⁉", "⁈"}
+
+    TITLE_ABBRVS = {
+        # Standard Professional (Universal Latin roots)
+        "dr", "drs", "prof", "hon", "rev", "supt", "insp",
+
+        # Global Social (Overlap across English/Spanish/Portuguese/French)
+        "mr", "mrs", "ms", "st",
+
+        # Military (NATO/International Standardized Ranks)
+        "adm", "brig", "capt", "cmdr", "col", "cpl", "gen", "lt", "maj", "sgt", "pvt",
+
+        # Political/Administrative (Common in Western bureaucracy)
+        "gov", "rep", "sen", "pres"
+    }
+
+    GEOPOLITICAL_ABBRVS = {
+        # North Atlantic / Western Europe
+        "u.s", "u.s.a", "u.k", "e.u",
+        "ｕ．ｓ", "ｕ．ｓ．ａ", "ｕ．ｋ", "ｅ．ｕ",
+
+        # Multilateral / Intergovernmental
+        "u.n", "u.s.s.r",
+        "ｕ．ｎ", "ｕ．ｓ．ｓ．ｒ",
+
+        # Asia / Middle East
+        "u.a.e", "p.r.c", "r.o.k",
+        "ｕ．ａ．ｅ", "ｐ．ｒ．ｃ", "ｒ．ｏ．ｋ",
+    }
+
+    REFERENCE_ABBRVS = {
+        # Publishing / Documents
+        "et al", "app", "apps", "cf", "ext", "fig", "figs", "l", "ll",
+        "n", "nn", "p", "pp", "pag", "pt", "pts", "ref", "refs", "tab",
+        "tbl", "tbls", "v", "vol", "vols",
+
+        # Section / Structure
+        "ann", "art", "arts", "cap", "cl", "cls", "col", "cols", "para",
+        "paras", "sec", "sect", "secs", "subsec",
+
+        # Legal / Numbering
+        "no", "nos", "reg", "regs",
+
+        # Scientific / Math / Technical
+        "approx", "eq", "eqn", "eqs", "est", "ex", "exs",
+
+        # Academic
+        "univ", "s",
+    }
+
+    DATE_ABBRVS = {
+        # Months
+        "jan", "feb", "mar", "apr", "jun", "jul", "sep",
+        "sept", "oct", "nov", "dec", "déc",
+
+        # Day
+        "mon", "tue", "wed", "thu", "fri", "sat", "sun",
+        "lun", "mar", "dom",
+    }
+
+    MID_SENTENCE_ABBRVS = {
+        # Business entity bridges
+        "assoc", "mfg",
+
+        # Bridge/connectors
+        "cf", "eg", "e.g", "ie", "i.e", "vs", "v", "viz", "ibid", "ca", "sc",
+
+        # Street & directional anchors
+        "mt", "dist",
+
+        # General
+        "approx", "est", "intl", "misc",
+    }
+
+    STREET_ABBRVS = {
+        "ave", "blvd", "blv", "ct", "ln", "pl", "rd", "sq", "st", "wy", "way"
+    }
+
+    NAMES_WITH_EXCLAMATION = {
+        # Tech, Corporate Entities, & Major Consumer Brands
+        "Yahoo", "Yum", "Chips Ahoy", "Kahoot", "JOOP", "Walla",
+        "I Can't Believe It's Not Butter", "Pop",
+
+        # Gaming, Media, Animation, & Entertainment
+        "Mamma Mia", "Jeopardy", "Oklahoma", "Oliver", "Shindig",
+        "Hailey's On It", "Airplane", "Osu", "Ha", "VSPO",
+
+        # Geopolitical Quirks / Municipalities
+        "Westward Ho", "Saint-Louis-du-Ha", "Baie-des-Ha",
+
+        # Public Figures, Politics, & Manufacturing Brands
+        "Jeb", "Éxito", "Hey Man", "Basta", "Elliot S"
+    }
+
+    COMMON_ORG_NOUNS = set()
+    COMMON_SENT_STARTERS = set()
+    QUOTATIVE_PARTICLES = set()
+    CASE_MARKERS = set()
+    REPORTING_WORDS = set()
+
+    # https://regex101.com/r/tI9Cmg/2
+    VERTICAL_LIST_START_FINDER = re2.compile(r"(?<=^\s*(?:[\p{L}\p{N}]\.){1,3})(?=\s)")
+
+    # https://regex101.com/r/JYdWZw/4
+    QUOTE_AND_PAREN_FINDER = re2.compile(
+        r"""
+        (?:\p{Pi}|»|(?<=[\s:])(['""])).+?(?:\p{Pf}|«|\1)|  # Quoted text
+        \p{Ps}.+?\p{Pe}  # Parenthesized text
+        """,
+        re2.X,
+    )
+
+    # https://regex101.com/r/0P9f2V/1
+    TOC_LEADER_FINDER = re.compile(r"[^\W_][\s\.]{4,}\d")
+
+    # https://regex101.com/r/ZOZlLb/2/substitution
+    NEWLINE_INSIDE_SENTENCE_FINDER = re2.compile(r"(?<=[,:;)\w\s])\n(?=([a-z(]))")
+
+    _REGEX_CACHED = False
+    # fmt: on
+    def __init__(self):
+        """Initialize rule instance with lazy-compiled regex patterns.
+
+        Patterns are compiled once per class and cached via ``_REGEX_CACHED``.
+        Subclasses can override data constants (abbreviation sets, terminators, etc.)
+        and the classmethod ``_compile_regex_dynamically`` will pick them up.
+        """
+        if not type(self).__dict__.get("_REGEX_CACHED", False):
+            self._compile_regex_dynamically()
+            type(self)._REGEX_CACHED = True
+
+    @classmethod
+    def _compile_regex_dynamically(cls):
+        """Compile language-specific regex patterns."""
+        terminators_pattern = "".join(cls.TERMINATORS)
+        dots_pattern = r"[.．]"
+        title_abbrvs_pattern = _build_abbr_pattern(cls.TITLE_ABBRVS)
+        geopolitical_abbrvs_pattern = _build_abbr_pattern(cls.GEOPOLITICAL_ABBRVS)
+        common_starters_pattern = _build_abbr_pattern(cls.COMMON_SENT_STARTERS)
+
+        # https://regex101.com/r/qBSyU5/12
+        # Handle flattened lists due to messy OCR.
+        cls.HORIZONTAL_LIST_FINDER = re.compile(
+            rf"""
+            (?:   #  Must preceded by
+                ^\s*|     # A string start
+                [:{terminators_pattern}]\s+  # A terminator or double colon + space
+            )
+            (?:[•◦]\s+)?   # Optional bullet point (e.g., • 9.)
+            (?:
+                [-*+]|      #  Markdown style list
+                (?:\d{{1,2}}[.)]{{1,2}}|[a-zA-Z]\))  #  Numbered and alphabetical list (e.g, a\), 34.\), 1.)
+            )
+            (?=\s)  # Must followed by a space
+            """,
+            re.X,
+        )
+
+        # https://regex101.com/r/VMzYsx/9
+        cls.NAIVE_BOUNDARY_FINDER = re2.compile(
+            rf"""
+            # Split if left token is a unicase letter (Always)
+            (?<=\p{{Lo}}\s*[{terminators_pattern}])|
+
+            # Split after any terminators followed by a a newline,
+            # common sentence starter, Space+Upper or unicase letter
+            (?<=[{terminators_pattern}])
+            (?=
+                \s*\n|
+                \s+(?:[^\p{{Ll}}]|
+                \s+(?<!\.\.)(?i:{common_starters_pattern})\b)|
+                \s*\p{{Lo}}
+            )|
+
+            # Split at transition between Latin letters separate by alien
+            (?<=[\p{{LU}}\p{{Ll}}][​。！？।])(?=[\p{{Lu}}])|
+
+            # Cluster of terminators (e.g hello!!! r u ok?)
+            (?<=[{terminators_pattern.replace('.', '')}]{{2,}})(?=\s)
+            """,
+            re2.X,
+        )
+
+        # fmt: off
+        # Faster than one big regex
+        # https://regex101.com/r/svyCoU/18
+        cls.MID_SENTENCE_FINDER_LST = [
+            # Title abbrv or initialisms (e.g., Dr. Paul)
+            re.compile(rf"\b(?i:{title_abbrvs_pattern}){dots_pattern}"),
+
+            # Geopolitical abbrv is followed by a common org noun (e.g., U.S.A Army)
+            re.compile(rf"""
+                \b(?i:{geopolitical_abbrvs_pattern}){dots_pattern}
+                (?=
+                    \s*(?:{_build_abbr_pattern(cls.CASE_MARKERS)})|
+                    \s+(?:{_build_abbr_pattern(cls.COMMON_ORG_NOUNS)})
+                )
+                """, re.X
+            ),
+
+            # Abbrv that NEVER ends a sentence
+            re.compile(
+               rf"\b(?i:{_build_abbr_pattern(cls.MID_SENTENCE_ABBRVS)}){dots_pattern}"
+            ),
+
+            # References abbrv followed by a number, a letter or opened paren (e.g., to p. 55, app. A)
+            re2.compile(rf"""
+                \b(?i:{_build_abbr_pattern(cls.REFERENCE_ABBRVS)}){dots_pattern}
+                (?=\s+(?:\(|\p{{Lu}}\b|\p{{N}}|[IVXLCDM]+))
+                """, re2.X
+            ),
+
+            # Date abbrv followed by a number
+            re2.compile(
+                rf"\b(?i:{_build_abbr_pattern(cls.DATE_ABBRVS)}){dots_pattern}(?=\s+\p{{N}})"
+            ),
+
+            # Streets/Acronyms/Exclamations words (e.g., Yahoo!, A.B. Holding, Ave. Central)
+            # excluding geopolitical ones not followed by a common starters
+            re2.compile(rf"""
+                (?:\p{{Lu}}\.){{2,}}(?<!(?i:{geopolitical_abbrvs_pattern}))
+                (?!\s+(?:{common_starters_pattern})\b)
+                """, re2.X
+            ),
+            re.compile(rf"""
+                (?:
+                    \b(?i:{_build_abbr_pattern(cls.STREET_ABBRVS)}){dots_pattern}|
+                    (?i:{_build_abbr_pattern(cls.NAMES_WITH_EXCLAMATION)})[! ！‼]
+                )
+                (?!\s+(?:{common_starters_pattern})\b)
+               """, re.X
+            ),
+
+            # Collapsed middle name (e.g, Jonas E. Smith)
+            re2.compile(rf"\s\b(?:\p{{Lu}}){dots_pattern}(?=\s)"),
+        ]
+        # fmt: on
+
+        # https://regex101.com/r/EGkRU8/6
+        cls.QUOTE_AND_PAREN_END_FINDER = re2.compile(
+            rf"""
+            (?<=
+                [{terminators_pattern}]   # A terminator
+                (?:'\s|["”]|\s*[»\p{{Pf}}\p{{Pe}}])     # Closing quotes/parens
+            )
+            (?!  # NOT followed by any continuation markers, punctuation, or space+lowercase
+                \s*\p{{Po}}|
+                {_build_abbr_pattern(cls.QUOTATIVE_PARTICLES | cls.REPORTING_WORDS)}|
+                \s+[\p{{Ll}}]
+            )
+            """,
+            re2.X,
+        )
+
+        # https://regex101.com/r/ffqwjh/2
+        cls.CONTIGUOUS_TERMINATORS_FINDER = re.compile(rf"(?:\s*+[{terminators_pattern}]){{2,}}")
+
+    def _remove_quote_and_paren_spans(
+        self,
+        main_boundaries: set[int],
+        text: str,
+        preserve_quote_and_paren: bool,
+    ) -> None:
+        """Remove boundaries inside quoted/parenthesised spans."""
+        if preserve_quote_and_paren:
+            # Ignore first pos to preserve splits before opening quote/paren,
+            # especially for non-whitespace languages
+            main_boundaries.difference_update(
+                pos
+                for m in self.QUOTE_AND_PAREN_FINDER.finditer(text)
+                for pos in range(m.start() + 1, m.end())
+            )
+
+            main_boundaries.update(
+                m.end() for m in self.QUOTE_AND_PAREN_END_FINDER.finditer(text)
+            )
+
+    def _remove_toc_spans(
+        self, main_boundaries: set[int], text: str
+    ) -> None:
+        """Remove boundaries inside TOC leader runs."""
+        if "..." in text:
+            for m in self.TOC_LEADER_FINDER.finditer(text):
+                main_boundaries.difference_update(range(*m.span()))
+
+    def _adjust_list_boundaries(self, main_boundaries: set[int], text: str) -> None:
+        """Remove and re-align boundaries around list markers."""
+        horiz_matches = list(self.HORIZONTAL_LIST_FINDER.finditer(text))
+        if len(horiz_matches) >= 2:
+            main_boundaries.difference_update(m.end() for m in horiz_matches)
+            # Shift boundaries the pointer back (1.\)| => |1.\), a. | => |a. ) to correctly
+            # terminate the preceding sentence before flattened horizontal list.
+            main_boundaries.update(m.start() + 1 for m in horiz_matches if m.start())
+
+        main_boundaries.difference_update(
+            m.end() for m in self.VERTICAL_LIST_START_FINDER.finditer(text)
+        )
+
+    def apply(
+        self,
+        text: str,
+        preserve_quote_and_paren: bool,
+    ) -> list[int]:
+        """Detect sentence boundaries in *text*.
+
+        Two-pass algorithm:
+        1. Collect boundary candidates from punctuation positions.
+        2. Remove false alarms (mid-sentence abbreviations, ellipsis,
+           quote/paren spans, list markers).
+
+        Args:
+            text: A string to find sentence boundaries in.
+            preserve_quote_and_paren: If ``True``, suppress boundaries
+               inside quote and parenthesis spans.
+
+        Returns:
+            Sorted list of character offsets at which sentences end.
+        """
+        text = self.NEWLINE_INSIDE_SENTENCE_FINDER.sub(" ", text)
+        main_boundaries = {
+            m.end() for m in self.NAIVE_BOUNDARY_FINDER.finditer(text)
+        }
+
+        # -- Remove false alarms --
+        main_boundaries.difference_update(
+            m.end() for pat in self.MID_SENTENCE_FINDER_LST
+            for m in pat.finditer(text)
+        )
+        self._remove_quote_and_paren_spans(
+            main_boundaries, text, preserve_quote_and_paren
+        )
+        self._remove_toc_spans(main_boundaries, text)
+        self._adjust_list_boundaries(main_boundaries, text)
+
+        # Remove contiguous term pos except last one (e.g., Hello! !!   !! )
+        main_boundaries.difference_update(
+            *(
+                range(m.start(), m.end() - 1)
+                for m in self.CONTIGUOUS_TERMINATORS_FINDER.finditer(text)
+            )
+        )
+
+        main_boundaries.update({0, len(text)})
+        return sorted(main_boundaries)