Autarkia-Textkit 0.1.0__py3-none-any.whl

autarkia_textkit/__init__.py

@@ -0,0 +1,100 @@
+"""Autarkia-Textkit: Chinese chat text segmentation and set similarity."""
+
+from .clean import (
+    clean_chat_text,
+    is_ellipsis_only,
+    split_for_display,
+    strip_ai_timestamp,
+    strip_trailing_punctuation,
+)
+from .emoji import EMOJI_RANGES, is_emoji_char
+from .force import (
+    force_split_long_sentence,
+    has_unclosed_inline_span,
+    repair_broken_inline_spans,
+)
+from .merge import (
+    HAS_JIEBA,
+    glue_cjk_ascii_spaces,
+    is_continuation_start,
+    looks_like_manual_space_split,
+    merge_chunks_to_limit,
+    merge_continuation_chunks,
+    merge_space_chunks_to_limit,
+)
+from .numbered import (
+    find_numbered_list_starts,
+    is_numbered_item_start,
+    normalize_numbered_list,
+)
+from .rules import (
+    AI_TIMESTAMP_PATTERN,
+    AI_TIMESTAMP_PATTERN_SOURCE,
+    CONTINUATION_ADVERBS,
+    CONTINUATION_ENDINGS,
+    ELLIPSIS_MERGE_PREFIX_LIMIT,
+    EXPLICIT_SPACE_BOUNDARY_ENDINGS,
+    HARD_BUBBLE_BOUNDARY_ENDINGS,
+    PUNCTUATION_FOR_SPACE_GUARD,
+    TRAILING_PUNCT_CHARS,
+)
+from .segmenter import split_chat_message
+from .similarity import (
+    char_ngrams,
+    cjk_bigrams,
+    cjk_chars,
+    containment,
+    intersection_over_max,
+    jaccard,
+    token_ngrams,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "__version__",
+    # rules
+    "AI_TIMESTAMP_PATTERN",
+    "AI_TIMESTAMP_PATTERN_SOURCE",
+    "CONTINUATION_ADVERBS",
+    "CONTINUATION_ENDINGS",
+    "ELLIPSIS_MERGE_PREFIX_LIMIT",
+    "EXPLICIT_SPACE_BOUNDARY_ENDINGS",
+    "HARD_BUBBLE_BOUNDARY_ENDINGS",
+    "PUNCTUATION_FOR_SPACE_GUARD",
+    "TRAILING_PUNCT_CHARS",
+    # emoji
+    "EMOJI_RANGES",
+    "is_emoji_char",
+    # cleaning / display
+    "clean_chat_text",
+    "is_ellipsis_only",
+    "split_for_display",
+    "strip_ai_timestamp",
+    "strip_trailing_punctuation",
+    # splitting
+    "split_chat_message",
+    "force_split_long_sentence",
+    "repair_broken_inline_spans",
+    "has_unclosed_inline_span",
+    # numbered lists
+    "find_numbered_list_starts",
+    "is_numbered_item_start",
+    "normalize_numbered_list",
+    # merging
+    "HAS_JIEBA",
+    "is_continuation_start",
+    "glue_cjk_ascii_spaces",
+    "looks_like_manual_space_split",
+    "merge_chunks_to_limit",
+    "merge_continuation_chunks",
+    "merge_space_chunks_to_limit",
+    # similarity
+    "intersection_over_max",
+    "containment",
+    "jaccard",
+    "char_ngrams",
+    "token_ngrams",
+    "cjk_chars",
+    "cjk_bigrams",
+]

autarkia_textkit/clean.py

@@ -0,0 +1,152 @@
+"""Cleaning and display grouping: timestamps, trailing punctuation, blocks."""
+
+from __future__ import annotations
+
+import re
+
+from .rules import (
+    AI_TIMESTAMP_PATTERN,
+    TRAILING_PUNCT_CHARS,
+    _LATEX_MARKER_RE,
+    _MARKDOWN_BLOCK_START_RE,
+    _MARKDOWN_TABLE_LINE_RE,
+)
+from .segmenter import split_chat_message
+
+_ELLIPSIS_ONLY_RE = re.compile(r"^[\s.…]+$")
+
+
+def is_ellipsis_only(text: str) -> bool:
+    """Whether the whole text consists only of ellipses and whitespace."""
+    stripped = str(text or "").strip()
+    if not stripped or not _ELLIPSIS_ONLY_RE.match(stripped):
+        return False
+    return "…" in stripped or stripped.count(".") >= 3
+
+
+def strip_ai_timestamp(text: str) -> str:
+    """Remove model-emitted history timestamps anywhere in the text."""
+    return AI_TIMESTAMP_PATTERN.sub("", str(text or "")).strip()
+
+
+def strip_trailing_punctuation(text: str) -> str:
+    """Remove trailing punctuation without touching ellipses."""
+    text = str(text or "")
+    if is_ellipsis_only(text):
+        return text.strip()
+    text = re.sub(r"[。.]\s*(?=[（(])", "", text)
+    text = re.sub(r"[。.]+\s*([）)])", r"\1", text)
+    text = text.rstrip(TRAILING_PUNCT_CHARS).strip()
+    text = re.sub(r"[。.]+$", "", text)
+    return text.strip()
+
+
+def clean_chat_text(text: str) -> str:
+    """Timestamp removal + trailing punctuation removal."""
+    return strip_trailing_punctuation(strip_ai_timestamp(text))
+
+
+def _group_display_lines(text: str) -> list[tuple[str, bool]]:
+    """Split into ``(text, atomic)`` groups by Markdown block structure.
+
+    Fenced code, ``$$`` blocks and table runs stay whole; single-line block
+    markers (heading/list/quote) stay on their line. Ordinary lines still go
+    through :func:`split_chat_message`.
+    """
+    lines = text.split("\n")
+    groups: list[tuple[str, bool]] = []
+    index = 0
+    total = len(lines)
+
+    while index < total:
+        line = lines[index]
+        stripped = line.strip()
+        if not stripped:
+            index += 1
+            continue
+
+        fence = None
+        if stripped.startswith("```"):
+            fence = "```"
+        elif stripped.startswith("~~~"):
+            fence = "~~~"
+        if fence is not None:
+            buffer: list[str] = []
+            cursor = index
+            while cursor < total:
+                buffer.append(lines[cursor])
+                cursor += 1
+                if cursor > index + 1 and lines[cursor - 1].strip().startswith(fence):
+                    break
+            groups.append(("\n".join(buffer), True))
+            index = cursor
+            continue
+
+        if stripped == "$$":
+            buffer = []
+            cursor = index
+            while cursor < total:
+                buffer.append(lines[cursor])
+                cursor += 1
+                if cursor > index + 1 and lines[cursor - 1].strip() == "$$":
+                    break
+            groups.append(("\n".join(buffer), True))
+            index = cursor
+            continue
+
+        if _MARKDOWN_TABLE_LINE_RE.match(line):
+            buffer = []
+            cursor = index
+            while cursor < total and _MARKDOWN_TABLE_LINE_RE.match(lines[cursor]):
+                buffer.append(lines[cursor].strip())
+                cursor += 1
+            groups.append(("\n".join(buffer), True))
+            index = cursor
+            continue
+
+        atomic = bool(
+            _MARKDOWN_BLOCK_START_RE.search(stripped) or _LATEX_MARKER_RE.search(stripped)
+        )
+        groups.append((stripped, atomic))
+        index += 1
+
+    return groups
+
+
+def split_for_display(
+    text: str,
+    max_len: int = 150,
+    comma_split_prob: float = 0.2,
+    min_split_len: int = 40,
+    rng=None,
+) -> list[str]:
+    """Clean, group by structure, split each group, clean each bubble again."""
+    cleaned = clean_chat_text(text)
+    if not cleaned:
+        return []
+    result: list[str] = []
+    for group_text, atomic in _group_display_lines(cleaned):
+        if atomic:
+            if group_text.strip():
+                result.append(group_text)
+            continue
+        for chunk in split_chat_message(
+            group_text,
+            max_len,
+            comma_split_prob,
+            min_split_len,
+            rng=rng,
+        ):
+            chunk = strip_trailing_punctuation(chunk)
+            if chunk:
+                result.append(chunk)
+    return result
+
+
+__all__ = [
+    "is_ellipsis_only",
+    "strip_ai_timestamp",
+    "strip_trailing_punctuation",
+    "clean_chat_text",
+    "split_for_display",
+]

autarkia_textkit/emoji.py

@@ -0,0 +1,85 @@
+"""Emoji ranges; a punctuation-adjacent emoji must stay with the current bubble."""
+
+from __future__ import annotations
+
+# Precise ranges, so Chinese punctuation is never mistaken for an emoji.
+EMOJI_RANGES = [
+    (0x1F600, 0x1F64F),
+    (0x1F300, 0x1F5FF),
+    (0x1F680, 0x1F6FF),
+    (0x1F1E0, 0x1F1FF),
+    (0x1F900, 0x1F9FF),
+    (0x1FA00, 0x1FA6F),
+    (0x1FA70, 0x1FAFF),
+    (0x2600, 0x26FF),
+    (0x2700, 0x27BF),
+    (0x2300, 0x23FF),
+    (0x2B50, 0x2B55),
+    (0x203C, 0x2049),
+    (0x2122, 0x2122),
+    (0x2194, 0x21AA),
+    (0x231A, 0x231B),
+    (0x23E9, 0x23F3),
+    (0x23F8, 0x23FA),
+    (0x25AA, 0x25AB),
+    (0x25B6, 0x25C0),
+    (0x25FB, 0x25FE),
+    (0x2614, 0x2615),
+    (0x2648, 0x2653),
+    (0x267F, 0x267F),
+    (0x2693, 0x2693),
+    (0x26A1, 0x26A1),
+    (0x26AA, 0x26AB),
+    (0x26BD, 0x26BE),
+    (0x26C4, 0x26C5),
+    (0x26CE, 0x26CE),
+    (0x26D4, 0x26D4),
+    (0x26EA, 0x26EA),
+    (0x26F2, 0x26F3),
+    (0x26F5, 0x26F5),
+    (0x26FA, 0x26FA),
+    (0x26FD, 0x26FD),
+    (0x2702, 0x2702),
+    (0x2705, 0x2705),
+    (0x2708, 0x270D),
+    (0x270F, 0x270F),
+    (0x2712, 0x2712),
+    (0x2714, 0x2714),
+    (0x2716, 0x2716),
+    (0x271D, 0x271D),
+    (0x2721, 0x2721),
+    (0x2728, 0x2728),
+    (0x2733, 0x2734),
+    (0x2744, 0x2744),
+    (0x2747, 0x2747),
+    (0x274C, 0x274C),
+    (0x274E, 0x274E),
+    (0x2753, 0x2755),
+    (0x2757, 0x2757),
+    (0x2763, 0x2764),
+    (0x2795, 0x2797),
+    (0x27A1, 0x27A1),
+    (0x27B0, 0x27B0),
+    (0x27BF, 0x27BF),
+    (0x2934, 0x2935),
+    (0x2B05, 0x2B07),
+    (0x2B1B, 0x2B1C),
+    (0x2B50, 0x2B50),
+    (0x2B55, 0x2B55),
+    (0x3030, 0x3030),
+    (0x303D, 0x303D),
+    (0x3297, 0x3297),
+    (0x3299, 0x3299),
+    (0xE0020, 0xE007F),
+]
+
+
+def is_emoji_char(ch: str) -> bool:
+    """Whether one character falls in an emoji range."""
+    if not ch:
+        return False
+    code_point = ord(ch)
+    return any(start <= code_point <= end for start, end in EMOJI_RANGES)
+
+
+__all__ = ["EMOJI_RANGES", "is_emoji_char"]

autarkia_textkit/force.py

@@ -0,0 +1,128 @@
+"""Forced splitting of long sentences and inline Markdown span repair."""
+
+from __future__ import annotations
+
+
+def has_unclosed_inline_span(text: str) -> bool:
+    """Whether ``**bold**`` / `` `code` `` pairs were split apart.
+
+    Only unescaped ``**`` and backticks count; escaped characters are skipped.
+    An odd count means an unclosed pair.
+    """
+    count = 0
+    index = 0
+    length = len(text)
+    while index < length:
+        char = text[index]
+        if char == "\\":
+            index += 2
+            continue
+        if text.startswith("**", index):
+            count += 1
+            index += 2
+            continue
+        if char == "`":
+            count += 1
+            index += 1
+            continue
+        index += 1
+    return count % 2 == 1
+
+
+def repair_broken_inline_spans(chunks: list[str]) -> list[str]:
+    """Merge following chunks until every inline span is balanced.
+
+    Joining without a separator is intentional: the splitter removed the
+    sentence punctuation at the break, so direct concatenation restores a
+    valid ``**...**`` pair.
+    """
+    if len(chunks) < 2:
+        return chunks
+    result: list[str] = []
+    buffer: str | None = None
+    for chunk in chunks:
+        candidate = chunk if buffer is None else buffer + chunk
+        if has_unclosed_inline_span(candidate):
+            buffer = candidate
+        else:
+            result.append(candidate)
+            buffer = None
+    if buffer is not None:
+        result.append(buffer)
+    return result
+
+
+def force_split_long_sentence(s: str, max_len: int = 100, max_chunks: int = 7) -> list[str]:
+    """Split an over-long sentence near punctuation, bounded by ``max_chunks``."""
+    if len(s) <= max_len:
+        return [s]
+
+    result: list[str] = []
+    target_chunk_size = max_len
+    position = 0
+    while position < len(s):
+        remaining_length = len(s) - position
+        if remaining_length <= max_len:
+            if remaining_length > 0:
+                result.append(s[position:].strip())
+            break
+
+        target_end = min(position + target_chunk_size, len(s))
+        split_char_found = False
+        for index in range(target_end - 1, max(position + max_len // 2, position), -1):
+            if s[index] in [",", "，", " ", "\u3000", ".", "。", ";", "；", "!", "！", "?", "？"]:
+                chunk = s[position : index + 1].strip()
+                if chunk:
+                    result.append(chunk)
+                    position = index + 1
+                    split_char_found = True
+                    break
+
+        if not split_char_found:
+            for index in range(target_end, min(target_end + 20, len(s))):
+                if s[index] in [",", "，", " ", "\u3000", ".", "。", ";", "；", "!", "！", "?", "？"]:
+                    chunk = s[position : index + 1].strip()
+                    if chunk:
+                        result.append(chunk)
+                        position = index + 1
+                        split_char_found = True
+                        break
+
+        if not split_char_found:
+            chunk = s[position:target_end].strip()
+            if chunk:
+                result.append(chunk)
+                position = target_end
+
+    if len(result) > max_chunks:
+        avg_chunk_size = len(s) // max_chunks
+        new_result: list[str] = []
+        current_chunk = ""
+        for chunk in result:
+            if len(current_chunk) + len(chunk) <= avg_chunk_size + 20:
+                current_chunk += chunk
+            else:
+                if current_chunk:
+                    new_result.append(current_chunk.strip())
+                current_chunk = chunk
+        if current_chunk:
+            new_result.append(current_chunk.strip())
+
+        if len(new_result) <= max_chunks:
+            result = new_result
+        else:
+            result = []
+            step = len(s) // max_chunks
+            for index in range(0, len(s), step):
+                chunk = s[index : index + step].strip()
+                if chunk:
+                    result.append(chunk)
+
+    return result if result else [s]
+
+
+__all__ = [
+    "has_unclosed_inline_span",
+    "repair_broken_inline_spans",
+    "force_split_long_sentence",
+]

autarkia_textkit/merge.py

@@ -0,0 +1,218 @@
+"""Continuation detection and chunk merging.
+
+With ``jieba`` installed, the first non-punctuation token's POS tag decides
+whether a chunk begins with a continuation (conjunction ``c`` or an adverb
+stem like 再/又/还/更). Without ``jieba`` the library falls back to a compact
+conjunction prefix list, so the package stays dependency-free by default.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+
+from .rules import (
+    CONTINUATION_ADVERBS,
+    CONTINUATION_ENDINGS,
+    EXPLICIT_SPACE_BOUNDARY_ENDINGS,
+    PUNCTUATION_FOR_SPACE_GUARD,
+)
+
+try:  # optional dependency
+    import jieba
+    import jieba.posseg as pseg
+
+    HAS_JIEBA = True
+    jieba.setLogLevel(logging.INFO)
+except ImportError:  # pragma: no cover - exercised only without the extra
+    pseg = None
+    HAS_JIEBA = False
+
+# Fallback prefixes used when jieba is unavailable. Kept intentionally small:
+# they cover the conjunctions/continuations that actually appear at bubble
+# starts, without pretending to be a full POS tagger.
+_CONTINUATION_PREFIXES = (
+    "然后",
+    "而且",
+    "并且",
+    "但是",
+    "不过",
+    "所以",
+    "因为",
+    "虽然",
+    "如果",
+    "要是",
+    "不然",
+    "否则",
+    "于是",
+    "接着",
+    "随后",
+    "同时",
+    "另外",
+    "再说",
+    "还有",
+    "反正",
+    "毕竟",
+    "其实",
+    "只是",
+    "就是",
+) + tuple(CONTINUATION_ADVERBS)
+
+
+_PUNCTUATION_CHARS = "".join(PUNCTUATION_FOR_SPACE_GUARD)
+
+
+def is_continuation_start(text: str) -> bool:
+    """Whether the text starts with a continuation word.
+
+    Leading punctuation is skipped. With jieba, the first token's POS decides;
+    otherwise a prefix list is used.
+    """
+    if not text or not text.strip():
+        return False
+    stripped = text.strip()[:30]
+
+    if HAS_JIEBA and pseg is not None:
+        for word, flag in pseg.cut(stripped):
+            if not word.strip():
+                continue
+            if flag in ("x", "w"):
+                continue
+            if flag == "c":
+                return True
+            return any(word.startswith(adverb) for adverb in CONTINUATION_ADVERBS)
+
+    cleaned = stripped.lstrip(_PUNCTUATION_CHARS)
+    return any(cleaned.startswith(prefix) for prefix in _CONTINUATION_PREFIXES)
+
+
+def merge_chunks_to_limit(chunks: list[str], max_chunks: int = 7) -> list[str]:
+    """Merge the shortest adjacent pairs until at most ``max_chunks`` remain."""
+    if len(chunks) <= max_chunks:
+        return chunks
+
+    result = list(chunks)
+    for _ in range(len(result) - max_chunks):
+        min_len = float("inf")
+        merge_index = -1
+        for index in range(len(result) - 1):
+            combined_len = len(result[index]) + len(result[index + 1])
+            if combined_len < min_len:
+                min_len = combined_len
+                merge_index = index
+        if merge_index < 0:
+            break
+        result[merge_index] = result[merge_index] + result[merge_index + 1]
+        result.pop(merge_index + 1)
+    return result
+
+
+def merge_continuation_chunks(
+    chunks: list[str], max_merge_len: int = 300, min_split_len: int = 40
+) -> list[str]:
+    """Merge unnatural breaks caused by continuation words or open endings."""
+    if len(chunks) <= 1:
+        return chunks
+
+    result = [chunks[0]]
+    for index in range(1, len(chunks)):
+        prev = result[-1]
+        curr = chunks[index]
+        curr_stripped = curr.strip()
+        prev_stripped = prev.strip()
+
+        should_merge = False
+        if is_continuation_start(curr_stripped):
+            prev_ends_with_sentence_punct = bool(
+                prev_stripped
+                and any(
+                    prev_stripped.endswith(ending)
+                    for ending in EXPLICIT_SPACE_BOUNDARY_ENDINGS
+                )
+            )
+            if not prev_ends_with_sentence_punct and len(prev_stripped) < min_split_len:
+                should_merge = True
+
+        if prev_stripped and any(
+            prev_stripped.endswith(ending) for ending in CONTINUATION_ENDINGS
+        ):
+            should_merge = True
+
+        if should_merge and len(prev) + len(curr) <= max_merge_len:
+            result[-1] = prev + curr
+        else:
+            result.append(curr)
+    return result
+
+
+# A space between a CJK character and a Latin/digit/Greek/math character is
+# typography, not a manual bubble separator.
+_CJK_ASCII_SPACE_RE = re.compile(
+    r"(?<=[\u4e00-\u9fff])[ \t]+"
+    r"(?=[0-9A-Za-z\u00c0-\u024f\u0370-\u03ff\u0400-\u04ff"
+    r"\u1e00-\u1eff\u2070-\u209f\u2150-\u218f\u2200-\u22ff])"
+    r"|(?<=[0-9A-Za-z\u00c0-\u024f\u0370-\u03ff\u0400-\u04ff"
+    r"\u1e00-\u1eff\u2070-\u209f\u2150-\u218f\u2200-\u22ff])[ \t]+"
+    r"(?=[\u4e00-\u9fff])"
+)
+
+
+def glue_cjk_ascii_spaces(text: str) -> str:
+    """Remove typographic spaces between CJK and Latin/number/math characters."""
+    return _CJK_ASCII_SPACE_RE.sub("", str(text or ""))
+
+
+def looks_like_manual_space_split(text: str) -> bool:
+    """Whether punctuation-free text looks like manually space-separated bubbles."""
+    normalized = str(text or "").strip()
+    if not normalized or "\n" in normalized:
+        return False
+    if any(char in PUNCTUATION_FOR_SPACE_GUARD for char in normalized):
+        return False
+
+    glued = glue_cjk_ascii_spaces(normalized)
+    parts = [part.strip() for part in re.split(r"\s+", glued) if part.strip()]
+    if len(parts) < 3:
+        return False
+
+    cjk_chars = re.findall(r"[\u4e00-\u9fff]", glued)
+    if len(cjk_chars) < max(6, len(glued.replace(" ", "")) // 3):
+        return False
+    if any(len(part) > 12 for part in parts):
+        return False
+
+    avg_len = sum(len(part) for part in parts) / max(1, len(parts))
+    return avg_len <= 8
+
+
+def merge_space_chunks_to_limit(chunks: list[str], max_chunks: int = 6) -> list[str]:
+    """Re-merge space-separated chunks until the bubble count fits."""
+    cleaned = [str(chunk or "").strip() for chunk in chunks if str(chunk or "").strip()]
+    if len(cleaned) <= max_chunks:
+        return cleaned
+
+    result = list(cleaned)
+    while len(result) > max_chunks:
+        min_len = float("inf")
+        merge_index = -1
+        for index in range(len(result) - 1):
+            combined_len = len(result[index]) + len(result[index + 1])
+            if combined_len < min_len:
+                min_len = combined_len
+                merge_index = index
+        if merge_index < 0:
+            break
+        result[merge_index] = f"{result[merge_index]} {result[merge_index + 1]}".strip()
+        result.pop(merge_index + 1)
+    return result
+
+
+__all__ = [
+    "HAS_JIEBA",
+    "is_continuation_start",
+    "merge_chunks_to_limit",
+    "merge_continuation_chunks",
+    "glue_cjk_ascii_spaces",
+    "looks_like_manual_space_split",
+    "merge_space_chunks_to_limit",
+]