chatgpt-md-converter 0.3.12__py3-none-any.whl → 0.4.0b1__py3-none-any.whl

chatgpt_md_converter/__init__.py

@@ -1,5 +1,14 @@
 from .html_splitter import split_html_for_telegram
 from .html_to_markdown import html_to_telegram_markdown
+from .telegram_entities import (EntityType, TelegramEntity,
+                                telegram_format_entities)
 from .telegram_formatter import telegram_format
 
-__all__ = ["telegram_format", "split_html_for_telegram", "html_to_telegram_markdown"]
+__all__ = [
+    "telegram_format",
+    "telegram_format_entities",
+    "TelegramEntity",
+    "EntityType",
+    "split_html_for_telegram",
+    "html_to_telegram_markdown",
+]

chatgpt_md_converter/telegram_entities/__init__.py

@@ -0,0 +1,68 @@
+"""
+Telegram entity conversion module.
+
+This module provides functions to convert Markdown text to Telegram's
+native entity format (plain text + MessageEntity objects).
+"""
+
+from typing import List, Tuple
+
+from .entity import EntityType, TelegramEntity
+from .parser import parse_entities
+
+
+def telegram_format_entities(text: str) -> Tuple[str, List[dict]]:
+    """
+    Convert Markdown text to Telegram format with entities.
+
+    This function parses Markdown syntax and returns plain text along with
+    a list of entity dictionaries suitable for the Telegram Bot API.
+
+    Supported Markdown elements:
+    - **bold**
+    - *italic* or _italic_
+    - __underline__
+    - ~~strikethrough~~
+    - ||spoiler||
+    - `inline code`
+    - ```language
+      code blocks
+      ```
+    - [link text](url)
+    - > blockquotes
+    - >** expandable blockquotes
+    - # Headings (converted to bold)
+    - Lists with - or *
+
+    Args:
+        text: Markdown-formatted text
+
+    Returns:
+        Tuple of (plain_text, entities) where:
+        - plain_text: Text with all Markdown markers removed
+        - entities: List of dicts with 'type', 'offset', 'length' keys
+          (plus 'url' for links, 'language' for code blocks)
+
+    Example:
+        >>> text, entities = telegram_format_entities("**Hello** world!")
+        >>> print(text)
+        Hello world!
+        >>> print(entities)
+        [{'type': 'bold', 'offset': 0, 'length': 5}]
+
+        # Use with python-telegram-bot:
+        await bot.send_message(chat_id, text=text, entities=entities)
+
+        # Use with aiogram:
+        await message.answer(text, entities=entities)
+    """
+    plain_text, entity_objects = parse_entities(text)
+    return plain_text, [e.to_dict() for e in entity_objects]
+
+
+__all__ = [
+    "telegram_format_entities",
+    "TelegramEntity",
+    "EntityType",
+    "parse_entities",
+]

chatgpt_md_converter/telegram_entities/entity.py

@@ -0,0 +1,64 @@
+"""Telegram entity data structures."""
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Optional
+
+
+class EntityType(Enum):
+    """Telegram MessageEntity types."""
+
+    BOLD = "bold"
+    ITALIC = "italic"
+    UNDERLINE = "underline"
+    STRIKETHROUGH = "strikethrough"
+    SPOILER = "spoiler"
+    CODE = "code"
+    PRE = "pre"
+    TEXT_LINK = "text_link"
+    BLOCKQUOTE = "blockquote"
+    EXPANDABLE_BLOCKQUOTE = "expandable_blockquote"
+
+
+@dataclass
+class TelegramEntity:
+    """
+    Represents a Telegram MessageEntity.
+
+    Attributes:
+        type: The entity type (bold, italic, code, etc.)
+        offset: Start position in UTF-16 code units
+        length: Length in UTF-16 code units
+        url: URL for TEXT_LINK entities
+        language: Programming language for PRE (code block) entities
+    """
+
+    type: EntityType
+    offset: int
+    length: int
+    url: Optional[str] = None
+    language: Optional[str] = None
+
+    def to_dict(self) -> dict:
+        """Convert to dict for JSON serialization / Telegram API."""
+        result = {
+            "type": self.type.value,
+            "offset": self.offset,
+            "length": self.length,
+        }
+        if self.url is not None:
+            result["url"] = self.url
+        if self.language is not None:
+            result["language"] = self.language
+        return result
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "TelegramEntity":
+        """Create entity from dictionary."""
+        return cls(
+            type=EntityType(data["type"]),
+            offset=data["offset"],
+            length=data["length"],
+            url=data.get("url"),
+            language=data.get("language"),
+        )

chatgpt_md_converter/telegram_entities/extractors/__init__.py

@@ -0,0 +1,13 @@
+"""Entity extractors for different Markdown elements."""
+
+from .blockquotes import extract_blockquote_entities
+from .headings import extract_heading_entities
+from .inline import extract_inline_formatting_entities
+from .links import extract_link_entities
+
+__all__ = [
+    "extract_inline_formatting_entities",
+    "extract_link_entities",
+    "extract_blockquote_entities",
+    "extract_heading_entities",
+]

chatgpt_md_converter/telegram_entities/extractors/blockquotes.py

@@ -0,0 +1,117 @@
+"""Blockquote entity extraction."""
+
+import re
+from typing import List, Tuple
+
+from ..entity import EntityType, TelegramEntity
+
+# Pattern for regular blockquotes: > text
+_BLOCKQUOTE_LINE_PATTERN = re.compile(r"^>(?!\*\*)\s?(.*)$", re.MULTILINE)
+
+# Pattern for expandable blockquotes: >** text or **> text
+_EXPANDABLE_BLOCKQUOTE_PATTERN = re.compile(
+    r"^(?:>\*\*|\*\*>)\s?(.*)$", re.MULTILINE
+)
+
+
+def extract_blockquote_entities(text: str) -> Tuple[str, List[TelegramEntity]]:
+    """
+    Extract blockquotes and return plain text with BLOCKQUOTE entities.
+
+    Handles both regular (>) and expandable (>** or **>) blockquotes.
+    Consecutive blockquote lines are combined into a single entity.
+
+    Args:
+        text: Input text with blockquote markers
+
+    Returns:
+        Tuple of (text_without_markers, list_of_entities)
+    """
+    entities: List[TelegramEntity] = []
+
+    # First, handle expandable blockquotes
+    result_parts: List[str] = []
+
+    # Find all expandable blockquote lines and group consecutive ones
+    lines = text.split("\n")
+    i = 0
+    current_offset = 0
+
+    while i < len(lines):
+        line = lines[i]
+
+        # Check for expandable blockquote
+        exp_match = _EXPANDABLE_BLOCKQUOTE_PATTERN.match(line)
+        if exp_match:
+            # Collect consecutive expandable blockquote lines
+            quote_lines = []
+
+            while i < len(lines):
+                m = _EXPANDABLE_BLOCKQUOTE_PATTERN.match(lines[i])
+                if m:
+                    quote_lines.append(m.group(1))
+                    i += 1
+                else:
+                    break
+
+            quote_content = "\n".join(quote_lines)
+            quote_offset = current_offset
+            current_offset += len(quote_content) + (1 if i < len(lines) else 0)
+
+            result_parts.append(quote_content)
+            if i < len(lines):
+                result_parts.append("\n")
+
+            entities.append(
+                TelegramEntity(
+                    type=EntityType.EXPANDABLE_BLOCKQUOTE,
+                    offset=quote_offset,
+                    length=len(quote_content),
+                )
+            )
+            continue
+
+        # Check for regular blockquote
+        reg_match = _BLOCKQUOTE_LINE_PATTERN.match(line)
+        if reg_match:
+            # Collect consecutive regular blockquote lines
+            quote_lines = []
+            start_offset = current_offset
+
+            while i < len(lines):
+                # Don't match expandable as regular
+                if _EXPANDABLE_BLOCKQUOTE_PATTERN.match(lines[i]):
+                    break
+                m = _BLOCKQUOTE_LINE_PATTERN.match(lines[i])
+                if m:
+                    quote_lines.append(m.group(1))
+                    i += 1
+                else:
+                    break
+
+            quote_content = "\n".join(quote_lines)
+            current_offset += len(quote_content) + (1 if i < len(lines) else 0)
+
+            result_parts.append(quote_content)
+            if i < len(lines):
+                result_parts.append("\n")
+
+            entities.append(
+                TelegramEntity(
+                    type=EntityType.BLOCKQUOTE,
+                    offset=start_offset,
+                    length=len(quote_content),
+                )
+            )
+            continue
+
+        # Regular line
+        current_offset += len(line) + (1 if i < len(lines) - 1 else 0)
+        result_parts.append(line)
+        if i < len(lines) - 1:
+            result_parts.append("\n")
+        i += 1
+
+    result_text = "".join(result_parts)
+
+    return result_text, entities

chatgpt_md_converter/telegram_entities/extractors/headings.py

@@ -0,0 +1,56 @@
+"""Heading entity extraction (converted to bold)."""
+
+import re
+from typing import List, Tuple
+
+from ..entity import EntityType, TelegramEntity
+
+# Pattern for Markdown headings: # Heading, ## Heading, etc.
+_HEADING_PATTERN = re.compile(r"^(#{1,6})\s+(.+)$", re.MULTILINE)
+
+
+def extract_heading_entities(text: str) -> Tuple[str, List[TelegramEntity]]:
+    """
+    Extract Markdown headings and convert them to bold entities.
+
+    Telegram doesn't have native heading support, so headings are converted
+    to bold text (matching the HTML converter behavior).
+
+    Args:
+        text: Input text with Markdown headings
+
+    Returns:
+        Tuple of (text_with_headings_converted, list_of_bold_entities)
+    """
+    entities: List[TelegramEntity] = []
+    result_parts: List[str] = []
+    last_end = 0
+
+    for match in _HEADING_PATTERN.finditer(text):
+        # Add text before this heading
+        result_parts.append(text[last_end : match.start()])
+
+        # Calculate position in output
+        current_offset = sum(len(p) for p in result_parts)
+
+        # Extract heading text (without the # markers)
+        heading_text = match.group(2)
+
+        # Add the heading text
+        result_parts.append(heading_text)
+
+        # Create bold entity for the heading
+        entities.append(
+            TelegramEntity(
+                type=EntityType.BOLD,
+                offset=current_offset,
+                length=len(heading_text),
+            )
+        )
+
+        last_end = match.end()
+
+    # Add remaining text
+    result_parts.append(text[last_end:])
+
+    return "".join(result_parts), entities

chatgpt_md_converter/telegram_entities/extractors/inline.py

@@ -0,0 +1,295 @@
+"""Inline formatting entity extraction (bold, italic, underline, etc.)."""
+
+import re
+from typing import List, Tuple
+
+from ..entity import EntityType, TelegramEntity
+
+# Patterns for different formatting types
+# Order matters - longer markers first to avoid partial matches
+_PATTERNS = [
+    # Bold+Italic: ***text***
+    (
+        re.compile(r"(?<![\\\*])\*\*\*(?!\*)(?=\S)([\s\S]*?)(?<=\S)\*\*\*(?!\*)", re.DOTALL),
+        [EntityType.BOLD, EntityType.ITALIC],
+        3,
+    ),
+    # Underline+Italic: ___text___
+    (
+        re.compile(
+            r"(?<![\\_])___(?!_)(?=\S)([\s\S]*?)(?<=\S)___(?!_)",
+            re.DOTALL,
+        ),
+        [EntityType.UNDERLINE, EntityType.ITALIC],
+        3,
+    ),
+    # Bold: **text**
+    (
+        re.compile(r"(?<![\\\*])\*\*(?!\*)(?=\S)([\s\S]*?)(?<=\S)(?<!\*)\*\*(?!\*)", re.DOTALL),
+        [EntityType.BOLD],
+        2,
+    ),
+    # Underline: __text__
+    (
+        re.compile(
+            r"(?<![\\_])__(?!_)(?=\S)([\s\S]*?)(?<=\S)(?<!_)__(?!_)",
+            re.DOTALL,
+        ),
+        [EntityType.UNDERLINE],
+        2,
+    ),
+    # Strikethrough: ~~text~~
+    (
+        re.compile(r"(?<![\\~])~~(?!~)(?=\S)([\s\S]*?)(?<=\S)(?<!~)~~(?!~)", re.DOTALL),
+        [EntityType.STRIKETHROUGH],
+        2,
+    ),
+    # Spoiler: ||text||
+    (
+        re.compile(r"(?<![\\|])\|\|(?!\|)(?=\S)([^\n]*?)(?<=\S)(?<!\|)\|\|(?!\|)"),
+        [EntityType.SPOILER],
+        2,
+    ),
+    # Italic with asterisk: *text* (must not be adjacent to other asterisks)
+    (
+        re.compile(
+            r"(?<![A-Za-z0-9\\\*])\*(?!\*)(?=\S)([\s\S]*?)(?<=\S)(?<!\*)\*(?![A-Za-z0-9\*])",
+            re.DOTALL,
+        ),
+        [EntityType.ITALIC],
+        1,
+    ),
+    # Italic with underscore: _text_
+    (
+        re.compile(
+            r"(?<![A-Za-z0-9\\_])_(?!_)(?=\S)([\s\S]*?)(?<=\S)(?<!_)_(?![A-Za-z0-9_])",
+            re.DOTALL,
+        ),
+        [EntityType.ITALIC],
+        1,
+    ),
+]
+
+
+class _Match:
+    """Represents a formatting match with its properties."""
+
+    def __init__(
+        self,
+        start: int,
+        end: int,
+        inner_start: int,
+        inner_end: int,
+        entity_types: List[EntityType],
+        marker_len: int,
+    ):
+        self.start = start
+        self.end = end
+        self.inner_start = inner_start
+        self.inner_end = inner_end
+        self.entity_types = entity_types
+        self.marker_len = marker_len
+        self.children: List["_Match"] = []
+
+    def contains(self, other: "_Match") -> bool:
+        """Check if this match's inner content fully contains another match."""
+        return self.inner_start <= other.start and other.end <= self.inner_end
+
+
+def _find_all_matches(text: str) -> List[_Match]:
+    """Find all formatting matches in text."""
+    matches = []
+
+    for pattern, entity_types, marker_len in _PATTERNS:
+        for match in pattern.finditer(text):
+            matches.append(
+                _Match(
+                    start=match.start(),
+                    end=match.end(),
+                    inner_start=match.start() + marker_len,
+                    inner_end=match.end() - marker_len,
+                    entity_types=list(entity_types),
+                    marker_len=marker_len,
+                )
+            )
+
+    # Sort by start position, then by length descending (longer first)
+    matches.sort(key=lambda m: (m.start, -(m.end - m.start)))
+
+    return matches
+
+
+def _build_match_tree(matches: List[_Match]) -> List[_Match]:
+    """
+    Build a tree of matches where nested matches are children.
+    Returns only top-level matches (others are nested as children).
+    """
+    if not matches:
+        return []
+
+    result: List[_Match] = []
+
+    for match in matches:
+        # Find if this match should be nested inside an existing result
+        placed = False
+        for existing in result:
+            if existing.contains(match):
+                # Recursively try to place in existing's children
+                placed = _try_place_in_children(existing, match)
+                if placed:
+                    break
+
+        if not placed:
+            # Check if this match overlaps with any existing (invalid)
+            overlaps = False
+            for existing in result:
+                if _matches_overlap(match, existing):
+                    overlaps = True
+                    break
+
+            if not overlaps:
+                result.append(match)
+
+    return result
+
+
+def _try_place_in_children(parent: _Match, child: _Match) -> bool:
+    """Try to place a child match in the parent's children list."""
+    # First check if it fits in any existing child
+    for existing_child in parent.children:
+        if existing_child.contains(child):
+            return _try_place_in_children(existing_child, child)
+
+    # Check for overlaps with existing children
+    for existing_child in parent.children:
+        if _matches_overlap(child, existing_child):
+            return False
+
+    # Can add as a direct child
+    parent.children.append(child)
+    return True
+
+
+def _matches_overlap(m1: _Match, m2: _Match) -> bool:
+    """Check if two matches have invalid overlap (partial, not nested)."""
+    # No overlap
+    if m1.end <= m2.start or m2.end <= m1.start:
+        return False
+    # m1 contains m2 in inner content
+    if m1.inner_start <= m2.start and m2.end <= m1.inner_end:
+        return False
+    # m2 contains m1 in inner content
+    if m2.inner_start <= m1.start and m1.end <= m2.inner_end:
+        return False
+    # Invalid overlap
+    return True
+
+
+def _process_match(
+    text: str,
+    match: _Match,
+    base_offset: int,
+) -> Tuple[str, List[TelegramEntity]]:
+    """
+    Process a single match and its children, returning plain text and entities.
+
+    Args:
+        text: The text containing the match
+        match: The match to process
+        base_offset: Offset in the final output where this match starts
+
+    Returns:
+        Tuple of (processed_text, entities)
+    """
+    inner_text = text[match.inner_start : match.inner_end]
+    entities: List[TelegramEntity] = []
+
+    # If there are children, process them
+    if match.children:
+        # Sort children by position
+        match.children.sort(key=lambda m: m.start)
+
+        # Process children recursively
+        processed_parts: List[str] = []
+        child_entities: List[TelegramEntity] = []
+        last_end = match.inner_start
+
+        for child in match.children:
+            # Add text before this child
+            processed_parts.append(text[last_end : child.start])
+
+            # Calculate child's offset in the final output
+            child_offset = base_offset + sum(len(p) for p in processed_parts)
+
+            # Process child recursively
+            child_text, child_ents = _process_match(text, child, child_offset)
+            processed_parts.append(child_text)
+            child_entities.extend(child_ents)
+
+            last_end = child.end
+
+        # Add remaining text after last child
+        processed_parts.append(text[last_end : match.inner_end])
+
+        inner_text = "".join(processed_parts)
+        entities.extend(child_entities)
+
+    # Create entities for this match
+    for entity_type in match.entity_types:
+        entities.append(
+            TelegramEntity(
+                type=entity_type,
+                offset=base_offset,
+                length=len(inner_text),
+            )
+        )
+
+    return inner_text, entities
+
+
+def extract_inline_formatting_entities(
+    text: str,
+) -> Tuple[str, List[TelegramEntity]]:
+    """
+    Extract inline formatting (bold, italic, etc.) and return plain text with entities.
+
+    Handles nested formatting where one style is fully contained within another.
+
+    Args:
+        text: Input text with Markdown formatting markers
+
+    Returns:
+        Tuple of (text_without_markers, list_of_entities)
+    """
+    matches = _find_all_matches(text)
+    top_level_matches = _build_match_tree(matches)
+
+    if not top_level_matches:
+        return text, []
+
+    # Sort by position
+    top_level_matches.sort(key=lambda m: m.start)
+
+    # Process all matches
+    result_parts: List[str] = []
+    all_entities: List[TelegramEntity] = []
+    last_end = 0
+
+    for match in top_level_matches:
+        # Add text before this match
+        result_parts.append(text[last_end : match.start])
+
+        # Calculate offset for this match
+        current_offset = sum(len(p) for p in result_parts)
+
+        # Process match and its children
+        processed_text, entities = _process_match(text, match, current_offset)
+        result_parts.append(processed_text)
+        all_entities.extend(entities)
+
+        last_end = match.end
+
+    # Add remaining text
+    result_parts.append(text[last_end:])
+
+    return "".join(result_parts), all_entities

chatgpt_md_converter/telegram_entities/extractors/links.py

@@ -0,0 +1,59 @@
+"""Link entity extraction."""
+
+import re
+from typing import List, Tuple
+
+from ..entity import EntityType, TelegramEntity
+
+# Pattern for Markdown links: [text](url)
+# Also handles image links: ![alt](url) - treated the same as regular links
+_LINK_PATTERN = re.compile(r"!?\[((?:[^\[\]]|\[.*?\])*)\]\(([^)]+)\)")
+
+
+def extract_link_entities(text: str) -> Tuple[str, List[TelegramEntity]]:
+    """
+    Extract Markdown links and return plain text with TEXT_LINK entities.
+
+    Handles both regular links [text](url) and image links ![alt](url).
+    Image links are converted to text links showing the alt text.
+
+    Args:
+        text: Input text with Markdown links
+
+    Returns:
+        Tuple of (text_with_links_replaced, list_of_entities)
+    """
+    entities: List[TelegramEntity] = []
+    result_parts: List[str] = []
+    last_end = 0
+
+    for match in _LINK_PATTERN.finditer(text):
+        # Add text before this link
+        result_parts.append(text[last_end : match.start()])
+
+        # Calculate position in output
+        current_offset = sum(len(p) for p in result_parts)
+
+        # Extract link text and URL
+        link_text = match.group(1)
+        url = match.group(2)
+
+        # Add the link text (without the markdown syntax)
+        result_parts.append(link_text)
+
+        # Create entity
+        entities.append(
+            TelegramEntity(
+                type=EntityType.TEXT_LINK,
+                offset=current_offset,
+                length=len(link_text),
+                url=url,
+            )
+        )
+
+        last_end = match.end()
+
+    # Add remaining text
+    result_parts.append(text[last_end:])
+
+    return "".join(result_parts), entities

chatgpt_md_converter/telegram_entities/parser.py

@@ -0,0 +1,300 @@
+"""Main parser that combines all entity extractors."""
+
+import re
+from typing import List, Tuple
+
+from .entity import EntityType, TelegramEntity
+from .extractors import (extract_blockquote_entities, extract_heading_entities,
+                         extract_inline_formatting_entities,
+                         extract_link_entities)
+from .utf16 import utf16_len
+
+# Placeholder prefix for protected content
+_CODE_BLOCK_PLACEHOLDER = "\x00CODEBLOCK"
+_INLINE_CODE_PLACEHOLDER = "\x00INLINECODE"
+
+
+def _convert_list_markers(text: str) -> str:
+    """Convert Markdown list markers (* or -) to bullet points."""
+    return re.sub(r"^(\s*)[\-\*]\s+", r"\1• ", text, flags=re.MULTILINE)
+
+
+def _remove_citation_markers(text: str) -> str:
+    """Remove ChatGPT-style citation markers like 【1】."""
+    return re.sub(r"【[^】]+】", "", text)
+
+
+def _adjust_entities_to_utf16(
+    text: str, entities: List[TelegramEntity]
+) -> List[TelegramEntity]:
+    """
+    Convert entity offsets and lengths from Python char indices to UTF-16 code units.
+
+    Telegram requires UTF-16 code units for entity positions.
+    """
+    adjusted = []
+    for entity in entities:
+        # Clamp offset and length to text bounds
+        offset = min(entity.offset, len(text))
+        length = min(entity.length, len(text) - offset)
+
+        if length <= 0:
+            continue
+
+        # Get the text portions
+        before_text = text[:offset]
+        entity_text = text[offset : offset + length]
+
+        # Convert to UTF-16 units
+        utf16_offset = utf16_len(before_text)
+        utf16_length = utf16_len(entity_text)
+
+        if utf16_length > 0:
+            adjusted.append(
+                TelegramEntity(
+                    type=entity.type,
+                    offset=utf16_offset,
+                    length=utf16_length,
+                    url=entity.url,
+                    language=entity.language,
+                )
+            )
+
+    return adjusted
+
+
+def _validate_and_sort_entities(
+    entities: List[TelegramEntity],
+) -> List[TelegramEntity]:
+    """
+    Sort entities by offset and filter invalid ones.
+    """
+    # Filter out zero-length and negative entities
+    entities = [e for e in entities if e.length > 0 and e.offset >= 0]
+
+    # Sort by offset, then by length descending (longer first for nesting)
+    entities = sorted(entities, key=lambda e: (e.offset, -e.length))
+
+    return entities
+
+
+def _clean_multiple_newlines(text: str) -> str:
+    """Reduce 3+ consecutive newlines to just 2."""
+    return re.sub(r"\n{3,}", "\n\n", text)
+
+
+def _extract_with_placeholders(
+    text: str, pattern: re.Pattern, placeholder_prefix: str
+) -> Tuple[str, dict]:
+    """
+    Extract matches and replace with placeholders.
+    Returns (modified_text, {placeholder: (content, entity_info)})
+    """
+    extractions = {}
+    counter = [0]
+
+    def replacer(match):
+        placeholder = f"{placeholder_prefix}{counter[0]}\x00"
+        counter[0] += 1
+        extractions[placeholder] = match
+        return placeholder
+
+    modified = pattern.sub(replacer, text)
+    return modified, extractions
+
+
+def parse_entities(text: str) -> Tuple[str, List[TelegramEntity]]:
+    """
+    Parse Markdown text and return plain text with Telegram entities.
+
+    Uses a placeholder-based approach to handle the order of extraction correctly:
+    1. Replace code blocks and inline code with placeholders
+    2. Extract all other formatting (blockquotes, headings, links, inline styles)
+    3. Restore placeholders and calculate final offsets
+
+    Args:
+        text: Markdown-formatted text
+
+    Returns:
+        Tuple of (plain_text, list_of_entities)
+        Entities have offsets/lengths in UTF-16 code units.
+    """
+    all_entities: List[TelegramEntity] = []
+
+    # Phase 1: Extract code blocks to placeholders
+    code_block_pattern = re.compile(
+        r"(?P<fence>`{3,})(?P<lang>\w+)?\n(?P<code>[\s\S]*?)(?P=fence)",
+        flags=re.MULTILINE,
+    )
+    code_block_map = {}
+    code_block_counter = [0]
+
+    def replace_code_block(match):
+        placeholder = f"{_CODE_BLOCK_PLACEHOLDER}{code_block_counter[0]}\x00"
+        code_block_counter[0] += 1
+        # Strip trailing newline from code content (appears before closing fence)
+        code_content = match.group("code").rstrip("\n")
+        language = match.group("lang") or None
+        code_block_map[placeholder] = (code_content, language)
+        return placeholder
+
+    # Ensure closing delimiters
+    text = _ensure_closing_delimiters(text)
+    text = code_block_pattern.sub(replace_code_block, text)
+
+    # Phase 2: Extract inline code to placeholders
+    inline_code_pattern = re.compile(r"`([^`\n]+)`")
+    inline_code_map = {}
+    inline_code_counter = [0]
+
+    def replace_inline_code(match):
+        placeholder = f"{_INLINE_CODE_PLACEHOLDER}{inline_code_counter[0]}\x00"
+        inline_code_counter[0] += 1
+        code_content = match.group(1)
+        inline_code_map[placeholder] = code_content
+        return placeholder
+
+    text = inline_code_pattern.sub(replace_inline_code, text)
+
+    # Phase 3: Extract other formatting (on text with placeholders)
+    # Order matters: inline formatting first (removes markers), then links
+    text, blockquote_entities = extract_blockquote_entities(text)
+    all_entities.extend(blockquote_entities)
+
+    text, heading_entities = extract_heading_entities(text)
+    all_entities.extend(heading_entities)
+
+    text, inline_entities = extract_inline_formatting_entities(text)
+    all_entities.extend(inline_entities)
+
+    # Extract links AFTER inline formatting so offsets are correct
+    text, link_entities = extract_link_entities(text)
+    all_entities.extend(link_entities)
+
+    # Phase 4: Restore code placeholders and create entities
+    # Collect all placeholders with their info
+    all_placeholders = []
+
+    for placeholder, (code_content, language) in code_block_map.items():
+        if placeholder in text:
+            pos = text.find(placeholder)
+            all_placeholders.append({
+                'placeholder': placeholder,
+                'content': code_content,
+                'position': pos,
+                'type': EntityType.PRE,
+                'language': language,
+            })
+
+    for placeholder, code_content in inline_code_map.items():
+        if placeholder in text:
+            pos = text.find(placeholder)
+            all_placeholders.append({
+                'placeholder': placeholder,
+                'content': code_content,
+                'position': pos,
+                'type': EntityType.CODE,
+                'language': None,
+            })
+
+    # Sort by position ascending (restore from start to end)
+    # This way, when we shift entities, the later entities get adjusted correctly
+    all_placeholders.sort(key=lambda x: x['position'])
+
+    code_entities: List[TelegramEntity] = []
+
+    for ph_info in all_placeholders:
+        placeholder = ph_info['placeholder']
+        code_content = ph_info['content']
+        offset = text.find(placeholder)
+        text = text.replace(placeholder, code_content, 1)
+
+        code_entities.append(
+            TelegramEntity(
+                type=ph_info['type'],
+                offset=offset,
+                length=len(code_content),
+                language=ph_info['language'],
+            )
+        )
+
+        # Adjust existing entities (both all_entities and code_entities) after this position
+        placeholder_len = len(placeholder)
+        content_len = len(code_content)
+        shift = content_len - placeholder_len
+        all_entities = _shift_entities_after(all_entities, offset, shift)
+        # Also shift already-created code entities (except the one we just added)
+        code_entities = _shift_entities_after(code_entities[:-1], offset, shift) + [code_entities[-1]]
+
+    all_entities.extend(code_entities)
+
+    # Phase 5: Clean up
+    text = _convert_list_markers(text)
+    text = _remove_citation_markers(text)
+    text = _clean_multiple_newlines(text)
+
+    # Validate and sort entities
+    all_entities = _validate_and_sort_entities(all_entities)
+
+    # Convert to UTF-16 offsets
+    all_entities = _adjust_entities_to_utf16(text, all_entities)
+
+    return text.strip(), all_entities
+
+
+def _shift_entities_after(
+    entities: List[TelegramEntity], position: int, shift: int
+) -> List[TelegramEntity]:
+    """Shift entity offsets that come after a given position."""
+    result = []
+    for e in entities:
+        if e.offset >= position:
+            result.append(
+                TelegramEntity(
+                    type=e.type,
+                    offset=e.offset + shift,
+                    length=e.length,
+                    url=e.url,
+                    language=e.language,
+                )
+            )
+        else:
+            result.append(e)
+    return result
+
+
+def _ensure_closing_delimiters(text: str) -> str:
+    """Append any missing closing backtick fences for Markdown code blocks."""
+    code_block_re = re.compile(
+        r"(?P<fence>`{3,})(?P<lang>\w+)?\n?[\s\S]*?(?<=\n)?(?P=fence)",
+        flags=re.DOTALL,
+    )
+
+    open_fence = None
+    for line in text.splitlines():
+        stripped = line.strip()
+        if open_fence is None:
+            match = re.match(r"^(?P<fence>`{3,})(?P<lang>\w+)?$", stripped)
+            if match:
+                open_fence = match.group("fence")
+        else:
+            if stripped == open_fence:
+                open_fence = None
+
+    if open_fence is not None:
+        if not text.endswith("\n"):
+            text += "\n"
+        text += open_fence
+
+    # Check for unclosed triple backticks
+    temp = code_block_re.sub("", text)
+    if temp.count("```") % 2 != 0:
+        text += "\n```"
+
+    # Check for unclosed single backticks (inline code)
+    temp = code_block_re.sub("", text)
+    temp = re.sub(r"``+", "", temp)
+    if temp.count("`") % 2 != 0:
+        text += "`"
+
+    return text

chatgpt_md_converter/telegram_entities/utf16.py

@@ -0,0 +1,50 @@
+"""UTF-16 encoding utilities for Telegram entity offset calculation."""
+
+
+def utf16_len(text: str) -> int:
+    """
+    Calculate the length of a string in UTF-16 code units.
+
+    Telegram uses UTF-16 code units for entity offsets and lengths.
+    Characters outside the Basic Multilingual Plane (like emoji) take 2 units.
+
+    Args:
+        text: The string to measure
+
+    Returns:
+        Length in UTF-16 code units
+    """
+    return len(text.encode("utf-16-le")) // 2
+
+
+def char_to_utf16_offset(text: str, char_index: int) -> int:
+    """
+    Convert a Python string index to a UTF-16 offset.
+
+    Args:
+        text: The full text string
+        char_index: Python string index (0-based)
+
+    Returns:
+        UTF-16 offset for the same position
+    """
+    return utf16_len(text[:char_index])
+
+
+def utf16_to_char_offset(text: str, utf16_offset: int) -> int:
+    """
+    Convert a UTF-16 offset to a Python string index.
+
+    Args:
+        text: The full text string
+        utf16_offset: UTF-16 offset
+
+    Returns:
+        Python string index for the same position
+    """
+    current_utf16 = 0
+    for i, char in enumerate(text):
+        if current_utf16 >= utf16_offset:
+            return i
+        current_utf16 += len(char.encode("utf-16-le")) // 2
+    return len(text)

{chatgpt_md_converter-0.3.12.dist-info → chatgpt_md_converter-0.4.0b1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: chatgpt_md_converter
-Version: 0.3.12
+Version: 0.4.0b1
 Summary: A package for converting markdown to HTML for chat Telegram bots
 Home-page: https://github.com/botfather-dev/formatter-chatgpt-telegram
 Author: Kostiantyn Kriuchkov

chatgpt_md_converter-0.4.0b1.dist-info/RECORD

@@ -0,0 +1,29 @@
+chatgpt_md_converter/__init__.py,sha256=pq0o14l7pBFPm-YsLj0A7nO2FPgF48MSRCEc7b9ktGQ,447
+chatgpt_md_converter/html_splitter.py,sha256=DdjJx0I-A9rZHOxS-0LXsy7YUrgrkrtdeqZtEQ7eooA,7853
+chatgpt_md_converter/html_to_markdown.py,sha256=XlLpQD7W_AooWrvTtvrGVwfPPa80tDKWuT1iT6Vzygw,174
+chatgpt_md_converter/telegram_formatter.py,sha256=w3tjoSdRH_UdoFmGeXe7I47dhDIceXuGOA1oCLMnUmM,87
+chatgpt_md_converter/html_markdown/escaping.py,sha256=wJA4vUJQVcxpkJ4sCIYIWKaqffb_O72R93H81hTgTxA,1808
+chatgpt_md_converter/html_markdown/handlers.py,sha256=zKGRg__41SP7bKs8jodOWAEZJb2FNaC5_raoUiWdBUE,6696
+chatgpt_md_converter/html_markdown/renderer.py,sha256=en-fAr3Bhmm4ZndDaPKV8nLVQ_7HpS_NFBSWcrQporY,438
+chatgpt_md_converter/html_markdown/state.py,sha256=sxbz0ucCakI0KgR86EMZx0nvfU1oiqgVUofujFTeKoo,432
+chatgpt_md_converter/html_markdown/tree.py,sha256=ryohrhO2X5QepZev3087qPoGmMznqHDwH00TNGoW6a4,2154
+chatgpt_md_converter/telegram_entities/__init__.py,sha256=dopG-8_gWX8xPeD-9dyHdurs5VPrz-wAFFRvHNKiUNg,1855
+chatgpt_md_converter/telegram_entities/entity.py,sha256=oygQxwBsE7AGm2etq6HFZIeo7tBCwsUGniLP17-_Oz0,1705
+chatgpt_md_converter/telegram_entities/parser.py,sha256=P7uQeGaNLLuFa5QLEkkEhdSqaB9xIlUwuZjbXZ8hkGQ,9885
+chatgpt_md_converter/telegram_entities/utf16.py,sha256=eH-yX7d1wZwb3nRdk3kq1LFd-NQMqYHutPbkvX5_DC0,1283
+chatgpt_md_converter/telegram_entities/extractors/__init__.py,sha256=FinTAoRNjuHza0LcEBtpNnBvSR8PFo6cVVDkLg0cV6w,407
+chatgpt_md_converter/telegram_entities/extractors/blockquotes.py,sha256=Di8nG5Oej0hLbBB-WJ3GtlZCvCaa_BNmoUdpFGo9mnY,3596
+chatgpt_md_converter/telegram_entities/extractors/headings.py,sha256=AzjF9jElWfw3d4Qx-81fku7gyTkvb0pKlmow0zUXSk4,1602
+chatgpt_md_converter/telegram_entities/extractors/inline.py,sha256=DYSs7cJEFY3-fGtdMdOA7DO5ERtEF8r2GQns5WcPyto,8745
+chatgpt_md_converter/telegram_entities/extractors/links.py,sha256=fe35PDGKzbF0cRac3HgQ9mVFvYvrtt9LDmS_pL9GPlk,1671
+chatgpt_md_converter/telegram_markdown/__init__.py,sha256=C0Oexz9brpdE-TqEpiAUV78TsZdSrnnH_5yYpEJ03Us,131
+chatgpt_md_converter/telegram_markdown/code_blocks.py,sha256=VPkSisvb6DiS5KAcq0OaX4sqR1YX4VgZvJEXZeAjIWk,3067
+chatgpt_md_converter/telegram_markdown/inline.py,sha256=MPzj5VpDqrlvPy69CCwUIOsWgtgIFfbB4CliV5Wz-TY,2207
+chatgpt_md_converter/telegram_markdown/postprocess.py,sha256=jUf01tAIqHQ1NxNlVGsvU-Yw8SDOHtMoS7MUzaQLf_8,775
+chatgpt_md_converter/telegram_markdown/preprocess.py,sha256=k9XBtwgXkh07SlsqbdcZHwOMHhUGOjiIbOehO5wBnu0,1561
+chatgpt_md_converter/telegram_markdown/renderer.py,sha256=39ZehJq6PVWm-sigeBz7vCycwzEmV4Mwiw36jkGIgXI,1960
+chatgpt_md_converter-0.4.0b1.dist-info/licenses/LICENSE,sha256=SDr2jeP-s2g4vf17-jdLXrrqA4_mU7L_RtSJlv4Y2mk,1077
+chatgpt_md_converter-0.4.0b1.dist-info/METADATA,sha256=9wvQrKaXzPu-_VKWRW_cK4vmbaZkwgtzMrbENVHLZb4,6606
+chatgpt_md_converter-0.4.0b1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+chatgpt_md_converter-0.4.0b1.dist-info/top_level.txt,sha256=T2o7csVtZgr-Pwm83aSUkZn0humJmDFNqW38tRSsNqw,21
+chatgpt_md_converter-0.4.0b1.dist-info/RECORD,,

chatgpt_md_converter-0.3.12.dist-info/RECORD

@@ -1,20 +0,0 @@
-chatgpt_md_converter/__init__.py,sha256=6ts2hnimdBn_qCA15LKuipUjSU9ZCqRk1GbDPc_JjO4,242
-chatgpt_md_converter/html_splitter.py,sha256=DdjJx0I-A9rZHOxS-0LXsy7YUrgrkrtdeqZtEQ7eooA,7853
-chatgpt_md_converter/html_to_markdown.py,sha256=XlLpQD7W_AooWrvTtvrGVwfPPa80tDKWuT1iT6Vzygw,174
-chatgpt_md_converter/telegram_formatter.py,sha256=w3tjoSdRH_UdoFmGeXe7I47dhDIceXuGOA1oCLMnUmM,87
-chatgpt_md_converter/html_markdown/escaping.py,sha256=wJA4vUJQVcxpkJ4sCIYIWKaqffb_O72R93H81hTgTxA,1808
-chatgpt_md_converter/html_markdown/handlers.py,sha256=zKGRg__41SP7bKs8jodOWAEZJb2FNaC5_raoUiWdBUE,6696
-chatgpt_md_converter/html_markdown/renderer.py,sha256=en-fAr3Bhmm4ZndDaPKV8nLVQ_7HpS_NFBSWcrQporY,438
-chatgpt_md_converter/html_markdown/state.py,sha256=sxbz0ucCakI0KgR86EMZx0nvfU1oiqgVUofujFTeKoo,432
-chatgpt_md_converter/html_markdown/tree.py,sha256=ryohrhO2X5QepZev3087qPoGmMznqHDwH00TNGoW6a4,2154
-chatgpt_md_converter/telegram_markdown/__init__.py,sha256=C0Oexz9brpdE-TqEpiAUV78TsZdSrnnH_5yYpEJ03Us,131
-chatgpt_md_converter/telegram_markdown/code_blocks.py,sha256=VPkSisvb6DiS5KAcq0OaX4sqR1YX4VgZvJEXZeAjIWk,3067
-chatgpt_md_converter/telegram_markdown/inline.py,sha256=MPzj5VpDqrlvPy69CCwUIOsWgtgIFfbB4CliV5Wz-TY,2207
-chatgpt_md_converter/telegram_markdown/postprocess.py,sha256=jUf01tAIqHQ1NxNlVGsvU-Yw8SDOHtMoS7MUzaQLf_8,775
-chatgpt_md_converter/telegram_markdown/preprocess.py,sha256=k9XBtwgXkh07SlsqbdcZHwOMHhUGOjiIbOehO5wBnu0,1561
-chatgpt_md_converter/telegram_markdown/renderer.py,sha256=39ZehJq6PVWm-sigeBz7vCycwzEmV4Mwiw36jkGIgXI,1960
-chatgpt_md_converter-0.3.12.dist-info/licenses/LICENSE,sha256=SDr2jeP-s2g4vf17-jdLXrrqA4_mU7L_RtSJlv4Y2mk,1077
-chatgpt_md_converter-0.3.12.dist-info/METADATA,sha256=CWvPYndrqJad_RD-zJABwdDSPhVhkcuXzVwf8Z7BTjw,6605
-chatgpt_md_converter-0.3.12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-chatgpt_md_converter-0.3.12.dist-info/top_level.txt,sha256=T2o7csVtZgr-Pwm83aSUkZn0humJmDFNqW38tRSsNqw,21
-chatgpt_md_converter-0.3.12.dist-info/RECORD,,