xgen-doc2chunk 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

xgen_doc2chunk/core/processor/rtf_helper/__init__.py

@@ -0,0 +1,128 @@
+# xgen_doc2chunk/core/processor/rtf_helper/__init__.py
+"""
+RTF Helper Module
+
+Provides RTF parsing and extraction utilities with proper interface separation.
+
+Architecture:
+    - RTFPreprocessor: Binary preprocessing (image extraction, \\bin handling)
+    - RTFFileConverter: Pass through (RTF uses raw binary)
+    - RTFMetadataExtractor: Metadata extraction
+    - Table extraction: extract_tables_with_positions()
+    - Content extraction: extract_inline_content(), extract_text_only()
+
+Usage:
+    from xgen_doc2chunk.core.processor.rtf_helper import (
+        RTFFileConverter,
+        RTFConvertedData,
+        RTFPreprocessor,
+        RTFMetadataExtractor,
+        RTFSourceInfo,
+        extract_tables_with_positions,
+        extract_inline_content,
+        extract_text_only,
+    )
+"""
+
+# Converter
+from xgen_doc2chunk.core.processor.rtf_helper.rtf_file_converter import (
+    RTFFileConverter,
+    RTFConvertedData,
+)
+
+# Preprocessor
+from xgen_doc2chunk.core.processor.rtf_helper.rtf_preprocessor import (
+    RTFPreprocessor,
+)
+
+# Metadata
+from xgen_doc2chunk.core.processor.rtf_helper.rtf_metadata_extractor import (
+    RTFMetadataExtractor,
+    RTFSourceInfo,
+)
+
+# Table extraction
+from xgen_doc2chunk.core.processor.rtf_helper.rtf_table_extractor import (
+    RTFCellInfo,
+    RTFTable,
+    extract_tables_with_positions,
+)
+
+# Content extraction
+from xgen_doc2chunk.core.processor.rtf_helper.rtf_content_extractor import (
+    extract_inline_content,
+    extract_text_only,
+)
+
+# Decoder utilities
+from xgen_doc2chunk.core.processor.rtf_helper.rtf_decoder import (
+    detect_encoding,
+    decode_content,
+    decode_bytes,
+    decode_hex_escapes,
+)
+
+# Text cleaning utilities
+from xgen_doc2chunk.core.processor.rtf_helper.rtf_text_cleaner import (
+    clean_rtf_text,
+    remove_destination_groups,
+    remove_shape_groups,
+    remove_shape_property_groups,
+    remove_shprslt_blocks,
+)
+
+# Region finder utilities
+from xgen_doc2chunk.core.processor.rtf_helper.rtf_region_finder import (
+    find_excluded_regions,
+    is_in_excluded_region,
+)
+
+# Constants
+from xgen_doc2chunk.core.processor.rtf_helper.rtf_constants import (
+    SHAPE_PROPERTY_NAMES,
+    SKIP_DESTINATIONS,
+    EXCLUDE_DESTINATION_KEYWORDS,
+    IMAGE_DESTINATIONS,
+    CODEPAGE_ENCODING_MAP,
+    DEFAULT_ENCODINGS,
+)
+
+
+__all__ = [
+    # Converter
+    'RTFFileConverter',
+    'RTFConvertedData',
+    # Preprocessor
+    'RTFPreprocessor',
+    # Metadata
+    'RTFMetadataExtractor',
+    'RTFSourceInfo',
+    # Table
+    'RTFCellInfo',
+    'RTFTable',
+    'extract_tables_with_positions',
+    # Content
+    'extract_inline_content',
+    'extract_text_only',
+    # Decoder
+    'detect_encoding',
+    'decode_content',
+    'decode_bytes',
+    'decode_hex_escapes',
+    # Text cleaner
+    'clean_rtf_text',
+    'remove_destination_groups',
+    'remove_shape_groups',
+    'remove_shape_property_groups',
+    'remove_shprslt_blocks',
+    # Region finder
+    'find_excluded_regions',
+    'is_in_excluded_region',
+    # Constants
+    'SHAPE_PROPERTY_NAMES',
+    'SKIP_DESTINATIONS',
+    'EXCLUDE_DESTINATION_KEYWORDS',
+    'IMAGE_DESTINATIONS',
+    'CODEPAGE_ENCODING_MAP',
+    'DEFAULT_ENCODINGS',
+]

xgen_doc2chunk/core/processor/rtf_helper/rtf_constants.py

@@ -0,0 +1,94 @@
+# xgen_doc2chunk/core/processor/rtf_helper/rtf_constants.py
+"""
+RTF Constants
+
+Constants used for RTF parsing.
+"""
+
+# Shape property names (to be removed)
+SHAPE_PROPERTY_NAMES = [
+    'shapeType', 'fFlipH', 'fFlipV', 'rotation',
+    'posh', 'posrelh', 'posv', 'posrelv',
+    'fLayoutInCell', 'fAllowOverlap', 'fBehindDocument',
+    'fPseudoInline', 'fLockAnchor', 'fLockPosition',
+    'fLockAspectRatio', 'fLockRotation', 'fLockAgainstSelect',
+    'fLockCropping', 'fLockVerticies', 'fLockText',
+    'fLockAdjustHandles', 'fLockAgainstGrouping',
+    'geoLeft', 'geoTop', 'geoRight', 'geoBottom',
+    'shapePath', 'pWrapPolygonVertices', 'dxWrapDistLeft',
+    'dyWrapDistTop', 'dxWrapDistRight', 'dyWrapDistBottom',
+    'fLine', 'fFilled', 'fillType', 'fillColor',
+    'fillOpacity', 'fillBackColor', 'fillBackOpacity',
+    'lineColor', 'lineOpacity', 'lineWidth', 'lineStyle',
+    'lineDashing', 'lineStartArrowhead', 'lineStartArrowWidth',
+    'lineStartArrowLength', 'lineEndArrowhead', 'lineEndArrowWidth',
+    'lineEndArrowLength', 'shadowType', 'shadowColor',
+    'shadowOpacity', 'shadowOffsetX', 'shadowOffsetY',
+]
+
+# RTF destination 키워드 (제외 대상)
+EXCLUDE_DESTINATION_KEYWORDS = [
+    'fonttbl', 'colortbl', 'stylesheet', 'listtable',
+    'listoverridetable', 'revtbl', 'rsidtbl', 'generator',
+    'info', 'xmlnstbl', 'mmathPr', 'themedata', 'colorschememapping',
+    'datastore', 'latentstyles', 'pgptbl', 'protusertbl',
+]
+
+# RTF skip destinations
+SKIP_DESTINATIONS = {
+    'fonttbl', 'colortbl', 'stylesheet', 'listtable',
+    'listoverridetable', 'revtbl', 'rsidtbl', 'generator',
+    'xmlnstbl', 'mmathPr', 'themedata', 'colorschememapping',
+    'datastore', 'latentstyles', 'pgptbl', 'protusertbl',
+    'bookmarkstart', 'bookmarkend', 'bkmkstart', 'bkmkend',
+    'fldinst', 'fldrslt',  # field instructions and results
+}
+
+# Image-related destinations
+IMAGE_DESTINATIONS = {
+    'pict', 'shppict', 'nonshppict', 'blipuid',
+}
+
+# Codepage to encoding mapping
+CODEPAGE_ENCODING_MAP = {
+    437: 'cp437',
+    850: 'cp850',
+    852: 'cp852',
+    855: 'cp855',
+    857: 'cp857',
+    860: 'cp860',
+    861: 'cp861',
+    863: 'cp863',
+    865: 'cp865',
+    866: 'cp866',
+    869: 'cp869',
+    874: 'cp874',
+    932: 'cp932',     # Japanese
+    936: 'gb2312',    # Simplified Chinese
+    949: 'cp949',     # Korean
+    950: 'big5',      # Traditional Chinese
+    1250: 'cp1250',   # Central European
+    1251: 'cp1251',   # Cyrillic
+    1252: 'cp1252',   # Western European
+    1253: 'cp1253',   # Greek
+    1254: 'cp1254',   # Turkish
+    1255: 'cp1255',   # Hebrew
+    1256: 'cp1256',   # Arabic
+    1257: 'cp1257',   # Baltic
+    1258: 'cp1258',   # Vietnamese
+    10000: 'mac_roman',
+    65001: 'utf-8',
+}
+
+# Default encodings to try
+DEFAULT_ENCODINGS = ['utf-8', 'cp949', 'euc-kr', 'cp1252', 'latin-1']
+
+
+__all__ = [
+    'SHAPE_PROPERTY_NAMES',
+    'EXCLUDE_DESTINATION_KEYWORDS',
+    'SKIP_DESTINATIONS',
+    'IMAGE_DESTINATIONS',
+    'CODEPAGE_ENCODING_MAP',
+    'DEFAULT_ENCODINGS',
+]

xgen_doc2chunk/core/processor/rtf_helper/rtf_content_extractor.py

@@ -0,0 +1,211 @@
+# xgen_doc2chunk/core/processor/rtf_helper/rtf_content_extractor.py
+"""
+RTF Content Extractor
+
+Extracts inline content (text + tables) from RTF documents.
+"""
+import logging
+import re
+from typing import List, Tuple
+
+from xgen_doc2chunk.core.processor.rtf_helper.rtf_decoder import (
+    decode_hex_escapes,
+)
+from xgen_doc2chunk.core.processor.rtf_helper.rtf_text_cleaner import (
+    clean_rtf_text,
+    remove_destination_groups,
+    remove_shape_groups,
+    remove_shape_property_groups,
+)
+from xgen_doc2chunk.core.processor.rtf_helper.rtf_region_finder import (
+    find_excluded_regions,
+)
+from xgen_doc2chunk.core.processor.rtf_helper.rtf_table_extractor import (
+    RTFTable,
+)
+
+logger = logging.getLogger("xgen_doc2chunk.rtf.content")
+
+
+def extract_inline_content(
+    content: str,
+    table_regions: List[Tuple[int, int, RTFTable]],
+    encoding: str = "cp949"
+) -> str:
+    """
+    Extract inline content from RTF with tables in original positions.
+    
+    Args:
+        content: RTF string content
+        table_regions: Table region list [(start, end, table), ...]
+        encoding: Encoding to use
+        
+    Returns:
+        Content string with tables inline
+    """
+    # Find header end (before first \pard)
+    header_end = 0
+    pard_match = re.search(r'\\pard\b', content)
+    if pard_match:
+        header_end = pard_match.start()
+    
+    # Find excluded regions (header, footer, footnote, etc.)
+    excluded_regions = find_excluded_regions(content)
+    
+    def clean_segment(segment: str, start_pos: int) -> str:
+        """Clean a segment while respecting excluded regions."""
+        if not excluded_regions:
+            segment = remove_destination_groups(segment)
+            decoded = decode_hex_escapes(segment, encoding)
+            return clean_rtf_text(decoded, encoding)
+        
+        result_parts = []
+        seg_pos = 0
+        
+        for excl_start, excl_end in excluded_regions:
+            rel_start = excl_start - start_pos
+            rel_end = excl_end - start_pos
+            
+            if rel_end <= 0 or rel_start >= len(segment):
+                continue
+            
+            rel_start = max(0, rel_start)
+            rel_end = min(len(segment), rel_end)
+            
+            if rel_start > seg_pos:
+                part = segment[seg_pos:rel_start]
+                part = remove_destination_groups(part)
+                decoded = decode_hex_escapes(part, encoding)
+                clean = clean_rtf_text(decoded, encoding)
+                if clean.strip():
+                    result_parts.append(clean)
+            
+            seg_pos = rel_end
+        
+        if seg_pos < len(segment):
+            part = segment[seg_pos:]
+            part = remove_destination_groups(part)
+            decoded = decode_hex_escapes(part, encoding)
+            clean = clean_rtf_text(decoded, encoding)
+            if clean.strip():
+                result_parts.append(clean)
+        
+        return ' '.join(result_parts)
+    
+    result_parts = []
+    
+    # No tables - just extract text
+    if not table_regions:
+        clean = clean_segment(content[header_end:], header_end)
+        if clean.strip():
+            result_parts.append(clean)
+        return '\n\n'.join(result_parts)
+    
+    # Adjust regions for header offset
+    adjusted_regions = []
+    for start_pos, end_pos, table in table_regions:
+        if end_pos > header_end:
+            adj_start = max(start_pos, header_end)
+            adjusted_regions.append((adj_start, end_pos, table))
+    
+    # Build content parts
+    last_end = header_end
+    
+    for start_pos, end_pos, table in adjusted_regions:
+        # Text before table
+        if start_pos > last_end:
+            segment = content[last_end:start_pos]
+            clean = clean_segment(segment, last_end)
+            if clean.strip():
+                result_parts.append(clean)
+        
+        # Table
+        if table.is_real_table():
+            result_parts.append(table.to_html())
+        else:
+            text_list = table.to_text_list()
+            if text_list:
+                result_parts.append(text_list)
+        
+        last_end = end_pos
+    
+    # Text after last table
+    if last_end < len(content):
+        segment = content[last_end:]
+        clean = clean_segment(segment, last_end)
+        if clean.strip():
+            result_parts.append(clean)
+    
+    return '\n\n'.join(result_parts)
+
+
+def extract_text_only(content: str, encoding: str = "cp949") -> str:
+    """
+    Extract only text from RTF (exclude tables).
+    
+    Legacy compatibility function.
+    
+    Args:
+        content: RTF string content
+        encoding: Encoding to use
+        
+    Returns:
+        Extracted text
+    """
+    # Remove header (fonttbl, colortbl, stylesheet, etc.)
+    pard_match = re.search(r'\\pard\b', content)
+    if pard_match:
+        content = content[pard_match.start():]
+    
+    # Remove destination groups
+    content = remove_destination_groups(content)
+    
+    # Handle shape groups (preserve shptxt content)
+    content = remove_shape_groups(content)
+    
+    # Remove shape property groups
+    content = remove_shape_property_groups(content)
+    
+    # Find table regions
+    table_regions = []
+    for match in re.finditer(r'\\trowd.*?\\row', content, re.DOTALL):
+        table_regions.append((match.start(), match.end()))
+    
+    # Merge adjacent tables
+    merged_regions = []
+    for start, end in table_regions:
+        if merged_regions and start - merged_regions[-1][1] < 100:
+            merged_regions[-1] = (merged_regions[-1][0], end)
+        else:
+            merged_regions.append((start, end))
+    
+    # Extract text excluding table regions
+    text_parts = []
+    last_end = 0
+    
+    for start, end in merged_regions:
+        if start > last_end:
+            segment = content[last_end:start]
+            decoded = decode_hex_escapes(segment, encoding)
+            clean = clean_rtf_text(decoded, encoding)
+            if clean:
+                text_parts.append(clean)
+        last_end = end
+    
+    if last_end < len(content):
+        segment = content[last_end:]
+        decoded = decode_hex_escapes(segment, encoding)
+        clean = clean_rtf_text(decoded, encoding)
+        if clean:
+            text_parts.append(clean)
+    
+    text = '\n'.join(text_parts)
+    text = re.sub(r'\n{3,}', '\n\n', text)
+    
+    return text.strip()
+
+
+__all__ = [
+    'extract_inline_content',
+    'extract_text_only',
+]

xgen_doc2chunk/core/processor/rtf_helper/rtf_decoder.py

@@ -0,0 +1,141 @@
+# xgen_doc2chunk/core/processor/rtf_helper/rtf_decoder.py
+"""
+RTF Decoding Utilities
+
+Encoding detection and decoding functions for RTF content.
+"""
+import logging
+import re
+from typing import List
+
+from xgen_doc2chunk.core.processor.rtf_helper.rtf_constants import (
+    CODEPAGE_ENCODING_MAP,
+    DEFAULT_ENCODINGS,
+)
+
+logger = logging.getLogger("xgen_doc2chunk.rtf.decoder")
+
+
+def detect_encoding(content: bytes, default_encoding: str = "cp949") -> str:
+    """
+    Detect encoding from RTF content.
+    
+    Looks for \\ansicpgXXXX pattern in the header.
+    
+    Args:
+        content: RTF binary data
+        default_encoding: Fallback encoding
+        
+    Returns:
+        Detected encoding string
+    """
+    try:
+        text = content[:1000].decode('ascii', errors='ignore')
+        
+        match = re.search(r'\\ansicpg(\d+)', text)
+        if match:
+            codepage = int(match.group(1))
+            encoding = CODEPAGE_ENCODING_MAP.get(codepage, 'cp1252')
+            logger.debug(f"RTF encoding detected: {encoding} (codepage {codepage})")
+            return encoding
+    except Exception as e:
+        logger.debug(f"Encoding detection failed: {e}")
+    
+    return default_encoding
+
+
+def decode_content(content: bytes, encoding: str = "cp949") -> str:
+    """
+    Decode RTF binary to string.
+    
+    Tries multiple encodings and returns first successful result.
+    
+    Args:
+        content: RTF binary data
+        encoding: Preferred encoding to try first
+        
+    Returns:
+        Decoded string
+    """
+    encodings = [encoding] + [e for e in DEFAULT_ENCODINGS if e != encoding]
+    
+    for enc in encodings:
+        try:
+            return content.decode(enc)
+        except (UnicodeDecodeError, LookupError):
+            continue
+    
+    return content.decode('cp1252', errors='replace')
+
+
+def decode_bytes(byte_list: List[int], encoding: str = "cp949") -> str:
+    """
+    Decode byte list to string.
+    
+    Args:
+        byte_list: List of byte values
+        encoding: Encoding to use
+        
+    Returns:
+        Decoded string
+    """
+    try:
+        return bytes(byte_list).decode(encoding)
+    except (UnicodeDecodeError, LookupError):
+        try:
+            return bytes(byte_list).decode('cp949')
+        except:
+            return bytes(byte_list).decode('latin-1', errors='replace')
+
+
+def decode_hex_escapes(text: str, encoding: str = "cp949") -> str:
+    """
+    Decode RTF hex escape sequences (\\'XX).
+    
+    Args:
+        text: RTF text with hex escapes
+        encoding: Encoding for decoding
+        
+    Returns:
+        Decoded text
+    """
+    if "\\'" not in text:
+        return text
+    
+    result = []
+    byte_buffer = []
+    i = 0
+    n = len(text)
+    
+    while i < n:
+        if i + 3 < n and text[i:i+2] == "\\'":
+            try:
+                hex_val = text[i+2:i+4]
+                byte_val = int(hex_val, 16)
+                byte_buffer.append(byte_val)
+                i += 4
+                continue
+            except ValueError:
+                pass
+        
+        # Flush byte buffer
+        if byte_buffer:
+            result.append(decode_bytes(byte_buffer, encoding))
+            byte_buffer = []
+        
+        result.append(text[i])
+        i += 1
+    
+    # Flush remaining bytes
+    if byte_buffer:
+        result.append(decode_bytes(byte_buffer, encoding))
+    
+    return ''.join(result)
+
+
+__all__ = [
+    'detect_encoding',
+    'decode_content',
+    'decode_bytes',
+    'decode_hex_escapes',
+]

xgen_doc2chunk/core/processor/rtf_helper/rtf_file_converter.py

@@ -0,0 +1,87 @@
+# xgen_doc2chunk/core/processor/rtf_helper/rtf_file_converter.py
+"""
+RTF File Converter
+
+RTF uses raw binary directly, so converter just passes through.
+All actual processing is done by Preprocessor in Handler.
+"""
+import logging
+from dataclasses import dataclass, field
+from typing import Any, BinaryIO, List, Optional
+
+from xgen_doc2chunk.core.functions.file_converter import BaseFileConverter
+
+logger = logging.getLogger("xgen_doc2chunk.rtf.converter")
+
+
+@dataclass
+class RTFConvertedData:
+    """
+    RTF converted data container.
+    
+    Attributes:
+        content: RTF content string (after preprocessing)
+        encoding: Detected encoding
+        image_tags: List of image tags from preprocessing
+        original_size: Original binary data size
+        has_images: Whether images were extracted
+    """
+    content: str
+    encoding: str = "cp949"
+    image_tags: List[str] = field(default_factory=list)
+    original_size: int = 0
+    has_images: bool = False
+    
+    def __post_init__(self):
+        """Set has_images based on image_tags."""
+        if self.image_tags:
+            self.has_images = True
+
+
+class RTFFileConverter(BaseFileConverter):
+    """
+    RTF file converter.
+    
+    RTF uses raw binary directly, so this converter just passes through.
+    All actual processing (image extraction, binary removal, decoding)
+    is done by RTFPreprocessor called from Handler.
+    """
+    
+    def __init__(self):
+        """Initialize RTFFileConverter."""
+        self.logger = logger
+    
+    def convert(
+        self,
+        file_data: bytes,
+        file_stream: Optional[BinaryIO] = None,
+        **kwargs
+    ) -> bytes:
+        """
+        Pass through binary data.
+        
+        RTF processing uses raw binary, so just return as-is.
+        
+        Args:
+            file_data: Raw binary RTF data
+            file_stream: Optional file stream (not used)
+            **kwargs: Not used
+                
+        Returns:
+            Original bytes (pass through)
+        """
+        return file_data
+    
+    def get_format_name(self) -> str:
+        """Return format name."""
+        return "RTF Document"
+    
+    def close(self, converted_object: Any) -> None:
+        """Nothing to close."""
+        pass
+
+
+__all__ = [
+    'RTFFileConverter',
+    'RTFConvertedData',
+]