xgen-doc2chunk 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

xgen_doc2chunk/core/processor/rtf_helper/rtf_text_cleaner.py

@@ -0,0 +1,389 @@
+# xgen_doc2chunk/core/processor/rtf_helper/rtf_text_cleaner.py
+"""
+RTF Text Cleaner
+
+Functions for removing RTF control codes and cleaning text.
+"""
+import re
+from typing import List
+
+from xgen_doc2chunk.core.processor.rtf_helper.rtf_constants import (
+    SHAPE_PROPERTY_NAMES,
+    SKIP_DESTINATIONS,
+    IMAGE_DESTINATIONS,
+)
+from xgen_doc2chunk.core.processor.rtf_helper.rtf_decoder import (
+    decode_bytes,
+)
+
+
+def clean_rtf_text(text: str, encoding: str = "cp949") -> str:
+    """
+    Remove RTF control codes and extract pure text.
+    
+    Uses token-based parsing to prevent content loss.
+    
+    Args:
+        text: RTF text
+        encoding: Encoding for decoding
+        
+    Returns:
+        Cleaned text
+    """
+    if not text:
+        return ""
+    
+    # Protect image tags (replace with temporary markers)
+    image_tags = []
+    def save_image_tag(m):
+        image_tags.append(m.group())
+        return f'\x00IMG{len(image_tags)-1}\x00'
+    
+    text = re.sub(r'\[image:[^\]]+\]', save_image_tag, text)
+    
+    # Remove shape properties
+    text = re.sub(r'\{\\sp\{\\sn\s*\w+\}\{\\sv\s*[^}]*\}\}', '', text)
+    text = re.sub(r'shapeType\d+[a-zA-Z0-9]+(?:posrelh\d+posrelv\d+)?', '', text)
+    text = re.sub(r'\\shp(?:inst|txt|left|right|top|bottom|bx\w+|by\w+|wr\d+|fblwtxt\d+|z\d+|lid\d+)\b\d*', '', text)
+    
+    result = []
+    i = 0
+    n = len(text)
+    
+    while i < n:
+        ch = text[i]
+        
+        # Restore image tag markers
+        if ch == '\x00' and i + 3 < n and text[i+1:i+4] == 'IMG':
+            end_idx = text.find('\x00', i + 4)
+            if end_idx != -1:
+                try:
+                    tag_idx = int(text[i+4:end_idx])
+                    result.append(image_tags[tag_idx])
+                    i = end_idx + 1
+                    continue
+                except (ValueError, IndexError):
+                    pass
+        
+        if ch == '\\':
+            if i + 1 < n:
+                next_ch = text[i + 1]
+                
+                # Special escapes
+                if next_ch == '\\':
+                    result.append('\\')
+                    i += 2
+                    continue
+                elif next_ch == '{':
+                    result.append('{')
+                    i += 2
+                    continue
+                elif next_ch == '}':
+                    result.append('}')
+                    i += 2
+                    continue
+                elif next_ch == '~':
+                    result.append('\u00A0')  # non-breaking space
+                    i += 2
+                    continue
+                elif next_ch == '-':
+                    result.append('\u00AD')  # soft hyphen
+                    i += 2
+                    continue
+                elif next_ch == '_':
+                    result.append('\u2011')  # non-breaking hyphen
+                    i += 2
+                    continue
+                elif next_ch == "'":
+                    # Hex escape \'XX
+                    if i + 3 < n:
+                        try:
+                            hex_val = text[i+2:i+4]
+                            byte_val = int(hex_val, 16)
+                            try:
+                                result.append(bytes([byte_val]).decode(encoding))
+                            except:
+                                try:
+                                    result.append(bytes([byte_val]).decode('cp1252'))
+                                except:
+                                    pass
+                            i += 4
+                            continue
+                        except (ValueError, IndexError):
+                            pass
+                    i += 1
+                    continue
+                elif next_ch == '*':
+                    # \* destination marker, skip
+                    i += 2
+                    continue
+                elif next_ch.isalpha():
+                    # Control word: \word[N][delimiter]
+                    j = i + 1
+                    while j < n and text[j].isalpha():
+                        j += 1
+                    
+                    control_word = text[i+1:j]
+                    
+                    # Skip numeric parameter
+                    while j < n and (text[j].isdigit() or text[j] == '-'):
+                        j += 1
+                    
+                    # Handle delimiter (space is part of control word)
+                    if j < n and text[j] == ' ':
+                        j += 1
+                    
+                    # Special control words
+                    if control_word in ('par', 'line'):
+                        result.append('\n')
+                    elif control_word == 'tab':
+                        result.append('\t')
+                    elif control_word == 'u':
+                        # Unicode: \uN?
+                        um = re.match(r'\\u(-?\d+)\??', text[i:])
+                        if um:
+                            try:
+                                code = int(um.group(1))
+                                if code < 0:
+                                    code += 65536
+                                result.append(chr(code))
+                            except:
+                                pass
+                            j = i + um.end()
+                    
+                    i = j
+                    continue
+            
+            i += 1
+        elif ch == '{' or ch == '}':
+            i += 1
+        elif ch == '\r' or ch == '\n':
+            i += 1
+        else:
+            result.append(ch)
+            i += 1
+    
+    text_result = ''.join(result)
+    
+    # Remove shape property names
+    shape_name_pattern = r'\b(' + '|'.join(SHAPE_PROPERTY_NAMES) + r')\b'
+    text_result = re.sub(shape_name_pattern, '', text_result)
+    
+    # Remove garbage numbers
+    text_result = re.sub(r'\s*-\d+\s*', ' ', text_result)
+    
+    # Remove hex data outside image tags
+    text_result = _remove_hex_outside_image_tags(text_result)
+    
+    # Normalize whitespace
+    text_result = re.sub(r'\s+', ' ', text_result)
+    
+    return text_result.strip()
+
+
+def _remove_hex_outside_image_tags(text: str) -> str:
+    """Remove long hex strings outside image tags."""
+    protected_ranges = []
+    for m in re.finditer(r'\[image:[^\]]+\]', text):
+        protected_ranges.append((m.start(), m.end()))
+    
+    if not protected_ranges:
+        return re.sub(r'(?<![a-zA-Z])[0-9a-fA-F]{32,}(?![a-zA-Z])', '', text)
+    
+    result = []
+    last_end = 0
+    for start, end in protected_ranges:
+        before = text[last_end:start]
+        before = re.sub(r'(?<![a-zA-Z])[0-9a-fA-F]{32,}(?![a-zA-Z])', '', before)
+        result.append(before)
+        result.append(text[start:end])
+        last_end = end
+    
+    after = text[last_end:]
+    after = re.sub(r'(?<![a-zA-Z])[0-9a-fA-F]{32,}(?![a-zA-Z])', '', after)
+    result.append(after)
+    return ''.join(result)
+
+
+def remove_destination_groups(content: str) -> str:
+    """
+    Remove RTF destination groups {\\*\\destination...}.
+    
+    Removes themedata, colorschememapping, latentstyles, datastore, etc.
+    to prevent metadata from being extracted as text.
+    
+    Args:
+        content: RTF content
+        
+    Returns:
+        Content with destination groups removed
+    """
+    result = []
+    i = 0
+    n = len(content)
+    
+    while i < n:
+        if content[i:i+3] == '{\\*':
+            j = i + 3
+            while j < n and content[j] in ' \t\r\n':
+                j += 1
+            
+            if j < n and content[j] == '\\':
+                k = j + 1
+                while k < n and content[k].isalpha():
+                    k += 1
+                ctrl_word = content[j+1:k]
+                
+                if ctrl_word in SKIP_DESTINATIONS:
+                    depth = 1
+                    i += 1
+                    while i < n and depth > 0:
+                        if content[i] == '{':
+                            depth += 1
+                        elif content[i] == '}':
+                            depth -= 1
+                        i += 1
+                    continue
+                
+                if ctrl_word in IMAGE_DESTINATIONS:
+                    depth = 1
+                    group_start = i
+                    i += 1
+                    while i < n and depth > 0:
+                        if content[i] == '{':
+                            depth += 1
+                        elif content[i] == '}':
+                            depth -= 1
+                        i += 1
+                    
+                    group_content = content[group_start:i]
+                    image_tag_match = re.search(r'\[image:[^\]]+\]', group_content)
+                    if image_tag_match:
+                        tag = image_tag_match.group()
+                        if '/uploads/.' not in tag and 'uploads/.' not in tag:
+                            result.append(tag)
+                    continue
+        
+        result.append(content[i])
+        i += 1
+    
+    return ''.join(result)
+
+
+def remove_shape_groups(content: str) -> str:
+    """
+    Remove shape groups but preserve text in shptxt.
+    
+    RTF Shape structure:
+    {\\shp{\\*\\shpinst...{\\sp{\\sn xxx}{\\sv yyy}}...{\\shptxt actual_text}}}
+    
+    Args:
+        content: RTF content
+        
+    Returns:
+        Content with shape groups cleaned
+    """
+    result = []
+    i = 0
+    
+    while i < len(content):
+        if content[i:i+5] == '{\\shp' or content[i:i+10] == '{\\*\\shpinst':
+            depth = 1
+            i += 1
+            shptxt_content = []
+            in_shptxt = False
+            shptxt_depth = 0
+            
+            while i < len(content) and depth > 0:
+                if content[i] == '{':
+                    if content[i:i+8] == '{\\shptxt':
+                        in_shptxt = True
+                        shptxt_depth = depth + 1
+                        i += 8
+                        continue
+                    depth += 1
+                elif content[i] == '}':
+                    if in_shptxt and depth == shptxt_depth:
+                        in_shptxt = False
+                    depth -= 1
+                elif in_shptxt:
+                    shptxt_content.append(content[i])
+                i += 1
+            
+            if shptxt_content:
+                result.append(''.join(shptxt_content))
+        else:
+            result.append(content[i])
+            i += 1
+    
+    return ''.join(result)
+
+
+def remove_shape_property_groups(content: str) -> str:
+    """
+    Remove shape property groups {\\sp{\\sn xxx}{\\sv yyy}}.
+    
+    Args:
+        content: RTF content
+        
+    Returns:
+        Content with shape properties removed
+    """
+    content = re.sub(r'\{\\sp\{\\sn\s*[^}]*\}\{\\sv\s*[^}]*\}\}', '', content)
+    content = re.sub(r'\{\\sp\s*[^}]*\}', '', content)
+    content = re.sub(r'\{\\sn\s*[^}]*\}', '', content)
+    content = re.sub(r'\{\\sv\s*[^}]*\}', '', content)
+    return content
+
+
+def remove_shprslt_blocks(content: str) -> str:
+    """
+    Remove \\shprslt{...} blocks.
+    
+    Word saves Shape (drawing/table) in \\shp block and duplicates
+    the same content in \\shprslt block for backward compatibility.
+    
+    Args:
+        content: RTF content
+        
+    Returns:
+        Content with \\shprslt blocks removed
+    """
+    result = []
+    i = 0
+    pattern = '\\shprslt'
+    
+    while i < len(content):
+        idx = content.find(pattern, i)
+        if idx == -1:
+            result.append(content[i:])
+            break
+        
+        result.append(content[i:idx])
+        
+        brace_start = content.find('{', idx)
+        if brace_start == -1:
+            i = idx + len(pattern)
+            continue
+        
+        depth = 1
+        j = brace_start + 1
+        while j < len(content) and depth > 0:
+            if content[j] == '{':
+                depth += 1
+            elif content[j] == '}':
+                depth -= 1
+            j += 1
+        
+        i = j
+    
+    return ''.join(result)
+
+
+__all__ = [
+    'clean_rtf_text',
+    'remove_destination_groups',
+    'remove_shape_groups',
+    'remove_shape_property_groups',
+    'remove_shprslt_blocks',
+]

xgen_doc2chunk/core/processor/text_handler.py

@@ -0,0 +1,95 @@
+# xgen_doc2chunk/core/processor/text_handler.py
+"""
+Text Handler - Text File Processor
+
+Class-based handler for text files inheriting from BaseHandler.
+"""
+import logging
+from typing import List, Optional, TYPE_CHECKING
+
+from xgen_doc2chunk.core.processor.base_handler import BaseHandler
+from xgen_doc2chunk.core.functions.utils import clean_text, clean_code_text
+from xgen_doc2chunk.core.functions.chart_extractor import BaseChartExtractor, NullChartExtractor
+from xgen_doc2chunk.core.processor.text_helper.text_image_processor import TextImageProcessor
+from xgen_doc2chunk.core.functions.img_processor import ImageProcessor
+
+if TYPE_CHECKING:
+    from xgen_doc2chunk.core.document_processor import CurrentFile
+
+logger = logging.getLogger("document-processor")
+
+
+DEFAULT_ENCODINGS = ['utf-8', 'utf-8-sig', 'cp949', 'euc-kr', 'latin-1', 'ascii']
+
+
+class TextHandler(BaseHandler):
+    """Text File Processing Handler Class"""
+
+    def _create_file_converter(self):
+        """Create text-specific file converter."""
+        from xgen_doc2chunk.core.processor.text_helper.text_file_converter import TextFileConverter
+        return TextFileConverter()
+
+    def _create_preprocessor(self):
+        """Create text-specific preprocessor."""
+        from xgen_doc2chunk.core.processor.text_helper.text_preprocessor import TextPreprocessor
+        return TextPreprocessor()
+
+    def _create_chart_extractor(self) -> BaseChartExtractor:
+        """Text files do not contain charts. Return NullChartExtractor."""
+        return NullChartExtractor(self._chart_processor)
+
+    def _create_metadata_extractor(self):
+        """Text files do not have embedded metadata. Return None (uses NullMetadataExtractor)."""
+        return None
+
+    def _create_format_image_processor(self) -> ImageProcessor:
+        """Create text-specific image processor."""
+        return TextImageProcessor()
+
+    def extract_text(
+        self,
+        current_file: "CurrentFile",
+        extract_metadata: bool = True,
+        file_type: Optional[str] = None,
+        encodings: Optional[List[str]] = None,
+        is_code: bool = False,
+        **kwargs
+    ) -> str:
+        """
+        Extract text from text file.
+
+        Args:
+            current_file: CurrentFile dict containing file info and binary data
+            extract_metadata: Whether to extract metadata (ignored for text files)
+            file_type: File type (extension)
+            encodings: List of encodings to try
+            is_code: Whether this is a code file
+            **kwargs: Additional options
+
+        Returns:
+            Extracted text
+        """
+        file_path = current_file.get("file_path", "unknown")
+        file_data = current_file.get("file_data", b"")
+        enc = encodings or DEFAULT_ENCODINGS
+
+        # Step 1: No file_converter for text files (direct decode)
+        # Step 2: Preprocess - clean_content is the TRUE SOURCE
+        preprocessed = self.preprocess(file_data)
+        file_data = preprocessed.clean_content  # TRUE SOURCE
+
+        for e in enc:
+            try:
+                text = file_data.decode(e)
+                self.logger.info(f"Successfully decoded {file_path} with {e} encoding")
+                return clean_code_text(text) if is_code else clean_text(text)
+            except UnicodeDecodeError:
+                self.logger.debug(f"Failed to decode {file_path} with {e}, trying next...")
+                continue
+            except Exception as ex:
+                self.logger.error(f"Error decoding file {file_path} with {e}: {ex}")
+                continue
+
+        raise Exception(f"Could not decode file {file_path} with any supported encoding")
+

xgen_doc2chunk/core/processor/text_helper/__init__.py

@@ -0,0 +1,17 @@
+# xgen_doc2chunk/core/processor/text_helper/__init__.py
+"""
+Text Helper 모듈
+
+텍스트 파일 처리에 필요한 유틸리티를 제공합니다.
+
+모듈 구성:
+- text_image_processor: 텍스트 파일용 이미지 프로세서
+"""
+
+from xgen_doc2chunk.core.processor.text_helper.text_image_processor import (
+    TextImageProcessor,
+)
+
+__all__ = [
+    "TextImageProcessor",
+]

xgen_doc2chunk/core/processor/text_helper/text_file_converter.py

@@ -0,0 +1,28 @@
+# xgen_doc2chunk/core/processor/text_helper/text_file_converter.py
+"""
+TextFileConverter - Text file format converter
+
+Converts binary text data to string with encoding detection.
+"""
+from typing import Optional, BinaryIO
+
+from xgen_doc2chunk.core.functions.file_converter import TextFileConverter as BaseTextFileConverter
+
+
+class TextFileConverter(BaseTextFileConverter):
+    """
+    Text file converter.
+    
+    Converts binary text data to decoded string.
+    Inherits from base TextFileConverter.
+    """
+    
+    def __init__(self):
+        """Initialize with common text encodings."""
+        super().__init__(encodings=['utf-8', 'utf-8-sig', 'cp949', 'euc-kr', 'latin-1', 'ascii'])
+    
+    def get_format_name(self) -> str:
+        """Return format name."""
+        enc = self._detected_encoding or 'unknown'
+        return f"Text File ({enc})"
+

xgen_doc2chunk/core/processor/text_helper/text_image_processor.py

@@ -0,0 +1,75 @@
+# xgen_doc2chunk/core/processor/text_helper/text_image_processor.py
+"""
+Text Image Processor
+
+Provides text-specific image processing that inherits from ImageProcessor.
+Text files do not contain embedded images, so this is a minimal implementation.
+"""
+import logging
+from typing import Any, Optional
+
+from xgen_doc2chunk.core.functions.img_processor import ImageProcessor
+from xgen_doc2chunk.core.functions.storage_backend import BaseStorageBackend
+
+logger = logging.getLogger("xgen_doc2chunk.image_processor.text")
+
+
+class TextImageProcessor(ImageProcessor):
+    """
+    Text-specific image processor.
+    
+    Inherits from ImageProcessor and provides text-specific processing.
+    Text files do not contain embedded images, so this processor
+    provides a consistent interface without additional functionality.
+    
+    This class exists to maintain interface consistency across all handlers.
+    
+    Example:
+        processor = TextImageProcessor()
+        
+        # No images in text files, but interface is consistent
+        tag = processor.process_image(image_data)  # Falls back to base implementation
+    """
+    
+    def __init__(
+        self,
+        directory_path: str = "temp/images",
+        tag_prefix: str = "[Image:",
+        tag_suffix: str = "]",
+        storage_backend: Optional[BaseStorageBackend] = None,
+    ):
+        """
+        Initialize TextImageProcessor.
+        
+        Args:
+            directory_path: Image save directory
+            tag_prefix: Tag prefix for image references
+            tag_suffix: Tag suffix for image references
+            storage_backend: Storage backend for saving images
+        """
+        super().__init__(
+            directory_path=directory_path,
+            tag_prefix=tag_prefix,
+            tag_suffix=tag_suffix,
+            storage_backend=storage_backend,
+        )
+    
+    def process_image(
+        self,
+        image_data: bytes,
+        **kwargs
+    ) -> Optional[str]:
+        """
+        Process and save image data.
+        
+        Text files do not contain embedded images, so this method
+        delegates to the base implementation.
+        
+        Args:
+            image_data: Raw image binary data
+            **kwargs: Additional options
+            
+        Returns:
+            Image tag string or None if processing failed
+        """
+        return super().process_image(image_data, **kwargs)

xgen_doc2chunk/core/processor/text_helper/text_preprocessor.py

@@ -0,0 +1,82 @@
+# xgen_doc2chunk/core/processor/text_helper/text_preprocessor.py
+"""
+Text Preprocessor - Process text content after conversion.
+
+Processing Pipeline Position:
+    1. TextFileConverter.convert() ??str
+    2. TextPreprocessor.preprocess() ??PreprocessedData (THIS STEP)
+    3. TextMetadataExtractor.extract() ??DocumentMetadata (if any)
+    4. Content extraction
+
+Current Implementation:
+    - Pass-through (Text uses decoded string content directly)
+"""
+import logging
+from typing import Any, Dict
+
+from xgen_doc2chunk.core.functions.preprocessor import (
+    BasePreprocessor,
+    PreprocessedData,
+)
+
+logger = logging.getLogger("xgen_doc2chunk.text.preprocessor")
+
+
+class TextPreprocessor(BasePreprocessor):
+    """
+    Text Content Preprocessor.
+
+    Currently a pass-through implementation as text processing
+    is straightforward.
+    """
+
+    def preprocess(
+        self,
+        converted_data: Any,
+        **kwargs
+    ) -> PreprocessedData:
+        """
+        Preprocess the converted text content.
+
+        Args:
+            converted_data: Text string from TextFileConverter
+            **kwargs: Additional options
+
+        Returns:
+            PreprocessedData with the content
+        """
+        metadata: Dict[str, Any] = {}
+
+        content = ""
+        encoding = kwargs.get("encoding", "utf-8")
+
+        if isinstance(converted_data, str):
+            content = converted_data
+            metadata['char_count'] = len(content)
+            metadata['line_count'] = len(content.split('\n'))
+        elif isinstance(converted_data, bytes):
+            content = converted_data.decode(encoding, errors='replace')
+            metadata['char_count'] = len(content)
+            metadata['line_count'] = len(content.split('\n'))
+
+        logger.debug("Text preprocessor: pass-through, metadata=%s", metadata)
+
+        # clean_content is the TRUE SOURCE - contains the processed text/bytes
+        return PreprocessedData(
+            raw_content=converted_data,
+            clean_content=converted_data,  # TRUE SOURCE - bytes or str
+            encoding=encoding,
+            extracted_resources={},
+            metadata=metadata,
+        )
+
+    def get_format_name(self) -> str:
+        """Return format name."""
+        return "Text Preprocessor"
+
+    def validate(self, data: Any) -> bool:
+        """Validate if data is text content."""
+        return isinstance(data, (str, bytes))
+
+
+__all__ = ['TextPreprocessor']