xgen-doc2chunk 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

xgen_doc2chunk/core/processor/hwp_helper/hwp_metadata.py

@@ -0,0 +1,236 @@
+# xgen_doc2chunk/core/processor/hwp_helper/hwp_metadata.py
+"""
+HWP Metadata Extraction Module
+
+Provides HWPMetadataExtractor class for extracting metadata from HWP 5.0 OLE files.
+Implements BaseMetadataExtractor interface.
+
+Extraction methods:
+1. olefile's get_metadata() - OLE standard metadata
+2. HwpSummaryInformation stream direct parsing - HWP-specific metadata
+
+Note: HWP is a Korean-native document format, so Korean metadata labels
+are preserved in output for proper display.
+"""
+import struct
+import logging
+from datetime import datetime
+from typing import Dict, Any, Optional
+
+import olefile
+
+from xgen_doc2chunk.core.functions.metadata_extractor import (
+    BaseMetadataExtractor,
+    DocumentMetadata,
+)
+
+logger = logging.getLogger("document-processor")
+
+
+class HWPMetadataExtractor(BaseMetadataExtractor):
+    """
+    HWP Metadata Extractor.
+    
+    Extracts metadata from olefile OleFileIO objects.
+    Supports both OLE standard metadata and HWP-specific HwpSummaryInformation.
+    
+    Supported fields:
+    - title, subject, author, keywords, comments
+    - last_saved_by, create_time, last_saved_time
+    
+    Usage:
+        extractor = HWPMetadataExtractor()
+        metadata = extractor.extract(ole_file)
+        text = extractor.format(metadata)
+    """
+    
+    def extract(self, source: olefile.OleFileIO) -> DocumentMetadata:
+        """
+        Extract metadata from HWP file.
+        
+        Args:
+            source: olefile OleFileIO object
+            
+        Returns:
+            DocumentMetadata instance containing extracted metadata.
+        """
+        metadata_dict: Dict[str, Any] = {}
+        
+        # Method 1: Use olefile's get_metadata()
+        try:
+            ole_meta = source.get_metadata()
+            
+            if ole_meta:
+                if ole_meta.title:
+                    metadata_dict['title'] = ole_meta.title
+                if ole_meta.subject:
+                    metadata_dict['subject'] = ole_meta.subject
+                if ole_meta.author:
+                    metadata_dict['author'] = ole_meta.author
+                if ole_meta.keywords:
+                    metadata_dict['keywords'] = ole_meta.keywords
+                if ole_meta.comments:
+                    metadata_dict['comments'] = ole_meta.comments
+                if ole_meta.last_saved_by:
+                    metadata_dict['last_saved_by'] = ole_meta.last_saved_by
+                if ole_meta.create_time:
+                    metadata_dict['create_time'] = ole_meta.create_time
+                if ole_meta.last_saved_time:
+                    metadata_dict['last_saved_time'] = ole_meta.last_saved_time
+            
+            self.logger.debug(f"Extracted OLE metadata: {list(metadata_dict.keys())}")
+            
+        except Exception as e:
+            self.logger.warning(f"Failed to extract OLE metadata: {e}")
+        
+        # Method 2: Parse HwpSummaryInformation stream directly
+        try:
+            hwp_summary_stream = '\x05HwpSummaryInformation'
+            if source.exists(hwp_summary_stream):
+                self.logger.debug("Found HwpSummaryInformation stream, attempting to parse...")
+                stream = source.openstream(hwp_summary_stream)
+                data = stream.read()
+                hwp_meta = parse_hwp_summary_information(data)
+                
+                # HWP-specific metadata takes priority
+                for key, value in hwp_meta.items():
+                    if value:
+                        metadata_dict[key] = value
+                        
+        except Exception as e:
+            self.logger.debug(f"Failed to parse HwpSummaryInformation: {e}")
+        
+        return DocumentMetadata(
+            title=metadata_dict.get('title'),
+            subject=metadata_dict.get('subject'),
+            author=metadata_dict.get('author'),
+            keywords=metadata_dict.get('keywords'),
+            comments=metadata_dict.get('comments'),
+            last_saved_by=metadata_dict.get('last_saved_by'),
+            create_time=metadata_dict.get('create_time'),
+            last_saved_time=metadata_dict.get('last_saved_time'),
+        )
+
+
+def parse_hwp_summary_information(data: bytes) -> Dict[str, Any]:
+    """
+    Parse HwpSummaryInformation stream (OLE Property Set format).
+    
+    OLE Property Set structure:
+    - Header (28 bytes)
+    - Section(s) containing property ID/offset pairs
+    - Property values (string, datetime, etc.)
+    
+    Args:
+        data: HwpSummaryInformation stream binary data
+        
+    Returns:
+        Dictionary containing parsed metadata.
+    """
+    metadata = {}
+    
+    try:
+        if len(data) < 28:
+            return metadata
+        
+        pos = 0
+        _byte_order = struct.unpack('<H', data[pos:pos+2])[0]  # noqa: F841
+        pos = 28  # Skip header
+        
+        if len(data) < pos + 20:
+            return metadata
+        
+        # Section Header: FMTID (16 bytes) + Offset (4 bytes)
+        section_offset = struct.unpack('<I', data[pos+16:pos+20])[0]
+        
+        if section_offset >= len(data):
+            return metadata
+        
+        # Parse section
+        pos = section_offset
+        if len(data) < pos + 8:
+            return metadata
+        
+        _section_size = struct.unpack('<I', data[pos:pos+4])[0]  # noqa: F841
+        num_properties = struct.unpack('<I', data[pos+4:pos+8])[0]
+        pos += 8
+        
+        # Read property ID/offset pairs
+        properties = []
+        for _ in range(min(num_properties, 50)):
+            if len(data) < pos + 8:
+                break
+            prop_id = struct.unpack('<I', data[pos:pos+4])[0]
+            prop_offset = struct.unpack('<I', data[pos+4:pos+8])[0]
+            properties.append((prop_id, prop_offset))
+            pos += 8
+        
+        # Read property values
+        for prop_id, prop_offset in properties:
+            abs_offset = section_offset + prop_offset
+            if abs_offset + 4 >= len(data):
+                continue
+            
+            prop_type = struct.unpack('<I', data[abs_offset:abs_offset+4])[0]
+            value_offset = abs_offset + 4
+            
+            value = None
+            
+            if prop_type == 0x1E:  # ANSI String
+                if value_offset + 4 < len(data):
+                    str_len = struct.unpack('<I', data[value_offset:value_offset+4])[0]
+                    if str_len > 0 and value_offset + 4 + str_len <= len(data):
+                        try:
+                            value = data[value_offset+4:value_offset+4+str_len].decode('cp949', errors='ignore').rstrip('\x00')
+                        except Exception:
+                            value = data[value_offset+4:value_offset+4+str_len].decode('utf-8', errors='ignore').rstrip('\x00')
+            
+            elif prop_type == 0x1F:  # Unicode String
+                if value_offset + 4 < len(data):
+                    str_len = struct.unpack('<I', data[value_offset:value_offset+4])[0]
+                    byte_len = str_len * 2
+                    if str_len > 0 and value_offset + 4 + byte_len <= len(data):
+                        value = data[value_offset+4:value_offset+4+byte_len].decode('utf-16le', errors='ignore').rstrip('\x00')
+            
+            elif prop_type == 0x40:  # FILETIME
+                if value_offset + 8 <= len(data):
+                    filetime = struct.unpack('<Q', data[value_offset:value_offset+8])[0]
+                    if filetime > 0:
+                        try:
+                            seconds = filetime / 10000000
+                            epoch_diff = 11644473600
+                            unix_time = seconds - epoch_diff
+                            if 0 < unix_time < 2000000000:
+                                value = datetime.fromtimestamp(unix_time)
+                        except Exception:
+                            pass
+            
+            # Property ID mapping
+            if value:
+                if prop_id == 0x02:
+                    metadata['title'] = value
+                elif prop_id == 0x03:
+                    metadata['subject'] = value
+                elif prop_id == 0x04:
+                    metadata['author'] = value
+                elif prop_id == 0x05:
+                    metadata['keywords'] = value
+                elif prop_id == 0x06:
+                    metadata['comments'] = value
+                elif prop_id == 0x08:
+                    metadata['last_saved_by'] = value
+                elif prop_id == 0x0C:
+                    metadata['create_time'] = value
+                elif prop_id == 0x0D:
+                    metadata['last_saved_time'] = value
+    
+    except Exception as e:
+        logger.debug(f"Error parsing HWP summary information: {e}")
+    
+    return metadata
+
+
+__all__ = [
+    'HWPMetadataExtractor',
+    'parse_hwp_summary_information',
+]

xgen_doc2chunk/core/processor/hwp_helper/hwp_preprocessor.py

@@ -0,0 +1,82 @@
+# xgen_doc2chunk/core/processor/hwp_helper/hwp_preprocessor.py
+"""
+HWP Preprocessor - Process HWP OLE document after conversion.
+
+Processing Pipeline Position:
+    1. HWPFileConverter.convert() ??olefile.OleFileIO
+    2. HWPPreprocessor.preprocess() ??PreprocessedData (THIS STEP)
+    3. HWPMetadataExtractor.extract() ??DocumentMetadata
+    4. Content extraction (body text, tables, images)
+
+Current Implementation:
+    - Pass-through (HWP uses olefile object directly)
+"""
+import logging
+from typing import Any, Dict
+
+from xgen_doc2chunk.core.functions.preprocessor import (
+    BasePreprocessor,
+    PreprocessedData,
+)
+
+logger = logging.getLogger("xgen_doc2chunk.hwp.preprocessor")
+
+
+class HWPPreprocessor(BasePreprocessor):
+    """
+    HWP OLE Document Preprocessor.
+
+    Currently a pass-through implementation as HWP processing
+    is handled during the content extraction phase using olefile.
+    """
+
+    def preprocess(
+        self,
+        converted_data: Any,
+        **kwargs
+    ) -> PreprocessedData:
+        """
+        Preprocess the converted HWP OLE document.
+
+        Args:
+            converted_data: olefile.OleFileIO object from HWPFileConverter
+            **kwargs: Additional options
+
+        Returns:
+            PreprocessedData with the OLE object and any extracted resources
+        """
+        metadata: Dict[str, Any] = {}
+
+        if hasattr(converted_data, 'listdir'):
+            try:
+                streams = converted_data.listdir()
+                metadata['stream_count'] = len(streams)
+                # Check for common HWP streams
+                has_body = any('BodyText' in '/'.join(s) for s in streams)
+                has_docinfo = any('DocInfo' in '/'.join(s) for s in streams)
+                metadata['has_body_text'] = has_body
+                metadata['has_doc_info'] = has_docinfo
+            except Exception:
+                pass
+
+        logger.debug("HWP preprocessor: pass-through, metadata=%s", metadata)
+
+        # clean_content is the TRUE SOURCE - contains the OLE object
+        return PreprocessedData(
+            raw_content=converted_data,
+            clean_content=converted_data,  # TRUE SOURCE - olefile.OleFileIO
+            encoding="utf-8",
+            extracted_resources={},
+            metadata=metadata,
+        )
+
+    def get_format_name(self) -> str:
+        """Return format name."""
+        return "HWP Preprocessor"
+
+    def validate(self, data: Any) -> bool:
+        """Validate if data is an OLE file object."""
+        return hasattr(data, 'listdir') and hasattr(data, 'openstream')
+
+
+__all__ = ['HWPPreprocessor']

xgen_doc2chunk/core/processor/hwp_helper/hwp_record.py

@@ -0,0 +1,149 @@
+"""
+HWP Record 파싱 클래스
+"""
+import struct
+import logging
+from itertools import islice
+from typing import Optional
+
+from xgen_doc2chunk.core.processor.hwp_helper.hwp_constants import HWPTAG_PARA_TEXT
+
+logger = logging.getLogger("document-processor")
+
+
+class HwpRecord:
+    def __init__(self, tag_id: int, payload: bytes, parent: 'HwpRecord' = None):
+        self.tag_id = tag_id
+        self.payload = payload
+        self.parent = parent
+        self.children = []
+
+    def get_next_siblings(self, count=None):
+        if not self.parent:
+            return []
+        try:
+            start_idx = self.parent.children.index(self) + 1
+            if count is None:
+                end_idx = None
+            else:
+                end_idx = start_idx + count
+            return islice(self.parent.children, start_idx, end_idx)
+        except ValueError:
+            return []
+
+    def get_text(self) -> str:
+        """
+        Extract text from HWPTAG_PARA_TEXT payload, handling control characters.
+        Returns text with \\x0b markers for extended controls.
+        """
+        if self.tag_id != HWPTAG_PARA_TEXT:
+            return ""
+
+        # HWP text is UTF-16LE
+        text = ''
+        payload = self.payload
+        cursor = 0
+
+        while cursor < len(payload):
+            if cursor + 1 >= len(payload):
+                break
+
+            code = struct.unpack('<H', payload[cursor:cursor+2])[0]
+
+            if code >= 32:
+                # Normal char
+                text += chr(code)
+                cursor += 2
+            else:
+                # Control char handling
+                if code == 13: # Para break
+                    text += '\n'
+                    cursor += 2
+                elif code == 10: # Line break
+                    text += '\n'
+                    cursor += 2
+                elif code == 9: # Tab
+                    text += '\t'
+                    cursor += 2
+                else:
+                    # Extended control chars have extra data
+                    # Simplified logic based on known HWP structure
+                    size = 1
+                    if code in [4, 5, 6, 7, 8, 9, 19, 20]: # Inline
+                        size = 8
+                    elif code in [1, 2, 3, 11, 12, 14, 15, 16, 17, 18, 21, 22, 23]: # Extended
+                        size = 8
+                        # Code 11 is the standard "Extended Control" marker (for Tables, GSO, etc.)
+                        if code == 11:
+                            text += '\x0b'
+                            # logger.debug(f"Found Extended Control Marker (11) at cursor {cursor}")
+
+                    cursor += size * 2
+
+        return text
+
+    @staticmethod
+    def build_tree(data: bytes) -> 'HwpRecord':
+        root = HwpRecord(0, b'')
+        pos = 0
+        size = len(data)
+
+        # Stack to keep track of parents based on level
+        # Level 0 is root children
+        # stack[0] = root
+        stack = {0: root}
+
+        while pos < size:
+            try:
+                if pos + 4 > size:
+                    break
+                header = struct.unpack('<I', data[pos:pos+4])[0]
+                pos += 4
+
+                tag_id = header & 0x3FF
+                level = (header >> 10) & 0x3FF
+                rec_len = (header >> 20) & 0xFFF
+
+                if rec_len == 0xFFF:
+                    if pos + 4 > size:
+                        break
+                    rec_len = struct.unpack('<I', data[pos:pos+4])[0]
+                    pos += 4
+
+                if pos + rec_len > size:
+                    # Truncated record, stop parsing
+                    break
+
+                payload = data[pos:pos+rec_len]
+                pos += rec_len
+
+                # Determine parent
+                parent = stack.get(level - 1, root)
+                if level == 0:
+                    parent = root
+
+                # If parent is not in stack (gap in levels), fallback to root or nearest
+                if parent is None:
+                    # Find nearest lower level
+                    for l in range(level - 1, -1, -1):
+                        if l in stack:
+                            parent = stack[l]
+                            break
+                    if parent is None:
+                        parent = root
+
+                record = HwpRecord(tag_id, payload, parent)
+                parent.children.append(record)
+
+                # Update stack for this level
+                stack[level] = record
+
+                # Clear deeper levels from stack as we moved to a new node at this level
+                keys_to_remove = [k for k in stack.keys() if k > level]
+                for k in keys_to_remove:
+                    del stack[k]
+            except Exception as e:
+                logger.debug(f"Error parsing HWP record at pos {pos}: {e}")
+                break
+
+        return root

xgen_doc2chunk/core/processor/hwp_helper/hwp_recovery.py

@@ -0,0 +1,217 @@
+# service/document_processor/processor/hwp_helper/hwp_recovery.py
+"""
+HWP 손상 파일 복구 유틸리티
+
+손상되었거나 비-OLE HWP 파일에서 텍스트와 이미지를 복구합니다.
+- extract_text_from_stream_raw: 바이너리에서 UTF-16LE 문자열 추출
+- recover_images_from_raw: 이미지 시그니처 스캔 후 추출
+- find_zlib_streams: zlib 압축 스트림 찾기 및 해제
+"""
+import zlib
+import struct
+import logging
+from typing import List, Tuple, Optional
+
+from xgen_doc2chunk.core.functions.img_processor import ImageProcessor
+
+logger = logging.getLogger("document-processor")
+
+
+def extract_text_from_stream_raw(data: bytes) -> str:
+    """
+    Fallback: 레코드 파싱 없이 바이너리 데이터에서 UTF-16LE 문자열을 추출합니다.
+
+    한글 완성형(0xAC00-0xD7A3), ASCII 인쇄 가능 문자, 한글 자모,
+    CJK 구두점 등 유효한 문자만 추출합니다.
+
+    Args:
+        data: 바이너리 데이터
+
+    Returns:
+        추출된 텍스트 문자열
+    """
+    text_parts = []
+    current_run = []
+
+    for i in range(0, len(data) - 1, 2):
+        chunk = data[i:i+2]
+        val = struct.unpack('<H', chunk)[0]
+
+        is_valid = (
+            (0xAC00 <= val <= 0xD7A3) or   # 한글 완성형
+            (0x0020 <= val <= 0x007E) or   # ASCII 인쇄 가능
+            (0x3130 <= val <= 0x318F) or   # 한글 호환 자모
+            (0x1100 <= val <= 0x11FF) or   # 한글 자모
+            (0x3000 <= val <= 0x303F) or   # CJK 구두점
+            val in [10, 13, 9]              # 줄바꿈, 탭
+        )
+
+        if is_valid:
+            if val in [10, 13]:
+                if current_run:
+                    text_parts.append("".join(current_run))
+                    current_run = []
+                text_parts.append("\n")
+            elif val == 9:
+                current_run.append("\t")
+            else:
+                current_run.append(chr(val))
+        else:
+            if len(current_run) > 0:
+                text_parts.append("".join(current_run))
+            current_run = []
+
+    if current_run:
+        text_parts.append("".join(current_run))
+
+    final_parts = [p for p in text_parts if len(p.strip()) > 0]
+    return "".join(final_parts)
+
+
+def find_zlib_streams(raw_data: bytes, min_size: int = 50) -> List[Tuple[int, bytes]]:
+    """
+    바이너리 데이터에서 zlib 압축 스트림을 찾아 압축 해제합니다.
+
+    zlib 헤더(0x78 0x9c, 0x78 0x01, 0x78 0xda)를 스캔하고
+    압축 해제를 시도합니다.
+
+    Args:
+        raw_data: 바이너리 데이터
+        min_size: 유효한 스트림으로 인정할 최소 압축 해제 크기
+
+    Returns:
+        (시작 오프셋, 압축 해제된 데이터) 튜플 리스트
+    """
+    zlib_headers = [b'\x78\x9c', b'\x78\x01', b'\x78\xda']
+
+    decompressed_chunks = []
+    start = 0
+    file_len = len(raw_data)
+
+    while start < file_len:
+        next_header_pos = -1
+
+        for h in zlib_headers:
+            pos = raw_data.find(h, start)
+            if pos != -1:
+                if next_header_pos == -1 or pos < next_header_pos:
+                    next_header_pos = pos
+
+        if next_header_pos == -1:
+            break
+
+        start = next_header_pos
+
+        try:
+            dobj = zlib.decompressobj()
+            decompressed = dobj.decompress(raw_data[start:])
+
+            if len(decompressed) > min_size:
+                decompressed_chunks.append((start, decompressed))
+
+            if dobj.unused_data:
+                compressed_size = len(raw_data[start:]) - len(dobj.unused_data)
+                start += compressed_size
+            else:
+                start += 1
+
+        except (zlib.error, Exception):
+            start += 1
+
+    return decompressed_chunks
+
+
+def recover_images_from_raw(
+    raw_data: bytes,
+    image_processor: ImageProcessor
+) -> str:
+    """
+    raw 바이너리 데이터에서 이미지 시그니처(JPEG, PNG)를 스캔하여 로컬에 저장합니다.
+
+    Args:
+        raw_data: 바이너리 데이터
+        image_processor: 이미지 프로세서 인스턴스
+
+    Returns:
+        이미지 태그들을 결합한 문자열
+    """
+
+    results = []
+
+    # JPEG 추출
+    start = 0
+    while True:
+        start = raw_data.find(b'\xff\xd8\xff', start)
+        if start == -1:
+            break
+
+        end = raw_data.find(b'\xff\xd9', start)
+        if end == -1:
+            break
+
+        end += 2
+
+        size = end - start
+        if 100 < size < 10 * 1024 * 1024:
+            img_data = raw_data[start:end]
+
+            image_tag = image_processor.save_image(img_data)
+            if image_tag:
+                results.append(image_tag)
+
+        start = end
+
+    # PNG 추출
+    png_sig = b'\x89PNG\r\n\x1a\n'
+    png_end = b'IEND\xae\x42\x60\x82'
+
+    start = 0
+    while True:
+        start = raw_data.find(png_sig, start)
+        if start == -1:
+            break
+
+        end = raw_data.find(png_end, start)
+        if end == -1:
+            break
+
+        end += len(png_end)
+
+        size = end - start
+        if 100 < size < 10 * 1024 * 1024:
+            img_data = raw_data[start:end]
+
+            image_tag = image_processor.save_image(img_data)
+            if image_tag:
+                results.append(image_tag)
+
+        start = end
+
+    return "\n\n".join(results)
+
+
+def check_file_signature(raw_data: bytes) -> Optional[str]:
+    """
+    파일 시그니처를 확인하여 파일 형식을 식별합니다.
+
+    Args:
+        raw_data: 바이너리 데이터
+
+    Returns:
+        파일 형식 문자열 또는 None
+    """
+    if raw_data.startswith(b'\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1'):
+        return "OLE"
+    elif raw_data.startswith(b'PK\x03\x04'):
+        return "ZIP/HWPX"
+    elif b'HWP Document File' in raw_data[:100]:
+        return "HWP3.0"
+    return None
+
+
+__all__ = [
+    'extract_text_from_stream_raw',
+    'find_zlib_streams',
+    'recover_images_from_raw',
+    'check_file_signature',
+]