xgen-doc2chunk 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

xgen_doc2chunk/core/processor/hwp_helper/hwp_table.py

@@ -0,0 +1,205 @@
+# service/document_processor/processor/hwp_helper/hwp_table.py
+"""
+HWP 테이블 파싱 유틸리티
+
+HWP 5.0 OLE 파일에서 테이블을 파싱하여 HTML로 변환합니다.
+- parse_table: 테이블 컨트롤을 파싱하여 HTML 또는 리스트로 변환
+- build_table_grid: 셀 정보를 그리드로 구성
+- render_table_html: 그리드를 HTML 테이블로 렌더링
+"""
+import struct
+import logging
+from typing import Dict, Callable, Optional, Any, Set
+
+import olefile
+
+from xgen_doc2chunk.core.processor.hwp_helper.hwp_constants import (
+    HWPTAG_TABLE,
+    HWPTAG_LIST_HEADER,
+)
+from xgen_doc2chunk.core.processor.hwp_helper.hwp_record import HwpRecord
+
+logger = logging.getLogger("document-processor")
+
+
+def parse_table(
+    ctrl_header: HwpRecord,
+    traverse_callback: Callable[[HwpRecord, Any, Dict, Set], str],
+    ole: olefile.OleFileIO = None,
+    bin_data_map: Dict = None,
+    processed_images: Optional[Set[str]] = None
+) -> str:
+    """
+    HWP 테이블 컨트롤을 파싱합니다.
+
+    테이블 구조를 분석하여:
+    - 1×1 테이블: 셀 내용만 텍스트로 반환 (컨테이너 테이블)
+    - 단일 컬럼 테이블 (1열, 다중 행): 셀 내용을 줄바꿈으로 구분하여 반환
+    - 다중 컬럼 테이블 (2+ 열): HTML 테이블로 변환
+
+    Args:
+        ctrl_header: 테이블을 포함하는 CTRL_HEADER 레코드
+        traverse_callback: 셀 내용을 추출하기 위한 트리 순회 콜백 함수
+        ole: OLE 파일 객체
+        bin_data_map: BinData 매핑 정보
+        processed_images: 처리된 이미지 경로 집합
+
+    Returns:
+        HTML 테이블, 텍스트 리스트, 또는 단순 텍스트
+    """
+    try:
+        table_rec = next((c for c in ctrl_header.children if c.tag_id == HWPTAG_TABLE), None)
+        if not table_rec:
+            return ""
+
+        if len(table_rec.payload) < 8:
+            return ""
+
+        row_cnt = struct.unpack('<H', table_rec.payload[4:6])[0]
+        col_cnt = struct.unpack('<H', table_rec.payload[6:8])[0]
+
+        # 셀 그리드 구성
+        grid = build_table_grid(
+            ctrl_header,
+            traverse_callback,
+            ole,
+            bin_data_map,
+            processed_images
+        )
+
+        # 1×1 테이블 -> 셀 내용만 반환 (컨테이너 테이블)
+        if row_cnt == 1 and col_cnt == 1:
+            if (0, 0) in grid:
+                return grid[(0, 0)]['text']
+            return ""
+
+        # 단일 컬럼 테이블 (1열, 다중 행) -> 셀 내용을 줄바꿈으로 구분
+        if col_cnt == 1:
+            text_items = []
+            for r in range(row_cnt):
+                if (r, 0) in grid:
+                    cell_text = grid[(r, 0)]['text']
+                    if cell_text:
+                        text_items.append(cell_text)
+            if text_items:
+                return "\n\n".join(text_items)
+            return ""
+
+        # HTML 테이블 생성 (2+ 컬럼)
+        return render_table_html(grid, row_cnt, col_cnt)
+
+    except Exception as e:
+        logger.warning(f"Failed to parse HWP table: {e}")
+        return "[Table Extraction Failed]"
+
+
+def build_table_grid(
+    ctrl_header: HwpRecord,
+    traverse_callback: Callable[[HwpRecord, Any, Dict, Set], str],
+    ole: olefile.OleFileIO = None,
+    bin_data_map: Dict = None,
+    processed_images: Optional[Set[str]] = None
+) -> Dict:
+    """
+    테이블 셀 정보를 그리드로 구성합니다.
+
+    Args:
+        ctrl_header: 테이블을 포함하는 CTRL_HEADER 레코드
+        traverse_callback: 셀 내용을 추출하기 위한 트리 순회 콜백 함수
+        ole: OLE 파일 객체
+        bin_data_map: BinData 매핑 정보
+        processed_images: 처리된 이미지 경로 집합
+
+    Returns:
+        (row_idx, col_idx) -> {'text', 'rowspan', 'colspan'} 딕셔너리
+    """
+    grid = {}
+
+    cells = [c for c in ctrl_header.children if c.tag_id == HWPTAG_LIST_HEADER]
+
+    for cell in cells:
+        if len(cell.payload) < 16:
+            continue
+
+        para_count = struct.unpack('<H', cell.payload[0:2])[0]
+        col_idx = struct.unpack('<H', cell.payload[8:10])[0]
+        row_idx = struct.unpack('<H', cell.payload[10:12])[0]
+        col_span = struct.unpack('<H', cell.payload[12:14])[0]
+        row_span = struct.unpack('<H', cell.payload[14:16])[0]
+
+        cell_text_parts = []
+
+        if cell.children:
+            for child in cell.children:
+                t = traverse_callback(child, ole, bin_data_map, processed_images)
+                cell_text_parts.append(t)
+        else:
+            siblings = list(cell.get_next_siblings(para_count))
+            for sibling in siblings:
+                t = traverse_callback(sibling, ole, bin_data_map, processed_images)
+                cell_text_parts.append(t)
+
+        cell_content = "".join(cell_text_parts).strip()
+
+        grid[(row_idx, col_idx)] = {
+            'text': cell_content,
+            'rowspan': row_span,
+            'colspan': col_span
+        }
+
+    return grid
+
+
+def render_table_html(grid: Dict, row_cnt: int, col_cnt: int) -> str:
+    """
+    그리드를 HTML 테이블로 렌더링합니다.
+
+    Args:
+        grid: (row_idx, col_idx) -> {'text', 'rowspan', 'colspan'} 딕셔너리
+        row_cnt: 테이블 행 수
+        col_cnt: 테이블 열 수
+
+    Returns:
+        HTML 테이블 문자열
+    """
+    html_parts = ["<table border='1'>"]
+    skip_map = set()
+
+    for r in range(row_cnt):
+        html_parts.append("<tr>")
+        for c in range(col_cnt):
+            if (r, c) in skip_map:
+                continue
+
+            if (r, c) in grid:
+                cell = grid[(r, c)]
+                rowspan = cell['rowspan']
+                colspan = cell['colspan']
+                text = cell['text']
+
+                attr = ""
+                if rowspan > 1:
+                    attr += f" rowspan='{rowspan}'"
+                if colspan > 1:
+                    attr += f" colspan='{colspan}'"
+
+                html_parts.append(f"<td{attr}>{text}</td>")
+
+                for rs in range(rowspan):
+                    for cs in range(colspan):
+                        if rs == 0 and cs == 0:
+                            continue
+                        skip_map.add((r + rs, c + cs))
+            else:
+                html_parts.append("<td></td>")
+        html_parts.append("</tr>")
+
+    html_parts.append("</table>")
+    return "\n".join(html_parts)
+
+
+__all__ = [
+    'parse_table',
+    'build_table_grid',
+    'render_table_html',
+]

xgen_doc2chunk/core/processor/hwpx_handler.py

@@ -0,0 +1,191 @@
+# xgen_doc2chunk/core/processor/hwpx_processor.py
+"""
+HWPX Handler - HWPX (ZIP/XML based) Document Processor
+
+Class-based handler for HWPX files inheriting from BaseHandler.
+"""
+import io
+import logging
+from typing import Dict, Any, Set, TYPE_CHECKING
+
+from xgen_doc2chunk.core.processor.base_handler import BaseHandler
+from xgen_doc2chunk.core.functions.chart_extractor import BaseChartExtractor
+from xgen_doc2chunk.core.processor.hwpx_helper import (
+    parse_bin_item_map,
+    parse_hwpx_section,
+)
+from xgen_doc2chunk.core.processor.hwpx_helper.hwpx_chart_extractor import HWPXChartExtractor
+from xgen_doc2chunk.core.processor.hwpx_helper.hwpx_metadata import HWPXMetadataExtractor
+from xgen_doc2chunk.core.processor.hwpx_helper.hwpx_image_processor import HWPXImageProcessor
+
+if TYPE_CHECKING:
+    from xgen_doc2chunk.core.document_processor import CurrentFile
+    from xgen_doc2chunk.core.functions.chart_extractor import ChartData
+
+logger = logging.getLogger("document-processor")
+
+
+class HWPXHandler(BaseHandler):
+    """HWPX (ZIP/XML based Korean document) Processing Handler Class"""
+
+    def _create_file_converter(self):
+        """Create HWPX-specific file converter."""
+        from xgen_doc2chunk.core.processor.hwpx_helper.hwpx_file_converter import HWPXFileConverter
+        return HWPXFileConverter()
+
+    def _create_preprocessor(self):
+        """Create HWPX-specific preprocessor."""
+        from xgen_doc2chunk.core.processor.hwpx_helper.hwpx_preprocessor import HWPXPreprocessor
+        return HWPXPreprocessor()
+
+    def _create_chart_extractor(self) -> BaseChartExtractor:
+        """Create HWPX-specific chart extractor."""
+        return HWPXChartExtractor(self._chart_processor)
+
+    def _create_metadata_extractor(self):
+        """Create HWPX-specific metadata extractor."""
+        return HWPXMetadataExtractor()
+
+    def _create_format_image_processor(self):
+        """Create HWPX-specific image processor."""
+        return HWPXImageProcessor(
+            directory_path=self._image_processor.config.directory_path,
+            tag_prefix=self._image_processor.config.tag_prefix,
+            tag_suffix=self._image_processor.config.tag_suffix,
+            storage_backend=self._image_processor.storage_backend,
+        )
+
+    def extract_text(
+        self,
+        current_file: "CurrentFile",
+        extract_metadata: bool = True,
+        **kwargs
+    ) -> str:
+        """
+        Extract text from HWPX file.
+
+        Args:
+            current_file: CurrentFile dict containing file info and binary data
+            extract_metadata: Whether to extract metadata
+            **kwargs: Additional options
+
+        Returns:
+            Extracted text
+        """
+        file_path = current_file.get("file_path", "unknown")
+        file_data = current_file.get("file_data", b"")
+        text_content = []
+
+        # Check if it's a valid ZIP file using file_converter.validate()
+        if not self.file_converter.validate(file_data):
+            self.logger.error("Not a valid Zip file: %s", file_path)
+            return ""
+
+        try:
+            # Get file stream
+            file_stream = self.get_file_stream(current_file)
+
+            # Pre-extract all charts using ChartExtractor with refs
+            # This creates a mapping from chartIDRef -> ChartData
+            chart_map = self.chart_extractor.extract_all_with_refs(file_stream)
+            processed_chart_refs = set()
+
+            def chart_callback(chart_id_ref: str) -> str:
+                """Callback to get chart content by chartIDRef."""
+                # chartIDRef is like "Chart/chart1.xml"
+                if chart_id_ref in processed_chart_refs:
+                    return ""  # Already processed
+                
+                chart_data = chart_map.get(chart_id_ref)
+                if chart_data:
+                    processed_chart_refs.add(chart_id_ref)
+                    return self._format_chart_data(chart_data)
+                return ""
+
+            # Step 1: Convert binary to ZipFile using file_converter
+            zf = self.file_converter.convert(file_data, file_stream)
+
+            # Step 2: Preprocess - clean_content is the TRUE SOURCE
+            preprocessed = self.preprocess(zf)
+            zf = preprocessed.clean_content  # TRUE SOURCE
+
+            try:
+                if extract_metadata:
+                    metadata_text = self.extract_and_format_metadata(zf)
+                    if metadata_text:
+                        text_content.append(metadata_text)
+                        text_content.append("")
+
+                bin_item_map = parse_bin_item_map(zf)
+
+                section_files = [
+                    f for f in zf.namelist()
+                    if f.startswith("Contents/section") and f.endswith(".xml")
+                ]
+                section_files.sort(key=lambda x: int(x.replace("Contents/section", "").replace(".xml", "")))
+
+                processed_images: Set[str] = set()
+
+                for sec_file in section_files:
+                    with zf.open(sec_file) as f:
+                        xml_content = f.read()
+                        section_text = parse_hwpx_section(
+                            xml_content, 
+                            zf, 
+                            bin_item_map, 
+                            processed_images, 
+                            image_processor=self.format_image_processor,
+                            chart_callback=chart_callback
+                        )
+                        text_content.append(section_text)
+
+                # Use format_image_processor directly
+                image_processor = self.format_image_processor
+                if hasattr(image_processor, 'get_remaining_images'):
+                    remaining_images = image_processor.get_remaining_images(zf, processed_images)
+                    if remaining_images and hasattr(image_processor, 'process_images'):
+                        image_text = image_processor.process_images(zf, remaining_images)
+                        if image_text:
+                            text_content.append("\n\n=== Extracted Images (Not Inline) ===\n")
+                            text_content.append(image_text)
+
+            finally:
+                # Close ZipFile using file_converter
+                self.file_converter.close(zf)
+
+        except Exception as e:  # noqa: BLE001
+            self.logger.error("Error processing HWPX file: %s", e)
+            return f"Error processing HWPX file: {str(e)}"
+
+        return "\n".join(text_content)
+
+    def _is_valid_zip(self, file_stream: io.BytesIO) -> bool:
+        """Check if stream is a valid ZIP file."""
+        try:
+            file_stream.seek(0)
+            header = file_stream.read(4)
+            file_stream.seek(0)
+            return header == b'PK\x03\x04'
+        except Exception:  # noqa: BLE001
+            return False
+
+    def _format_chart_data(self, chart_data: "ChartData") -> str:
+        """Format ChartData using ChartProcessor."""
+        from xgen_doc2chunk.core.functions.chart_extractor import ChartData
+
+        if not isinstance(chart_data, ChartData):
+            return ""
+
+        if chart_data.has_data():
+            return self.chart_processor.format_chart_data(
+                chart_type=chart_data.chart_type,
+                title=chart_data.title,
+                categories=chart_data.categories,
+                series=chart_data.series
+            )
+        else:
+            return self.chart_processor.format_chart_fallback(
+                chart_type=chart_data.chart_type,
+                title=chart_data.title
+            )
+

xgen_doc2chunk/core/processor/hwpx_helper/__init__.py

@@ -0,0 +1,85 @@
+# hwpx_helper/__init__.py
+"""
+HWPX Helper 모듈
+
+hwpx_processor.py에서 사용하는 기능적 구성요소들을 모듈화하여 제공합니다.
+
+모듈 구성:
+- hwpx_constants: 상수 및 네임스페이스 정의
+- hwpx_metadata: 메타데이터 추출 및 BinItem 매핑
+- hwpx_table_extractor: 테이블 추출 (HWPXTableExtractor) - BaseTableExtractor 구현
+- hwpx_table_processor: 테이블 포맷팅 (HWPXTableProcessor) - TableProcessor 확장
+- hwpx_section: 섹션 XML 파싱
+- hwpx_image_processor: 이미지 처리 및 업로드
+- hwpx_chart_extractor: 차트 추출 (ChartExtractor)
+"""
+
+# Constants
+from xgen_doc2chunk.core.processor.hwpx_helper.hwpx_constants import (
+    HWPX_NAMESPACES,
+    OPF_NAMESPACES,
+    SUPPORTED_IMAGE_EXTENSIONS,
+    SKIP_IMAGE_EXTENSIONS,
+    HEADER_FILE_PATHS,
+    HPF_PATH,
+)
+
+# Metadata
+from xgen_doc2chunk.core.processor.hwpx_helper.hwpx_metadata import (
+    HWPXMetadataExtractor,
+    parse_bin_item_map,
+)
+
+# Table Extractor (NEW - BaseTableExtractor implementation)
+from xgen_doc2chunk.core.processor.hwpx_helper.hwpx_table_extractor import (
+    HWPXTableExtractor,
+    create_hwpx_table_extractor,
+)
+
+# Table Processor (NEW - TableProcessor extension)
+from xgen_doc2chunk.core.processor.hwpx_helper.hwpx_table_processor import (
+    HWPXTableProcessor,
+    HWPXTableProcessorConfig,
+    create_hwpx_table_processor,
+)
+
+# Section
+from xgen_doc2chunk.core.processor.hwpx_helper.hwpx_section import (
+    parse_hwpx_section,
+)
+
+# Image Processor (replaces hwpx_image.py utility functions)
+from xgen_doc2chunk.core.processor.hwpx_helper.hwpx_image_processor import (
+    HWPXImageProcessor,
+)
+
+# Chart Extractor
+from xgen_doc2chunk.core.processor.hwpx_helper.hwpx_chart_extractor import (
+    HWPXChartExtractor,
+)
+
+__all__ = [
+    # Constants
+    "HWPX_NAMESPACES",
+    "OPF_NAMESPACES",
+    "SUPPORTED_IMAGE_EXTENSIONS",
+    "SKIP_IMAGE_EXTENSIONS",
+    "HEADER_FILE_PATHS",
+    "HPF_PATH",
+    # Metadata
+    "HWPXMetadataExtractor",
+    "parse_bin_item_map",
+    # Table Extractor (NEW)
+    "HWPXTableExtractor",
+    "create_hwpx_table_extractor",
+    # Table Processor (NEW)
+    "HWPXTableProcessor",
+    "HWPXTableProcessorConfig",
+    "create_hwpx_table_processor",
+    # Section
+    "parse_hwpx_section",
+    # Image Processor
+    "HWPXImageProcessor",
+    # Chart Extractor
+    "HWPXChartExtractor",
+]