xgen-doc2chunk 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

xgen_doc2chunk/core/functions/table_processor.py

@@ -0,0 +1,299 @@
+# xgen_doc2chunk/core/functions/table_processor.py
+"""
+Table Processor - Common Table Processing Module
+
+Provides common table processing utilities for formatting tables.
+This module handles HTML, Markdown, and Text conversion of TableData.
+
+================================================================================
+TABLE PROCESSOR ARCHITECTURE
+================================================================================
+
+Main Entry Point:
+    format_table(table: TableData) -> str
+
+Internal Processing Functions (called from format_table):
+    - format_table_as_html()     : HTML conversion (rowspan/colspan support)
+    - format_table_as_markdown() : Markdown conversion (simple table)
+    - format_table_as_text()     : Text conversion (plain text)
+
+Common Utility:
+    - _clean_cell_content()      : Cell content cleaning (whitespace handling)
+
+================================================================================
+PROCESSING FLOW
+================================================================================
+
+format_table(table) -> Main Entry Point
+|
++-- config.output_format == HTML?
+|   YES -> format_table_as_html(table)
+|       +-- colgroup generation (column width)
+|       +-- row/cell iteration
+|       |   +-- rowspan/colspan handling
+|       |   +-- nested_table recursive handling
+|       |   +-- _clean_cell_content()
+|       +-- HTML string return
+|
++-- config.output_format == MARKDOWN?
+|   YES -> format_table_as_markdown(table)
+|       +-- row/cell iteration
+|       |   +-- _clean_cell_content()
+|       +-- header separator addition
+|       +-- Markdown string return
+|
++-- config.output_format == TEXT?
+    YES -> format_table_as_text(table)
+        +-- row/cell iteration
+        |   +-- _clean_cell_content()
+        +-- Text string return
+
+================================================================================
+OUTPUT FORMAT COMPARISON
+================================================================================
+
+| Format   | Use Case                    | Merge Support | Structure  |
+|----------|-----------------------------|--------------:|:-----------|
+| HTML     | Web rendering, full convert | Full support  | Complex    |
+| Markdown | GitHub, docs, simple render | Not supported | Simplified |
+| Text     | Search index, logs, debug   | Not supported | Minimal    |
+
+================================================================================
+"""
+import logging
+import re
+from enum import Enum
+from dataclasses import dataclass
+from typing import Optional
+
+from xgen_doc2chunk.core.functions.table_extractor import TableData, TableCell
+
+logger = logging.getLogger("document-processor")
+
+
+class TableOutputFormat(Enum):
+    """Table output format options."""
+    HTML = "html"
+    MARKDOWN = "markdown"
+    TEXT = "text"
+
+
+@dataclass
+class TableProcessorConfig:
+    """Configuration for table processing."""
+    output_format: TableOutputFormat = TableOutputFormat.HTML
+    clean_whitespace: bool = True
+    preserve_merged_cells: bool = True
+
+
+class TableProcessor:
+    """
+    Main table processing class.
+    
+    ============================================================================
+    CLASS STRUCTURE
+    ============================================================================
+    
+    Public Methods:
+        format_table()            -> Main Entry Point (routes by config.output_format)
+        format_table_as_html()    -> HTML conversion (internal: called from format_table)
+        format_table_as_markdown()-> Markdown conversion (internal: called from format_table)
+        format_table_as_text()    -> Text conversion (internal: called from format_table)
+    
+    Private Methods:
+        _clean_cell_content()     -> Common utility (cell content cleaning)
+    
+    ============================================================================
+    """
+    
+    def __init__(self, config: Optional[TableProcessorConfig] = None):
+        self.config = config or TableProcessorConfig()
+        self.logger = logging.getLogger("document-processor")
+    
+    # ==========================================================================
+    # format_table() - Main Entry Point
+    # ==========================================================================
+    #
+    # +------------------------------------------------------------------------+
+    # | format_table(table)                                                    |
+    # |                                                                        |
+    # |   Check config.output_format                                           |
+    # |       |                                                                |
+    # |       +-- HTML -------> format_table_as_html()                         |
+    # |       |                     +-- colgroup generation (col width)        |
+    # |       |                     +-- row/cell iteration                     |
+    # |       |                     |   +-- rowspan/colspan                    |
+    # |       |                     |   +-- nested_table (recursive)           |
+    # |       |                     |   +-- _clean_cell_content()              |
+    # |       |                     +-- HTML return                            |
+    # |       |                                                                |
+    # |       +-- MARKDOWN ---> format_table_as_markdown()                     |
+    # |       |                     +-- row/cell iteration                     |
+    # |       |                     |   +-- _clean_cell_content()              |
+    # |       |                     +-- header separator                       |
+    # |       |                     +-- Markdown return                        |
+    # |       |                                                                |
+    # |       +-- TEXT -------> format_table_as_text()                         |
+    # |                             +-- row/cell iteration                     |
+    # |                             |   +-- _clean_cell_content()              |
+    # |                             +-- Text return                            |
+    # +------------------------------------------------------------------------+
+    #
+    # ==========================================================================
+    
+    def format_table(self, table: TableData) -> str:
+        """
+        Main entry point for table formatting.
+        
+        Routes to appropriate format handler based on config.output_format.
+        
+        Args:
+            table: TableData from table extractor
+            
+        Returns:
+            Formatted string (HTML/Markdown/Text)
+        """
+        if self.config.output_format == TableOutputFormat.HTML:
+            return self.format_table_as_html(table)
+        elif self.config.output_format == TableOutputFormat.MARKDOWN:
+            return self.format_table_as_markdown(table)
+        else:
+            return self.format_table_as_text(table)
+    
+    # ==========================================================================
+    # format_table_as_html() - HTML conversion (called from format_table)
+    # ==========================================================================
+    
+    def format_table_as_html(self, table: TableData) -> str:
+        """
+        Convert TableData to HTML string.
+        
+        Called from format_table() when output_format == HTML.
+        
+        Features:
+        - colgroup for column widths
+        - rowspan/colspan for merged cells
+        - nested_table support (recursive)
+        """
+        if not table.rows:
+            return ""
+        
+        html_parts = ["<table>"]
+        
+        if table.col_widths_percent:
+            html_parts.append("  <colgroup>")
+            for width_pct in table.col_widths_percent:
+                html_parts.append(f'    <col style="width: {width_pct:.1f}%">')
+            html_parts.append("  </colgroup>")
+        
+        for row_idx, row in enumerate(table.rows):
+            html_parts.append("  <tr>")
+            
+            for cell in row:
+                tag = "th" if cell.is_header else "td"
+                attrs = []
+                if self.config.preserve_merged_cells:
+                    if cell.row_span > 1:
+                        attrs.append(f'rowspan="{cell.row_span}"')
+                    if cell.col_span > 1:
+                        attrs.append(f'colspan="{cell.col_span}"')
+                
+                attr_str = " " + " ".join(attrs) if attrs else ""
+                
+                if cell.nested_table:
+                    nested_html = self.format_table_as_html(cell.nested_table)
+                    html_parts.append(f"    <{tag}{attr_str}>{nested_html}</{tag}>")
+                else:
+                    content = self._clean_cell_content(cell.content)
+                    html_parts.append(f"    <{tag}{attr_str}>{content}</{tag}>")
+            
+            html_parts.append("  </tr>")
+        
+        html_parts.append("</table>")
+        return "\n".join(html_parts)
+    
+    # ==========================================================================
+    # format_table_as_markdown() - Markdown conversion (called from format_table)
+    # ==========================================================================
+    
+    def format_table_as_markdown(self, table: TableData) -> str:
+        """
+        Convert TableData to Markdown string.
+        
+        Called from format_table() when output_format == MARKDOWN.
+        
+        Note: Markdown does NOT support rowspan/colspan.
+        """
+        if not table.rows:
+            return ""
+        
+        lines = []
+        for row_idx, row in enumerate(table.rows):
+            cells = [self._clean_cell_content(cell.content) for cell in row]
+            line = "| " + " | ".join(cells) + " |"
+            lines.append(line)
+            
+            if row_idx == 0 and table.has_header:
+                separator = "| " + " | ".join(["---"] * len(row)) + " |"
+                lines.append(separator)
+        
+        return "\n".join(lines)
+    
+    # ==========================================================================
+    # format_table_as_text() - Text conversion (called from format_table)
+    # ==========================================================================
+    
+    def format_table_as_text(self, table: TableData) -> str:
+        """
+        Convert TableData to plain text string.
+        
+        Called from format_table() when output_format == TEXT.
+        
+        Note: No table structure preserved. Useful for search indexing.
+        """
+        if not table.rows:
+            return ""
+        
+        lines = []
+        for row in table.rows:
+            cells = [self._clean_cell_content(cell.content) for cell in row]
+            lines.append("\t".join(cells))
+        
+        return "\n".join(lines)
+    
+    # ==========================================================================
+    # _clean_cell_content() - Common utility (called from all format functions)
+    # ==========================================================================
+    
+    def _clean_cell_content(self, content: str) -> str:
+        """
+        Clean cell content (whitespace normalization).
+        
+        Called from all format_table_as_* methods.
+        """
+        if not content:
+            return ""
+        
+        if self.config.clean_whitespace:
+            content = re.sub(r'\s+', ' ', content)
+            content = content.strip()
+        
+        return content
+
+
+def create_table_processor(config: Optional[TableProcessorConfig] = None) -> TableProcessor:
+    """
+    Factory function to create a TableProcessor.
+    
+    Args:
+        config: Table processing configuration
+        
+    Returns:
+        Configured TableProcessor instance
+    """
+    return TableProcessor(config)
+
+
+# Default configuration
+DEFAULT_PROCESSOR_CONFIG = TableProcessorConfig()
+

xgen_doc2chunk/core/functions/utils.py

@@ -0,0 +1,159 @@
+# your_package/document_processor/utils.py
+"""
+Common utility module for document processing
+"""
+import io
+import os
+import hashlib
+import tempfile
+import logging
+import re
+import bisect
+from typing import Any, Dict, List, Optional, Set
+
+from PIL import Image
+
+def sanitize_text_for_json(text: Optional[str]) -> str:
+    """
+    Sanitizes text to be safely encodable in a UTF-8 JSON response.
+
+    Removes or replaces the following characters:
+    - Invalid surrogate pairs (U+D800-U+DFFF): removes isolated high/low surrogates
+    - Private Use Area characters (U+E000-U+F8FF, U+F0000 and above): removed
+    - Non-character code points (U+FFFE, U+FFFF): removed
+    - Problematic control characters (except tab, newline, carriage return)
+
+    Args:
+        text: Input text that may contain invalid characters
+
+    Returns:
+        Sanitized text safe for JSON encoding
+    """
+    if not text:
+        return text if text is not None else ""
+
+    result = []
+    i = 0
+    text_len = len(text)
+
+    while i < text_len:
+        char = text[i]
+        code = ord(char)
+
+        # Check for surrogate pairs (\uD800-\uDFFF)
+        if 0xD800 <= code <= 0xDFFF:
+            # High surrogate (\uD800-\uDBFF)
+            if 0xD800 <= code <= 0xDBFF:
+                # Check if followed by a valid low surrogate
+                if i + 1 < text_len:
+                    next_code = ord(text[i + 1])
+                    if 0xDC00 <= next_code <= 0xDFFF:
+                        # Valid surrogate pair, calculate actual code point
+                        full_code = 0x10000 + ((code - 0xD800) << 10) + (next_code - 0xDC00)
+                        # Supplementary Private Use Area-A: U+F0000 ~ U+FFFFF
+                        # Supplementary Private Use Area-B: U+100000 ~ U+10FFFF
+                        if full_code >= 0xF0000:
+                            # Skip Private Use Supplementary characters
+                            i += 2
+                            continue
+                        else:
+                            # Valid supplementary character, keep it
+                            result.append(char)
+                            result.append(text[i + 1])
+                            i += 2
+                            continue
+                # Invalid isolated high surrogate, skip it
+                i += 1
+                continue
+            else:
+                # Low surrogate without high surrogate, skip it
+                i += 1
+                continue
+
+        # Check Basic Private Use Area (U+E000 ~ U+F8FF)
+        if 0xE000 <= code <= 0xF8FF:
+            # Skip Private Use characters
+            i += 1
+            continue
+
+        # Check for problematic control characters
+        # Keep: \t (9), \n (10), \r (13), space (32) and above
+        # Remove: \x00-\x08, \x0B, \x0C, \x0E-\x1F (except those above)
+        if code < 32 and code not in (9, 10, 13):
+            # Skip problematic control characters
+            i += 1
+            continue
+
+        # Check for non-characters (U+FFFE, U+FFFF)
+        if code in (0xFFFE, 0xFFFF):
+            i += 1
+            continue
+
+        # Valid character, keep it
+        result.append(char)
+        i += 1
+
+    return ''.join(result)
+
+
+def clean_text(text: Optional[str]) -> str:
+   if not text:
+       return ""
+   text = re.sub(r'\n\s*\n\s*\n+', '\n\n', text)
+   return text.strip()
+
+def clean_code_text(text: str) -> str:
+    if not text:
+        return ""
+    text = text.rstrip().replace('\t', '    ')
+    return text
+
+def is_text_quality_sufficient(text: Optional[str], min_chars: int = 500, min_word_ratio: float = 0.6) -> bool:
+    try:
+        if not text or len(text) < min_chars:
+            return False
+        word_chars = re.findall(r"[\w\u1100-\u11FF\u3130-\u318F\uAC00-\uD7AF]", text)
+        ratio = len(word_chars) / max(1, len(text))
+        return ratio >= min_word_ratio
+    except Exception:
+        return False
+
+def find_chunk_position(chunk: str, full_text: str, start_pos: int = 0) -> int:
+    try:
+        pos = full_text.find(chunk, start_pos)
+        if pos != -1:
+            return pos
+        lines = chunk.strip().split('\n')
+        if lines and len(lines[0]) >= 10:
+            first_line = lines[0].strip()
+            pos = full_text.find(first_line, start_pos)
+            if pos != -1:
+                chunk_start = full_text.find(chunk[:50] if len(chunk) > 50 else chunk, pos)
+                return chunk_start if chunk_start != -1 else pos
+        if len(chunk.strip()) >= 10:
+            start = chunk.strip()[:50]
+            pos = full_text.find(start, start_pos)
+            if pos != -1:
+                return pos
+        return -1
+    except Exception:
+        return -1
+
+def build_line_starts(text: str) -> List[int]:
+    try:
+        starts = [0]
+        for i, ch in enumerate(text):
+            if ch == '\n' and i + 1 < len(text):
+                starts.append(i + 1)
+        return starts
+    except Exception:
+        return [0]
+
+def pos_to_line(pos: int, line_starts: List[int]) -> int:
+    try:
+        if pos < 0:
+            return 1
+        idx = bisect.bisect_right(line_starts, pos) - 1
+        return max(1, idx + 1)
+    except Exception:
+        return 1

xgen_doc2chunk/core/processor/__init__.py

@@ -0,0 +1,96 @@
+# xgen_doc2chunk/core/processor/__init__.py
+"""
+Processor - Document Type-specific Handler Module
+
+Provides handlers for processing individual document formats.
+
+Handler List:
+- pdf_handler: PDF document processing (adaptive complexity-based)
+- docx_handler: DOCX document processing
+- doc_handler: DOC document processing (OLE, HTML, misnamed DOCX)
+- rtf_handler: RTF document processing
+- ppt_handler: PPT/PPTX document processing
+- excel_handler: Excel (XLSX/XLS) document processing
+- hwp_processor: HWP document processing
+- hwpx_processor: HWPX document processing
+- csv_handler: CSV file processing
+- text_handler: Text file processing
+- html_reprocessor: HTML reprocessing
+
+Helper Modules (subdirectories):
+- csv_helper/: CSV processing helper
+- docx_helper/: DOCX processing helper
+- doc_helpers/: DOC processing helper
+- rtf_helper/: RTF processing helper
+- excel_helper/: Excel processing helper
+- hwp_helper/: HWP processing helper
+- hwpx_helper/: HWPX processing helper
+- pdf_helpers/: PDF processing helper
+- ppt_helper/: PPT processing helper
+
+Usage Example:
+    from xgen_doc2chunk.core.processor import PDFHandler
+    from xgen_doc2chunk.core.processor import DOCXHandler
+    from xgen_doc2chunk.core.processor import RTFHandler
+    from xgen_doc2chunk.core.processor.pdf_helpers import extract_pdf_metadata
+"""
+
+# === PDF Handler ===
+from xgen_doc2chunk.core.processor.pdf_handler import PDFHandler
+
+# === Document Handlers ===
+from xgen_doc2chunk.core.processor.docx_handler import DOCXHandler
+from xgen_doc2chunk.core.processor.doc_handler import DOCHandler
+from xgen_doc2chunk.core.processor.rtf_handler import RTFHandler
+from xgen_doc2chunk.core.processor.ppt_handler import PPTHandler
+
+# === Data Handlers ===
+from xgen_doc2chunk.core.processor.excel_handler import ExcelHandler
+from xgen_doc2chunk.core.processor.csv_handler import CSVHandler
+from xgen_doc2chunk.core.processor.text_handler import TextHandler
+
+# === HWP Handlers ===
+from xgen_doc2chunk.core.processor.hwp_handler import HWPHandler
+from xgen_doc2chunk.core.processor.hwpx_handler import HWPXHandler
+
+# === Other Processors ===
+# from xgen_doc2chunk.core.processor.html_reprocessor import ...  # HTML reprocessing
+
+# === Helper Modules (subpackages) ===
+from xgen_doc2chunk.core.processor import csv_helper
+from xgen_doc2chunk.core.processor import doc_helpers
+from xgen_doc2chunk.core.processor import docx_helper
+from xgen_doc2chunk.core.processor import excel_helper
+from xgen_doc2chunk.core.processor import hwp_helper
+from xgen_doc2chunk.core.processor import hwpx_helper
+from xgen_doc2chunk.core.processor import pdf_helpers
+from xgen_doc2chunk.core.processor import ppt_helper
+from xgen_doc2chunk.core.processor import rtf_helper
+
+__all__ = [
+    # PDF Handler
+    "PDFHandler",
+    # Document Handlers
+    "DOCXHandler",
+    "DOCHandler",
+    "RTFHandler",
+    "PPTHandler",
+    # Data Handlers
+    "ExcelHandler",
+    "CSVHandler",
+    "TextHandler",
+    # HWP Handlers
+    "HWPHandler",
+    "HWPXHandler",
+    # Helper subpackages
+    "csv_helper",
+    "doc_helpers",
+    "docx_helper",
+    "excel_helper",
+    "hwp_helper",
+    "hwpx_helper",
+    "pdf_helpers",
+    "ppt_helper",
+    "rtf_helper",
+]
+