xgen-doc2chunk 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

xgen_doc2chunk/core/processor/csv_helper/__init__.py

@@ -0,0 +1,89 @@
+# csv_helper/__init__.py
+"""
+CSV Helper 모듈
+
+csv_handler.py에서 사용하는 기능적 구성요소들을 모듈화하여 제공합니다.
+
+모듈 구성:
+- csv_constants: 상수 및 데이터 클래스 정의
+- csv_metadata: 메타데이터 추출 및 포맷팅
+- csv_encoding: 인코딩 감지 및 파일 읽기
+- csv_parser: CSV 파싱, 구분자/헤더 감지
+- csv_table: 테이블 변환 (Markdown/HTML)
+"""
+
+# Constants
+from xgen_doc2chunk.core.processor.csv_helper.csv_constants import (
+    ENCODING_CANDIDATES,
+    DELIMITER_CANDIDATES,
+    DELIMITER_NAMES,
+    MAX_ROWS,
+    MAX_COLS,
+    CSVMetadata,
+)
+
+# Metadata
+from xgen_doc2chunk.core.processor.csv_helper.csv_metadata import (
+    CSVMetadataExtractor,
+    CSVSourceInfo,
+)
+
+# Image Processor
+from xgen_doc2chunk.core.processor.csv_helper.csv_image_processor import (
+    CSVImageProcessor,
+)
+
+# Encoding
+from xgen_doc2chunk.core.processor.csv_helper.csv_encoding import (
+    detect_bom,
+    read_file_with_encoding,
+)
+
+# Parser
+from xgen_doc2chunk.core.processor.csv_helper.csv_parser import (
+    detect_delimiter,
+    parse_csv_content,
+    parse_csv_simple,
+    detect_header,
+    is_numeric,
+)
+
+# Table
+from xgen_doc2chunk.core.processor.csv_helper.csv_table import (
+    has_merged_cells,
+    analyze_merge_info,
+    convert_rows_to_table,
+    convert_rows_to_markdown,
+    convert_rows_to_html,
+)
+
+__all__ = [
+    # Constants
+    "ENCODING_CANDIDATES",
+    "DELIMITER_CANDIDATES",
+    "DELIMITER_NAMES",
+    "MAX_ROWS",
+    "MAX_COLS",
+    "CSVMetadata",
+    # Metadata
+    "CSVMetadataExtractor",
+    "CSVSourceInfo",
+    # Image Processor
+    "CSVImageProcessor",
+    # Encoding
+    # Encoding
+    "detect_bom",
+    "read_file_with_encoding",
+    # Parser
+    "detect_delimiter",
+    "parse_csv_content",
+    "parse_csv_simple",
+    "detect_header",
+    "is_numeric",
+    # Table
+    "has_merged_cells",
+    "analyze_merge_info",
+    "convert_rows_to_table",
+    "convert_rows_to_markdown",
+    "convert_rows_to_html",
+]

xgen_doc2chunk/core/processor/csv_helper/csv_constants.py

@@ -0,0 +1,63 @@
+# csv_helper/csv_constants.py
+"""
+CSV Handler 상수 및 타입 정의
+
+CSV/TSV 파일 처리에 필요한 상수, 설정값, 데이터 클래스를 정의합니다.
+"""
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Optional
+
+
+# === 인코딩 관련 상수 ===
+
+# 시도할 인코딩 목록 (우선순위 순)
+ENCODING_CANDIDATES = [
+    "utf-8",
+    "utf-8-sig",  # BOM 포함 UTF-8
+    "cp949",      # 한국어 Windows
+    "euc-kr",     # 한국어 레거시
+    "utf-16",
+    "utf-16-le",
+    "utf-16-be",
+    "latin-1",    # 폴백 (모든 바이트 허용)
+    "iso-8859-1",
+]
+
+
+# === 구분자 관련 상수 ===
+
+# CSV delimiter candidates
+DELIMITER_CANDIDATES = [',', '\t', ';', '|']
+
+# Delimiter name mapping (Korean for output display)
+DELIMITER_NAMES = {
+    ',': '쉼표 (,)',
+    '\t': '탭 (\\t)',
+    ';': '세미콜론 (;)',
+    '|': '파이프 (|)',
+}
+
+
+# === Processing limit constants ===
+
+# Maximum rows to process (memory protection)
+MAX_ROWS = 100000
+
+# Maximum columns
+MAX_COLS = 1000
+
+
+# === 데이터 클래스 ===
+
+@dataclass
+class CSVMetadata:
+    """CSV 파일 메타데이터"""
+    encoding: str
+    delimiter: str
+    has_header: bool
+    row_count: int
+    col_count: int
+    file_size: int
+    file_name: Optional[str] = None
+    modified_time: Optional[datetime] = None

xgen_doc2chunk/core/processor/csv_helper/csv_encoding.py

@@ -0,0 +1,104 @@
+# csv_helper/csv_encoding.py
+"""
+CSV 인코딩 감지 및 파일 읽기
+
+파일의 인코딩을 자동 감지하고 올바르게 디코딩합니다.
+BOM 감지, chardet 라이브러리, 휴리스틱 방식을 사용합니다.
+"""
+import logging
+from typing import Optional, Tuple
+import chardet
+
+from xgen_doc2chunk.core.processor.csv_helper.csv_constants import ENCODING_CANDIDATES
+
+logger = logging.getLogger("document-processor")
+
+
+def detect_bom(data: bytes) -> Optional[str]:
+    """
+    BOM(Byte Order Mark)을 감지합니다.
+
+    Args:
+        data: 파일의 바이너리 데이터
+
+    Returns:
+        감지된 인코딩 또는 None
+    """
+    if data.startswith(b'\xef\xbb\xbf'):
+        return 'utf-8-sig'
+    elif data.startswith(b'\xff\xfe\x00\x00'):
+        return 'utf-32-le'
+    elif data.startswith(b'\x00\x00\xfe\xff'):
+        return 'utf-32-be'
+    elif data.startswith(b'\xff\xfe'):
+        return 'utf-16-le'
+    elif data.startswith(b'\xfe\xff'):
+        return 'utf-16-be'
+    return None
+
+
+def read_file_with_encoding(
+    file_path: str,
+    preferred_encoding: str = None
+) -> Tuple[str, str]:
+    """
+    파일을 읽고 인코딩을 자동 감지합니다.
+
+    감지 순서:
+    1. BOM 확인
+    2. 선호 인코딩 시도 (지정된 경우)
+    3. chardet 라이브러리 사용 (가능한 경우)
+    4. 인코딩 후보 목록 순차 시도
+    5. latin-1 폴백 (항상 성공)
+
+    Args:
+        file_path: 파일 경로
+        preferred_encoding: 선호 인코딩 (None이면 자동 감지)
+
+    Returns:
+        (content, detected_encoding) 튜플
+    """
+    # 바이너리로 먼저 읽기
+    with open(file_path, mode='rb') as f:
+        raw_data = f.read()
+
+    # BOM 확인
+    bom_encoding = detect_bom(raw_data)
+    if bom_encoding:
+        logger.debug(f"BOM detected: {bom_encoding}")
+        try:
+            return raw_data.decode(bom_encoding), bom_encoding
+        except UnicodeDecodeError:
+            pass
+
+    # 선호 인코딩 시도
+    if preferred_encoding:
+        try:
+            return raw_data.decode(preferred_encoding), preferred_encoding
+        except UnicodeDecodeError:
+            logger.debug(f"Preferred encoding {preferred_encoding} failed")
+
+    # chardet 사용
+    detected = chardet.detect(raw_data[:10000])  # 처음 10KB만 분석
+    if detected and detected.get('encoding'):
+        detected_enc = detected['encoding']
+        confidence = detected.get('confidence', 0)
+        logger.debug(f"chardet detected: {detected_enc} (confidence: {confidence})")
+
+        if confidence > 0.7:
+            try:
+                return raw_data.decode(detected_enc), detected_enc
+            except UnicodeDecodeError:
+                pass
+
+    # 인코딩 후보 시도
+    for enc in ENCODING_CANDIDATES:
+        try:
+            content = raw_data.decode(enc)
+            logger.debug(f"Successfully decoded with: {enc}")
+            return content, enc
+        except UnicodeDecodeError:
+            continue
+
+    # 최후의 수단: latin-1 (항상 성공)
+    return raw_data.decode('latin-1', errors='replace'), 'latin-1'

xgen_doc2chunk/core/processor/csv_helper/csv_file_converter.py

@@ -0,0 +1,78 @@
+# xgen_doc2chunk/core/processor/csv_helper/csv_file_converter.py
+"""
+CSVFileConverter - CSV file format converter
+
+Converts binary CSV data to text string with encoding detection.
+"""
+from typing import Any, Optional, BinaryIO, Tuple
+
+from xgen_doc2chunk.core.functions.file_converter import TextFileConverter
+
+
+class CSVFileConverter(TextFileConverter):
+    """
+    CSV file converter.
+    
+    Converts binary CSV data to decoded text string.
+    Extends TextFileConverter with BOM detection.
+    """
+    
+    # BOM markers
+    BOM_UTF8 = b'\xef\xbb\xbf'
+    BOM_UTF16_LE = b'\xff\xfe'
+    BOM_UTF16_BE = b'\xfe\xff'
+    
+    def __init__(self):
+        """Initialize CSVFileConverter."""
+        super().__init__(encodings=['utf-8', 'utf-8-sig', 'cp949', 'euc-kr', 'iso-8859-1', 'latin-1'])
+        self._delimiter: Optional[str] = None
+    
+    def convert(
+        self,
+        file_data: bytes,
+        file_stream: Optional[BinaryIO] = None,
+        encoding: Optional[str] = None,
+        delimiter: Optional[str] = None,
+        **kwargs
+    ) -> Tuple[str, str]:
+        """
+        Convert binary CSV data to text string.
+        
+        Args:
+            file_data: Raw binary CSV data
+            file_stream: Ignored
+            encoding: Specific encoding to use
+            delimiter: CSV delimiter (for reference)
+            **kwargs: Additional options
+            
+        Returns:
+            Tuple of (decoded text, detected encoding)
+        """
+        self._delimiter = delimiter
+        
+        # Check for BOM
+        bom_encoding = self._detect_bom(file_data)
+        if bom_encoding:
+            text = file_data.decode(bom_encoding)
+            self._detected_encoding = bom_encoding
+            return text, bom_encoding
+        
+        # Use parent's convert logic
+        text = super().convert(file_data, file_stream, encoding, **kwargs)
+        return text, self._detected_encoding or 'utf-8'
+    
+    def _detect_bom(self, file_data: bytes) -> Optional[str]:
+        """Detect encoding from BOM."""
+        if file_data.startswith(self.BOM_UTF8):
+            return 'utf-8-sig'
+        elif file_data.startswith(self.BOM_UTF16_LE):
+            return 'utf-16-le'
+        elif file_data.startswith(self.BOM_UTF16_BE):
+            return 'utf-16-be'
+        return None
+    
+    def get_format_name(self) -> str:
+        """Return format name."""
+        enc = self._detected_encoding or 'unknown'
+        return f"CSV ({enc})"
+

xgen_doc2chunk/core/processor/csv_helper/csv_image_processor.py

@@ -0,0 +1,75 @@
+# xgen_doc2chunk/core/processor/csv_helper/csv_image_processor.py
+"""
+CSV Image Processor
+
+Provides CSV-specific image processing that inherits from ImageProcessor.
+CSV files do not contain embedded images, so this is a minimal implementation.
+"""
+import logging
+from typing import Any, Optional
+
+from xgen_doc2chunk.core.functions.img_processor import ImageProcessor
+from xgen_doc2chunk.core.functions.storage_backend import BaseStorageBackend
+
+logger = logging.getLogger("xgen_doc2chunk.image_processor.csv")
+
+
+class CSVImageProcessor(ImageProcessor):
+    """
+    CSV-specific image processor.
+    
+    Inherits from ImageProcessor and provides CSV-specific processing.
+    CSV files do not contain embedded images, so this processor
+    provides a consistent interface without additional functionality.
+    
+    This class exists to maintain interface consistency across all handlers.
+    
+    Example:
+        processor = CSVImageProcessor()
+        
+        # No images in CSV, but interface is consistent
+        tag = processor.process_image(image_data)  # Falls back to base implementation
+    """
+    
+    def __init__(
+        self,
+        directory_path: str = "temp/images",
+        tag_prefix: str = "[Image:",
+        tag_suffix: str = "]",
+        storage_backend: Optional[BaseStorageBackend] = None,
+    ):
+        """
+        Initialize CSVImageProcessor.
+        
+        Args:
+            directory_path: Image save directory
+            tag_prefix: Tag prefix for image references
+            tag_suffix: Tag suffix for image references
+            storage_backend: Storage backend for saving images
+        """
+        super().__init__(
+            directory_path=directory_path,
+            tag_prefix=tag_prefix,
+            tag_suffix=tag_suffix,
+            storage_backend=storage_backend,
+        )
+    
+    def process_image(
+        self,
+        image_data: bytes,
+        **kwargs
+    ) -> Optional[str]:
+        """
+        Process and save image data.
+        
+        CSV files do not contain embedded images, so this method
+        delegates to the base implementation.
+        
+        Args:
+            image_data: Raw image binary data
+            **kwargs: Additional options
+            
+        Returns:
+            Image tag string or None if processing failed
+        """
+        return super().process_image(image_data, **kwargs)

xgen_doc2chunk/core/processor/csv_helper/csv_metadata.py

@@ -0,0 +1,168 @@
+# xgen_doc2chunk/core/processor/csv_helper/csv_metadata.py
+"""
+CSV Metadata Extraction Module
+
+Provides CSVMetadataExtractor class for extracting metadata from CSV files.
+Implements BaseMetadataExtractor interface.
+
+CSV differs from regular documents - it provides file structure information as metadata:
+- File name, file size, modification time
+- Encoding, delimiter
+- Row/column count, header information
+"""
+import logging
+import os
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+from xgen_doc2chunk.core.functions.metadata_extractor import (
+    BaseMetadataExtractor,
+    DocumentMetadata,
+    MetadataFormatter,
+)
+from xgen_doc2chunk.core.processor.csv_helper.csv_constants import DELIMITER_NAMES
+
+logger = logging.getLogger("document-processor")
+
+
+def format_file_size(size_bytes: int) -> str:
+    """
+    Convert file size to human-readable format.
+
+    Args:
+        size_bytes: File size in bytes
+
+    Returns:
+        Formatted file size string (e.g., "1.5 MB")
+    """
+    if size_bytes < 1024:
+        return f"{size_bytes} B"
+    elif size_bytes < 1024 * 1024:
+        return f"{size_bytes / 1024:.1f} KB"
+    elif size_bytes < 1024 * 1024 * 1024:
+        return f"{size_bytes / (1024 * 1024):.1f} MB"
+    else:
+        return f"{size_bytes / (1024 * 1024 * 1024):.1f} GB"
+
+
+def get_delimiter_name(delimiter: str) -> str:
+    """
+    Convert delimiter to human-readable name.
+
+    Args:
+        delimiter: Delimiter character
+
+    Returns:
+        Human-readable delimiter name (e.g., "Comma (,)")
+    """
+    return DELIMITER_NAMES.get(delimiter, repr(delimiter))
+
+
+@dataclass
+class CSVSourceInfo:
+    """
+    Source information for CSV metadata extraction.
+    
+    Container for data passed to CSVMetadataExtractor.extract().
+    """
+    file_path: str
+    encoding: str
+    delimiter: str
+    rows: List[List[str]]
+    has_header: bool
+
+
+class CSVMetadataExtractor(BaseMetadataExtractor):
+    """
+    CSV Metadata Extractor.
+    
+    CSV 파일의 구조 정보를 메타데이터로 추출합니다.
+    
+    지원 필드 (custom 필드에 저장):
+    - file_name, file_size, modified_time
+    - encoding, delimiter
+    - row_count, col_count, has_header, columns
+    
+    사용법:
+        extractor = CSVMetadataExtractor()
+        source = CSVSourceInfo(
+            file_path="data.csv",
+            encoding="utf-8",
+            delimiter=",",
+            rows=parsed_rows,
+            has_header=True
+        )
+        metadata = extractor.extract(source)
+        text = extractor.format(metadata)
+    """
+    
+    # CSV 특화 필드 라벨
+    CSV_FIELD_LABELS = {
+        'file_name': '파일명',
+        'file_size': '파일 크기',
+        'modified_time': '수정일',
+        'encoding': '인코딩',
+        'delimiter': '구분자',
+        'row_count': '행 수',
+        'col_count': '열 수',
+        'has_header': '헤더 존재',
+        'columns': '컬럼 목록',
+    }
+    
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        # CSV용 커스텀 포맷터 설정
+        self._formatter.field_labels.update(self.CSV_FIELD_LABELS)
+    
+    def extract(self, source: CSVSourceInfo) -> DocumentMetadata:
+        """
+        CSV 파일에서 메타데이터를 추출합니다.
+        
+        Args:
+            source: CSVSourceInfo 객체 (파일 경로, 인코딩, 구분자, 행 데이터, 헤더 여부)
+            
+        Returns:
+            추출된 메타데이터가 담긴 DocumentMetadata 인스턴스
+        """
+        custom_fields: Dict[str, Any] = {}
+
+        try:
+            # 파일 정보
+            file_stat = os.stat(source.file_path)
+            file_name = os.path.basename(source.file_path)
+
+            custom_fields['file_name'] = file_name
+            custom_fields['file_size'] = format_file_size(file_stat.st_size)
+            custom_fields['modified_time'] = datetime.fromtimestamp(file_stat.st_mtime)
+
+            # CSV 구조 정보
+            custom_fields['encoding'] = source.encoding
+            custom_fields['delimiter'] = get_delimiter_name(source.delimiter)
+            custom_fields['row_count'] = len(source.rows)
+            custom_fields['col_count'] = len(source.rows[0]) if source.rows else 0
+            custom_fields['has_header'] = '예' if source.has_header else '아니오'
+
+            # 헤더 정보 (있는 경우)
+            if source.has_header and source.rows:
+                headers = [h.strip() for h in source.rows[0] if h.strip()]
+                if headers:
+                    custom_fields['columns'] = ', '.join(headers[:10])  # 최대 10개
+                    if len(source.rows[0]) > 10:
+                        custom_fields['columns'] += f' 외 {len(source.rows[0]) - 10}개'
+
+            self.logger.debug(f"Extracted CSV metadata: {list(custom_fields.keys())}")
+
+        except Exception as e:
+            self.logger.warning(f"Failed to extract CSV metadata: {e}")
+        
+        # CSV는 표준 필드가 없고 모두 custom 필드
+        return DocumentMetadata(custom=custom_fields)
+
+
+__all__ = [
+    'CSVMetadataExtractor',
+    'CSVSourceInfo',
+    'format_file_size',
+    'get_delimiter_name',
+]