xgen-doc2chunk 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

xgen_doc2chunk/core/processor/hwpx_helper/hwpx_image_processor.py

@@ -0,0 +1,258 @@
+# xgen_doc2chunk/core/processor/hwpx_helper/hwpx_image_processor.py
+"""
+HWPX Image Processor
+
+Provides HWPX-specific image processing that inherits from ImageProcessor.
+Handles images in HWPX (ZIP/XML based) Korean document format.
+
+This class consolidates all HWPX image extraction logic including:
+- BinData images extraction from ZIP
+- Remaining images processing
+- Image filtering by extension
+"""
+import logging
+import os
+from typing import Any, Dict, List, Optional, Set, TYPE_CHECKING
+import zipfile
+
+from xgen_doc2chunk.core.functions.img_processor import ImageProcessor
+from xgen_doc2chunk.core.functions.storage_backend import BaseStorageBackend
+
+logger = logging.getLogger("xgen_doc2chunk.image_processor.hwpx")
+
+# Supported image extensions
+SUPPORTED_IMAGE_EXTENSIONS = frozenset(['.png', '.jpg', '.jpeg', '.gif', '.bmp', '.tiff'])
+
+
+class HWPXImageProcessor(ImageProcessor):
+    """
+    HWPX-specific image processor.
+    
+    Inherits from ImageProcessor and provides HWPX-specific processing.
+    
+    Handles:
+    - BinData images in HWPX ZIP structure
+    - Embedded images
+    - Referenced images via bin_item_map
+    
+    Example:
+        processor = HWPXImageProcessor()
+        
+        # Process image from ZIP
+        with zipfile.ZipFile(file_stream, 'r') as zf:
+            tag = processor.process_from_zip(zf, "BinData/image1.png")
+    """
+    
+    def __init__(
+        self,
+        directory_path: str = "temp/images",
+        tag_prefix: str = "[Image:",
+        tag_suffix: str = "]",
+        storage_backend: Optional[BaseStorageBackend] = None,
+    ):
+        """
+        Initialize HWPXImageProcessor.
+        
+        Args:
+            directory_path: Image save directory
+            tag_prefix: Tag prefix for image references
+            tag_suffix: Tag suffix for image references
+            storage_backend: Storage backend for saving images
+        """
+        super().__init__(
+            directory_path=directory_path,
+            tag_prefix=tag_prefix,
+            tag_suffix=tag_suffix,
+            storage_backend=storage_backend,
+        )
+    
+    def process_image(
+        self,
+        image_data: bytes,
+        bin_item_id: Optional[str] = None,
+        image_path: Optional[str] = None,
+        **kwargs
+    ) -> Optional[str]:
+        """
+        Process and save HWPX image data.
+        
+        Args:
+            image_data: Raw image binary data
+            bin_item_id: BinItem ID from HWPX
+            image_path: Original path in ZIP (for naming)
+            **kwargs: Additional options
+            
+        Returns:
+            Image tag string, or None on failure
+        """
+        custom_name = None
+        if bin_item_id is not None:
+            custom_name = f"hwpx_{bin_item_id}"
+        elif image_path is not None:
+            # Extract filename from path
+            filename = image_path.split('/')[-1] if '/' in image_path else image_path
+            # Remove extension and sanitize
+            name_base = filename.rsplit('.', 1)[0] if '.' in filename else filename
+            custom_name = f"hwpx_{name_base}"
+        
+        return self.save_image(image_data, custom_name=custom_name)
+    
+    def process_from_zip(
+        self,
+        zf: zipfile.ZipFile,
+        image_path: str,
+        bin_item_id: Optional[str] = None,
+    ) -> Optional[str]:
+        """
+        Process image from HWPX ZIP archive.
+        
+        Args:
+            zf: ZipFile object
+            image_path: Path to image in ZIP
+            bin_item_id: BinItem ID
+            
+        Returns:
+            Image tag string, or None on failure
+        """
+        try:
+            with zf.open(image_path) as f:
+                image_data = f.read()
+            
+            return self.process_image(
+                image_data,
+                bin_item_id=bin_item_id,
+                image_path=image_path
+            )
+            
+        except Exception as e:
+            self._logger.warning(f"Failed to process image from ZIP {image_path}: {e}")
+            return None
+    
+    def process_embedded_image(
+        self,
+        image_data: bytes,
+        image_name: Optional[str] = None,
+        bin_item_id: Optional[str] = None,
+        **kwargs
+    ) -> Optional[str]:
+        """
+        Process embedded HWPX image.
+        
+        Args:
+            image_data: Image binary data
+            image_name: Original image filename
+            bin_item_id: BinItem ID
+            **kwargs: Additional options
+            
+        Returns:
+            Image tag string, or None on failure
+        """
+        custom_name = image_name
+        if custom_name is None and bin_item_id is not None:
+            custom_name = f"hwpx_embed_{bin_item_id}"
+        
+        return self.save_image(image_data, custom_name=custom_name)
+    
+    def process_bindata_images(
+        self,
+        zf: zipfile.ZipFile,
+        bin_item_map: Dict[str, str],
+        exclude_processed: Optional[Set[str]] = None,
+    ) -> Dict[str, str]:
+        """
+        Process all BinData images from HWPX.
+        
+        Args:
+            zf: ZipFile object
+            bin_item_map: Mapping of bin_item_id to path
+            exclude_processed: Set of already processed IDs to skip
+            
+        Returns:
+            Dictionary mapping bin_item_id to image tag
+        """
+        exclude = exclude_processed or set()
+        result = {}
+        
+        for bin_id, image_path in bin_item_map.items():
+            if bin_id in exclude:
+                continue
+            
+            tag = self.process_from_zip(zf, image_path, bin_item_id=bin_id)
+            if tag:
+                result[bin_id] = tag
+        
+        return result
+    
+    def process_images(
+        self,
+        zf: zipfile.ZipFile,
+        image_files: List[str],
+    ) -> str:
+        """
+        Extract images from HWPX zip and save locally.
+
+        Args:
+            zf: Open ZipFile object
+            image_files: List of image file paths to process
+
+        Returns:
+            Image tag strings joined by newlines
+        """
+        results = []
+
+        for img_path in image_files:
+            ext = os.path.splitext(img_path)[1].lower()
+            if ext in SUPPORTED_IMAGE_EXTENSIONS:
+                tag = self.process_from_zip(zf, img_path)
+                if tag:
+                    results.append(tag)
+
+        return "\n\n".join(results)
+    
+    def get_remaining_images(
+        self,
+        zf: zipfile.ZipFile,
+        processed_images: Set[str],
+    ) -> List[str]:
+        """
+        Return list of image files not yet processed.
+
+        Args:
+            zf: Open ZipFile object
+            processed_images: Set of already processed image paths
+
+        Returns:
+            List of unprocessed image file paths
+        """
+        image_files = [
+            f for f in zf.namelist()
+            if f.startswith("BinData/") and not f.endswith("/")
+        ]
+
+        remaining_images = []
+        for img in image_files:
+            if img not in processed_images:
+                remaining_images.append(img)
+
+        return remaining_images
+    
+    def process_remaining_images(
+        self,
+        zf: zipfile.ZipFile,
+        processed_images: Set[str],
+    ) -> str:
+        """
+        Process all images not yet processed.
+
+        Args:
+            zf: Open ZipFile object
+            processed_images: Set of already processed image paths
+
+        Returns:
+            Image tag strings joined by newlines
+        """
+        remaining = self.get_remaining_images(zf, processed_images)
+        return self.process_images(zf, remaining)
+
+
+__all__ = ["HWPXImageProcessor"]

xgen_doc2chunk/core/processor/hwpx_helper/hwpx_metadata.py

@@ -0,0 +1,163 @@
+# xgen_doc2chunk/core/processor/hwpx_helper/hwpx_metadata.py
+"""
+HWPX Metadata Extraction Module
+
+Provides HWPXMetadataExtractor class for extracting metadata from HWPX files.
+Implements BaseMetadataExtractor interface.
+
+Metadata locations in HWPX:
+- version.xml: Document version information
+- META-INF/container.xml: Container information
+- Contents/header.xml: Document properties (author, date, etc.)
+
+Note: HWPX is a Korean-native document format, so Korean metadata labels
+are preserved in output for proper display.
+"""
+import logging
+import xml.etree.ElementTree as ET
+import zipfile
+from typing import Any, Dict
+
+from xgen_doc2chunk.core.functions.metadata_extractor import (
+    BaseMetadataExtractor,
+    DocumentMetadata,
+)
+from xgen_doc2chunk.core.processor.hwpx_helper.hwpx_constants import HWPX_NAMESPACES, HEADER_FILE_PATHS
+
+logger = logging.getLogger("document-processor")
+
+
+class HWPXMetadataExtractor(BaseMetadataExtractor):
+    """
+    HWPX Metadata Extractor.
+    
+    Extracts HWPX metadata from zipfile.ZipFile objects.
+    
+    Supported fields:
+    - Standard fields: title, subject, author, keywords, comments, etc.
+    - HWPX-specific: version, media_type, etc. (stored in custom fields)
+    
+    Usage:
+        extractor = HWPXMetadataExtractor()
+        metadata = extractor.extract(zip_file)
+        text = extractor.format(metadata)
+    """
+    
+    def extract(self, source: zipfile.ZipFile) -> DocumentMetadata:
+        """
+        Extract metadata from HWPX file.
+        
+        Args:
+            source: Open zipfile.ZipFile object
+            
+        Returns:
+            DocumentMetadata instance containing extracted metadata.
+        """
+        raw_metadata: Dict[str, Any] = {}
+
+        try:
+            # Try to read header.xml for document properties
+            for header_path in HEADER_FILE_PATHS:
+                if header_path in source.namelist():
+                    with source.open(header_path) as f:
+                        header_content = f.read()
+                        header_root = ET.fromstring(header_content)
+
+                        # Try to find document properties
+                        # <hh:docInfo> contains metadata
+                        doc_info = header_root.find('.//hh:docInfo', HWPX_NAMESPACES)
+                        if doc_info is not None:
+                            # Get properties
+                            for prop in doc_info:
+                                tag = prop.tag.split('}')[-1] if '}' in prop.tag else prop.tag
+                                if prop.text:
+                                    raw_metadata[tag.lower()] = prop.text
+                    break
+
+            # Try to read version.xml
+            if 'version.xml' in source.namelist():
+                with source.open('version.xml') as f:
+                    version_content = f.read()
+                    version_root = ET.fromstring(version_content)
+
+                    # Get version info
+                    if version_root.text:
+                        raw_metadata['version'] = version_root.text
+                    for attr in version_root.attrib:
+                        raw_metadata[f'version_{attr}'] = version_root.get(attr)
+
+            # Try to read META-INF/manifest.xml for additional info
+            if 'META-INF/manifest.xml' in source.namelist():
+                with source.open('META-INF/manifest.xml') as f:
+                    manifest_content = f.read()
+                    manifest_root = ET.fromstring(manifest_content)
+
+                    # Get mimetype and other info
+                    for child in manifest_root:
+                        tag = child.tag.split('}')[-1] if '}' in child.tag else child.tag
+                        if tag == 'file-entry':
+                            full_path = child.get('full-path', child.get('{urn:oasis:names:tc:opendocument:xmlns:manifest:1.0}full-path', ''))
+                            if full_path == '/':
+                                media_type = child.get('media-type', child.get('{urn:oasis:names:tc:opendocument:xmlns:manifest:1.0}media-type', ''))
+                                if media_type:
+                                    raw_metadata['media_type'] = media_type
+
+            self.logger.debug(f"Extracted HWPX metadata: {list(raw_metadata.keys())}")
+
+        except Exception as e:
+            self.logger.warning(f"Failed to extract HWPX metadata: {e}")
+        
+        # Separate standard fields from custom fields
+        standard_fields = {'title', 'subject', 'author', 'keywords', 'comments', 
+                          'last_saved_by', 'create_time', 'last_saved_time'}
+        custom_fields = {k: v for k, v in raw_metadata.items() if k not in standard_fields}
+        
+        return DocumentMetadata(
+            title=raw_metadata.get('title'),
+            subject=raw_metadata.get('subject'),
+            author=raw_metadata.get('author'),
+            keywords=raw_metadata.get('keywords'),
+            comments=raw_metadata.get('comments'),
+            last_saved_by=raw_metadata.get('last_saved_by'),
+            create_time=raw_metadata.get('create_time'),
+            last_saved_time=raw_metadata.get('last_saved_time'),
+            custom=custom_fields,
+        )
+
+
+def parse_bin_item_map(zf: zipfile.ZipFile) -> Dict[str, str]:
+    """
+    Parse content.hpf file to create BinItem ID to file path mapping.
+
+    Args:
+        zf: Open ZipFile object
+
+    Returns:
+        Dictionary mapping BinItem ID to file path.
+    """
+    from .hwpx_constants import HPF_PATH, OPF_NAMESPACES
+
+    bin_item_map = {}
+
+    try:
+        if HPF_PATH in zf.namelist():
+            with zf.open(HPF_PATH) as f:
+                hpf_content = f.read()
+                hpf_root = ET.fromstring(hpf_content)
+
+                for item in hpf_root.findall('.//opf:item', OPF_NAMESPACES):
+                    item_id = item.get('id')
+                    href = item.get('href')
+                    if item_id and href:
+                        bin_item_map[item_id] = href
+
+    except Exception as e:
+        logger.warning(f"Failed to parse content.hpf: {e}")
+
+    return bin_item_map
+
+
+__all__ = [
+    'HWPXMetadataExtractor',
+    'parse_bin_item_map',
+]

xgen_doc2chunk/core/processor/hwpx_helper/hwpx_preprocessor.py

@@ -0,0 +1,80 @@
+# xgen_doc2chunk/core/processor/hwpx_helper/hwpx_preprocessor.py
+"""
+HWPX Preprocessor - Process HWPX ZIP document after conversion.
+
+Processing Pipeline Position:
+    1. HWPXFileConverter.convert() ??zipfile.ZipFile
+    2. HWPXPreprocessor.preprocess() ??PreprocessedData (THIS STEP)
+    3. HWPXMetadataExtractor.extract() ??DocumentMetadata
+    4. Content extraction (sections, tables, images)
+
+Current Implementation:
+    - Pass-through (HWPX uses zipfile object directly)
+"""
+import logging
+from typing import Any, Dict
+
+from xgen_doc2chunk.core.functions.preprocessor import (
+    BasePreprocessor,
+    PreprocessedData,
+)
+
+logger = logging.getLogger("xgen_doc2chunk.hwpx.preprocessor")
+
+
+class HWPXPreprocessor(BasePreprocessor):
+    """
+    HWPX ZIP Document Preprocessor.
+
+    Currently a pass-through implementation as HWPX processing
+    is handled during the content extraction phase.
+    """
+
+    def preprocess(
+        self,
+        converted_data: Any,
+        **kwargs
+    ) -> PreprocessedData:
+        """
+        Preprocess the converted HWPX ZIP document.
+
+        Args:
+            converted_data: zipfile.ZipFile object from HWPXFileConverter
+            **kwargs: Additional options
+
+        Returns:
+            PreprocessedData with the ZIP object and any extracted resources
+        """
+        metadata: Dict[str, Any] = {}
+
+        if hasattr(converted_data, 'namelist'):
+            try:
+                files = converted_data.namelist()
+                metadata['file_count'] = len(files)
+                # Check for section files
+                sections = [f for f in files if 'section' in f.lower() and f.endswith('.xml')]
+                metadata['section_count'] = len(sections)
+            except Exception:  # noqa: BLE001
+                pass
+
+        logger.debug("HWPX preprocessor: pass-through, metadata=%s", metadata)
+
+        # clean_content is the TRUE SOURCE - contains the ZipFile
+        return PreprocessedData(
+            raw_content=converted_data,
+            clean_content=converted_data,  # TRUE SOURCE - zipfile.ZipFile
+            encoding="utf-8",
+            extracted_resources={},
+            metadata=metadata,
+        )
+
+    def get_format_name(self) -> str:
+        """Return format name."""
+        return "HWPX Preprocessor"
+
+    def validate(self, data: Any) -> bool:
+        """Validate if data is a ZipFile object."""
+        return hasattr(data, 'namelist') and hasattr(data, 'open')
+
+
+__all__ = ['HWPXPreprocessor']