xgen-doc2chunk 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

xgen_doc2chunk/core/processor/doc_helpers/doc_file_converter.py

@@ -0,0 +1,160 @@
+# xgen_doc2chunk/core/processor/doc_helpers/doc_file_converter.py
+"""
+DOCFileConverter - DOC file format converter
+
+Converts binary DOC data to appropriate format based on detection.
+Supports RTF, OLE, HTML, and misnamed DOCX files.
+"""
+from io import BytesIO
+from typing import Any, Optional, BinaryIO, Tuple
+from enum import Enum
+import zipfile
+
+from xgen_doc2chunk.core.functions.file_converter import BaseFileConverter
+
+
+class DocFormat(Enum):
+    """Detected DOC file format."""
+    RTF = "rtf"
+    OLE = "ole"
+    HTML = "html"
+    DOCX = "docx"
+    UNKNOWN = "unknown"
+
+
+class DOCFileConverter(BaseFileConverter):
+    """
+    DOC file converter with format auto-detection.
+
+    Detects actual format (RTF, OLE, HTML, DOCX) and converts accordingly.
+    """
+
+    # Magic numbers for format detection
+    MAGIC_RTF = b'{\\rtf'
+    MAGIC_OLE = b'\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1'
+    MAGIC_ZIP = b'PK\x03\x04'
+
+    def __init__(self):
+        """Initialize DOCFileConverter."""
+        self._detected_format: DocFormat = DocFormat.UNKNOWN
+
+    def convert(
+        self,
+        file_data: bytes,
+        file_stream: Optional[BinaryIO] = None,
+        **kwargs
+    ) -> Tuple[Any, DocFormat]:
+        """
+        Convert binary DOC data to appropriate format.
+
+        Args:
+            file_data: Raw binary DOC data
+            file_stream: Optional file stream
+            **kwargs: Additional options
+
+        Returns:
+            Tuple of (converted object, detected format)
+            - RTF: (bytes, DocFormat.RTF) - Returns raw binary (processed by RTFHandler)
+            - OLE: (olefile.OleFileIO, DocFormat.OLE)
+            - HTML: (BeautifulSoup, DocFormat.HTML)
+            - DOCX: (docx.Document, DocFormat.DOCX)
+
+        Raises:
+            Exception: If conversion fails
+        """
+        self._detected_format = self._detect_format(file_data)
+
+        if self._detected_format == DocFormat.RTF:
+            # RTF returns raw binary - processed by RTFHandler.extract_text()
+            return file_data, self._detected_format
+        elif self._detected_format == DocFormat.OLE:
+            return self._convert_ole(file_data), self._detected_format
+        elif self._detected_format == DocFormat.HTML:
+            return self._convert_html(file_data), self._detected_format
+        elif self._detected_format == DocFormat.DOCX:
+            return self._convert_docx(file_data), self._detected_format
+        else:
+            # Try OLE as fallback
+            return self._convert_ole(file_data), DocFormat.OLE
+
+    def _detect_format(self, file_data: bytes) -> DocFormat:
+        """Detect actual file format from binary data."""
+        if not file_data:
+            return DocFormat.UNKNOWN
+
+        header = file_data[:32] if len(file_data) >= 32 else file_data
+
+        # Check RTF
+        if header.startswith(self.MAGIC_RTF):
+            return DocFormat.RTF
+
+        # Check OLE
+        if header.startswith(self.MAGIC_OLE):
+            return DocFormat.OLE
+
+        # Check ZIP (possible DOCX)
+        if header.startswith(self.MAGIC_ZIP):
+            try:
+                with zipfile.ZipFile(BytesIO(file_data), 'r') as zf:
+                    if '[Content_Types].xml' in zf.namelist():
+                        return DocFormat.DOCX
+            except zipfile.BadZipFile:
+                pass
+
+        # Check HTML
+        header_lower = header.lower()
+        if (header_lower.startswith(b'<!doctype') or
+            header_lower.startswith(b'<html') or
+            b'<html' in header_lower[:100]):
+            return DocFormat.HTML
+
+        # Check for BOM + RTF
+        if header.startswith(b'\xef\xbb\xbf'):
+            text_header = header[3:].decode('utf-8', errors='ignore').lower()
+            if text_header.startswith('{\\rtf'):
+                return DocFormat.RTF
+
+        return DocFormat.UNKNOWN
+
+    def _convert_ole(self, file_data: bytes) -> Any:
+        """Convert OLE data."""
+        import olefile
+        return olefile.OleFileIO(BytesIO(file_data))
+
+    def _convert_html(self, file_data: bytes) -> Any:
+        """Convert HTML data."""
+        from bs4 import BeautifulSoup
+        # Decode with fallback
+        try:
+            text = file_data.decode('utf-8')
+        except UnicodeDecodeError:
+            text = file_data.decode('cp949', errors='replace')
+        return BeautifulSoup(text, 'html.parser')
+
+    def _convert_docx(self, file_data: bytes) -> Any:
+        """Convert misnamed DOCX data."""
+        from docx import Document
+        return Document(BytesIO(file_data))
+
+    def get_format_name(self) -> str:
+        """Return detected format name."""
+        format_names = {
+            DocFormat.RTF: "RTF Document",
+            DocFormat.OLE: "OLE Document (DOC)",
+            DocFormat.HTML: "HTML Document",
+            DocFormat.DOCX: "DOCX Document (misnamed)",
+            DocFormat.UNKNOWN: "Unknown DOC Format",
+        }
+        return format_names.get(self._detected_format, "Unknown")
+
+    @property
+    def detected_format(self) -> DocFormat:
+        """Return detected format after conversion."""
+        return self._detected_format
+
+    def close(self, converted_object: Any) -> None:
+        """Close the converted object if needed."""
+        if converted_object is not None:
+            if hasattr(converted_object, 'close'):
+                converted_object.close()
+

xgen_doc2chunk/core/processor/doc_helpers/doc_image_processor.py

@@ -0,0 +1,179 @@
+# xgen_doc2chunk/core/processor/doc_helpers/doc_image_processor.py
+"""
+DOC Image Processor
+
+Provides DOC-specific image processing that inherits from ImageProcessor.
+Handles images from RTF, OLE compound documents, and HTML-formatted DOC files.
+"""
+import logging
+from typing import Any, Dict, Optional, Set
+
+from xgen_doc2chunk.core.functions.img_processor import ImageProcessor
+from xgen_doc2chunk.core.functions.storage_backend import BaseStorageBackend
+
+logger = logging.getLogger("xgen_doc2chunk.image_processor.doc")
+
+
+class DOCImageProcessor(ImageProcessor):
+    """
+    DOC-specific image processor.
+    
+    Inherits from ImageProcessor and provides DOC-specific processing.
+    
+    Handles:
+    - RTF embedded images (pict, shppict, blipuid)
+    - OLE compound document images (Pictures stream, embedded objects)
+    - HTML-format DOC images (base64 encoded)
+    - WMF/EMF metafiles
+    
+    Example:
+        processor = DOCImageProcessor()
+        
+        # Process RTF picture
+        tag = processor.process_image(image_data, source="rtf", blipuid="abc123")
+        
+        # Process OLE embedded image
+        tag = processor.process_ole_image(ole_data, stream_name="Pictures/image1.png")
+        
+        # Process HTML base64 image
+        tag = processor.process_html_image(base64_data, src_attr="data:image/png;base64,...")
+    """
+    
+    def __init__(
+        self,
+        directory_path: str = "temp/images",
+        tag_prefix: str = "[Image:",
+        tag_suffix: str = "]",
+        storage_backend: Optional[BaseStorageBackend] = None,
+    ):
+        """
+        Initialize DOCImageProcessor.
+        
+        Args:
+            directory_path: Image save directory
+            tag_prefix: Tag prefix for image references
+            tag_suffix: Tag suffix for image references
+            storage_backend: Storage backend for saving images
+        """
+        super().__init__(
+            directory_path=directory_path,
+            tag_prefix=tag_prefix,
+            tag_suffix=tag_suffix,
+            storage_backend=storage_backend,
+        )
+        self._processed_blipuids: Set[str] = set()
+    
+    def process_image(
+        self,
+        image_data: bytes,
+        source: Optional[str] = None,
+        blipuid: Optional[str] = None,
+        stream_name: Optional[str] = None,
+        **kwargs
+    ) -> Optional[str]:
+        """
+        Process and save DOC image data.
+        
+        Args:
+            image_data: Raw image binary data
+            source: Image source type ("rtf", "ole", "html")
+            blipuid: RTF BLIP unique ID (for deduplication)
+            stream_name: OLE stream name
+            **kwargs: Additional options
+            
+        Returns:
+            Image tag string or None if processing failed
+        """
+        # Custom naming based on source
+        custom_name = None
+        
+        if source == "rtf" and blipuid:
+            # Use blipuid for RTF images (deduplication key)
+            if blipuid in self._processed_blipuids:
+                logger.debug(f"Skipping duplicate RTF image: {blipuid}")
+                return None
+            self._processed_blipuids.add(blipuid)
+            custom_name = f"rtf_{blipuid[:16]}"
+        elif source == "ole" and stream_name:
+            # Use stream name for OLE images
+            import os
+            custom_name = f"ole_{os.path.basename(stream_name).split('.')[0]}"
+        elif source == "html":
+            custom_name = None  # Use hash-based naming
+        
+        return self.save_image(image_data, custom_name=custom_name)
+    
+    def process_ole_image(
+        self,
+        image_data: bytes,
+        stream_name: Optional[str] = None,
+        **kwargs
+    ) -> Optional[str]:
+        """
+        Process OLE compound document embedded image.
+        
+        Args:
+            image_data: Raw image binary data from OLE stream
+            stream_name: Name of the OLE stream
+            **kwargs: Additional options
+            
+        Returns:
+            Image tag string or None if processing failed
+        """
+        return self.process_image(
+            image_data,
+            source="ole",
+            stream_name=stream_name,
+            **kwargs
+        )
+    
+    def process_rtf_image(
+        self,
+        image_data: bytes,
+        blipuid: Optional[str] = None,
+        **kwargs
+    ) -> Optional[str]:
+        """
+        Process RTF embedded image.
+        
+        Args:
+            image_data: Raw image binary data from RTF
+            blipuid: BLIP unique ID for deduplication
+            **kwargs: Additional options
+            
+        Returns:
+            Image tag string or None if processing failed
+        """
+        return self.process_image(
+            image_data,
+            source="rtf",
+            blipuid=blipuid,
+            **kwargs
+        )
+    
+    def process_html_image(
+        self,
+        image_data: bytes,
+        src_attr: Optional[str] = None,
+        **kwargs
+    ) -> Optional[str]:
+        """
+        Process HTML-format DOC base64 image.
+        
+        Args:
+            image_data: Decoded image binary data
+            src_attr: Original src attribute value
+            **kwargs: Additional options
+            
+        Returns:
+            Image tag string or None if processing failed
+        """
+        return self.process_image(
+            image_data,
+            source="html",
+            **kwargs
+        )
+    
+    def reset_tracking(self) -> None:
+        """Reset processed image tracking for new document."""
+        self._processed_blipuids.clear()

xgen_doc2chunk/core/processor/doc_helpers/doc_preprocessor.py

@@ -0,0 +1,83 @@
+# xgen_doc2chunk/core/processor/doc_helpers/doc_preprocessor.py
+"""
+DOC Preprocessor - Process DOC content after conversion.
+
+Processing Pipeline Position:
+    1. DOCFileConverter.convert() ??(converted_obj, DocFormat)
+    2. DOCPreprocessor.preprocess() ??PreprocessedData (THIS STEP)
+    3. Content extraction (depends on format: RTF, OLE, HTML, DOCX)
+
+Current Implementation:
+    - Pass-through (DOC delegates to format-specific handlers)
+"""
+import logging
+from typing import Any, Dict
+
+from xgen_doc2chunk.core.functions.preprocessor import (
+    BasePreprocessor,
+    PreprocessedData,
+)
+
+logger = logging.getLogger("xgen_doc2chunk.doc.preprocessor")
+
+
+class DOCPreprocessor(BasePreprocessor):
+    """
+    DOC Document Preprocessor.
+
+    Currently a pass-through implementation as DOC processing
+    delegates to format-specific handlers (RTF, OLE, HTML, DOCX).
+    """
+
+    def preprocess(
+        self,
+        converted_data: Any,
+        **kwargs
+    ) -> PreprocessedData:
+        """
+        Preprocess the converted DOC content.
+
+        Args:
+            converted_data: Tuple of (converted_obj, DocFormat) from DOCFileConverter
+            **kwargs: Additional options
+
+        Returns:
+            PreprocessedData with the converted object
+        """
+        metadata: Dict[str, Any] = {}
+
+        converted_obj = converted_data
+        doc_format = None
+
+        # Handle tuple return from DOCFileConverter
+        if isinstance(converted_data, tuple) and len(converted_data) >= 2:
+            converted_obj, doc_format = converted_data[0], converted_data[1]
+            if hasattr(doc_format, 'value'):
+                metadata['detected_format'] = doc_format.value
+            else:
+                metadata['detected_format'] = str(doc_format)
+
+        logger.debug("DOC preprocessor: pass-through, metadata=%s", metadata)
+
+        # clean_content is the TRUE SOURCE - contains the converted object
+        # For DOC, this is the format-specific object (OLE, BeautifulSoup, etc.)
+        return PreprocessedData(
+            raw_content=converted_data,
+            clean_content=converted_obj,  # TRUE SOURCE - the converted object
+            encoding="utf-8",
+            extracted_resources={"doc_format": doc_format},
+            metadata=metadata,
+        )
+
+    def get_format_name(self) -> str:
+        """Return format name."""
+        return "DOC Preprocessor"
+
+    def validate(self, data: Any) -> bool:
+        """Validate if data is DOC conversion result."""
+        if isinstance(data, tuple) and len(data) >= 2:
+            return True
+        return data is not None
+
+
+__all__ = ['DOCPreprocessor']