xgen-doc2chunk 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

xgen_doc2chunk/core/processor/docx_helper/docx_paragraph.py

@@ -0,0 +1,126 @@
+# xgen_doc2chunk/core/processor/docx_helper/docx_paragraph.py
+"""
+DOCX Paragraph Processing Utility
+
+Processes Paragraph elements in DOCX documents.
+- process_paragraph_element: Process Paragraph element
+- has_page_break_element: Check for page break
+
+Image and drawing extraction is handled by DOCXImageProcessor.
+"""
+import logging
+from typing import Optional, Set, Tuple, Callable, TYPE_CHECKING
+
+from docx import Document
+
+from xgen_doc2chunk.core.processor.docx_helper.docx_constants import ElementType, NAMESPACES
+
+if TYPE_CHECKING:
+    from xgen_doc2chunk.core.processor.docx_helper.docx_image_processor import DOCXImageProcessor
+
+logger = logging.getLogger("document-processor")
+
+
+def process_paragraph_element(
+    para_elem,
+    doc: Document,
+    processed_images: Set[str],
+    file_path: str = None,
+    image_processor: Optional["DOCXImageProcessor"] = None,
+    chart_callback: Optional[Callable[[], str]] = None
+) -> Tuple[str, bool, int, int]:
+    """
+    Process Paragraph element.
+
+    Extracts text, images, charts and detects page breaks.
+
+    Args:
+        para_elem: paragraph XML element
+        doc: python-docx Document object
+        processed_images: Set of processed image paths (deduplication)
+        file_path: Original file path
+        image_processor: DOCXImageProcessor instance
+        chart_callback: Callback function to get next chart content
+
+    Returns:
+        (content, has_page_break, image_count, chart_count) tuple
+    """
+    content_parts = []
+    has_page_break = False
+    image_count = 0
+    chart_count = 0
+
+    try:
+        # Check for page break
+        has_page_break = has_page_break_element(para_elem)
+
+        # Traverse Run elements
+        for run_elem in para_elem.findall('.//w:r', NAMESPACES):
+            # Extract text
+            for t_elem in run_elem.findall('w:t', NAMESPACES):
+                if t_elem.text:
+                    content_parts.append(t_elem.text)
+
+            # Process Drawing (image/chart/diagram) via DOCXImageProcessor
+            for drawing_elem in run_elem.findall('w:drawing', NAMESPACES):
+                if image_processor and hasattr(image_processor, 'process_drawing_element'):
+                    drawing_content, drawing_type = image_processor.process_drawing_element(
+                        drawing_elem, doc, processed_images, chart_callback=chart_callback
+                    )
+                else:
+                    drawing_content, drawing_type = "", None
+                if drawing_content:
+                    content_parts.append(drawing_content)
+                    if drawing_type == ElementType.IMAGE:
+                        image_count += 1
+                    elif drawing_type == ElementType.CHART:
+                        chart_count += 1
+
+            # Process pict element (legacy VML image) - use DOCXImageProcessor
+            for pict_elem in run_elem.findall('w:pict', NAMESPACES):
+                if image_processor and hasattr(image_processor, 'extract_from_pict'):
+                    pict_content = image_processor.extract_from_pict(pict_elem, doc, processed_images)
+                else:
+                    pict_content = "[Unknown Image]"
+                if pict_content:
+                    content_parts.append(pict_content)
+                    image_count += 1
+
+    except Exception as e:
+        logger.warning(f"Error processing paragraph: {e}")
+        # Fallback: simple text extraction
+        try:
+            texts = para_elem.findall('.//w:t', NAMESPACES)
+            content_parts = [t.text or '' for t in texts]
+        except:
+            pass
+
+    return ''.join(content_parts), has_page_break, image_count, chart_count
+
+
+def has_page_break_element(element) -> bool:
+    """
+    Check if element contains a page break.
+
+    Args:
+        element: XML element
+
+    Returns:
+        Whether page break exists
+    """
+    try:
+        # Explicit page break
+        if element.findall('.//w:br[@w:type="page"]', NAMESPACES):
+            return True
+        # Rendered page break
+        if element.findall('.//w:lastRenderedPageBreak', NAMESPACES):
+            return True
+        return False
+    except Exception:
+        return False
+
+
+__all__ = [
+    'process_paragraph_element',
+    'has_page_break_element',
+]

xgen_doc2chunk/core/processor/docx_helper/docx_preprocessor.py

@@ -0,0 +1,82 @@
+# xgen_doc2chunk/core/processor/docx_helper/docx_preprocessor.py
+"""
+DOCX Preprocessor - Process DOCX document after conversion.
+
+Processing Pipeline Position:
+    1. DOCXFileConverter.convert() ??docx.Document
+    2. DOCXPreprocessor.preprocess() ??PreprocessedData (THIS STEP)
+    3. DOCXMetadataExtractor.extract() ??DocumentMetadata
+    4. Content extraction (paragraphs, tables, images)
+
+Current Implementation:
+    - Pass-through (DOCX uses python-docx Document object directly)
+"""
+import logging
+from typing import Any, Dict
+
+from xgen_doc2chunk.core.functions.preprocessor import (
+    BasePreprocessor,
+    PreprocessedData,
+)
+
+logger = logging.getLogger("xgen_doc2chunk.docx.preprocessor")
+
+
+class DOCXPreprocessor(BasePreprocessor):
+    """
+    DOCX Document Preprocessor.
+
+    Currently a pass-through implementation as DOCX processing
+    is handled during the content extraction phase using python-docx.
+    """
+
+    def preprocess(
+        self,
+        converted_data: Any,
+        **kwargs
+    ) -> PreprocessedData:
+        """
+        Preprocess the converted DOCX document.
+
+        Args:
+            converted_data: docx.Document object from DOCXFileConverter
+            **kwargs: Additional options
+
+        Returns:
+            PreprocessedData with the document and any extracted resources
+        """
+        metadata: Dict[str, Any] = {}
+
+        # Extract basic document info if available
+        if hasattr(converted_data, 'core_properties'):
+            props = converted_data.core_properties
+            if hasattr(props, 'title') and props.title:
+                metadata['title'] = props.title
+
+        if hasattr(converted_data, 'paragraphs'):
+            metadata['paragraph_count'] = len(converted_data.paragraphs)
+
+        if hasattr(converted_data, 'tables'):
+            metadata['table_count'] = len(converted_data.tables)
+
+        logger.debug("DOCX preprocessor: pass-through, metadata=%s", metadata)
+
+        # clean_content is the TRUE SOURCE - contains the docx.Document
+        return PreprocessedData(
+            raw_content=converted_data,
+            clean_content=converted_data,  # TRUE SOURCE - docx.Document
+            encoding="utf-8",
+            extracted_resources={},
+            metadata=metadata,
+        )
+
+    def get_format_name(self) -> str:
+        """Return format name."""
+        return "DOCX Preprocessor"
+
+    def validate(self, data: Any) -> bool:
+        """Validate if data is a DOCX Document object."""
+        return hasattr(data, 'paragraphs') and hasattr(data, 'tables')
+
+
+__all__ = ['DOCXPreprocessor']