super-document-processor 0.1.3__py3-none-any.whl

super_document_processor/document_processors/process_docx.py

@@ -0,0 +1,824 @@
+import os
+import re
+import logging
+from collections import Counter
+from docx import Document
+from docx.text.paragraph import Paragraph
+from docx.table import Table
+from docx.oxml.ns import qn
+
+logger = logging.getLogger("super_document_processor.docx")
+try:
+    from super_document_processor.utils.docx_image_utils import quick_image_check, extract_images_from_docx
+    from super_document_processor.utils.docx_xml_parser import print_list_item, extract_footnotes, extract_endnotes
+    from super_document_processor.utils.vlm import generate_image_description
+    from super_document_processor.utils.chunker import collector
+except ImportError:
+    from utils.docx_image_utils import quick_image_check, extract_images_from_docx
+    from utils.docx_xml_parser import print_list_item, extract_footnotes, extract_endnotes
+    from utils.vlm import generate_image_description
+    from utils.chunker import collector
+
+class DocxElement:
+    """Base class to mimic unstructured element structure"""
+    def __init__(self, text="", element_type="", metadata=None):
+        self.text = text
+        self.element_type = element_type
+        self.metadata = metadata or {}
+
+class DocxMetadata:
+    """Metadata container to mimic unstructured metadata"""
+    def __init__(self):
+        self.emphasized_text_contents = []
+        self.emphasized_text_tags = []
+        self.links = []
+        self.text_as_html = None
+        self.category_depth = 0
+        self.header_footer_type = 'primary'
+        self.list_level = 0
+        self.is_ordered = False
+
+def extract_text_formatting(paragraph):
+    """Extract formatting information and inline hyperlinks in true child order"""
+    emphasized_contents = []
+    emphasized_tags = []
+    links = []
+
+    # Check for hyperlinks across child elements
+    for child in paragraph._element:
+        tag = child.tag.split('}')[-1]
+        if tag == 'hyperlink':
+            r_id = child.get(qn('r:id'))
+            if r_id:
+                try:
+                    rel = paragraph.part.rels.get(r_id)
+                    url = rel.target_ref if rel else None
+                    link_txt = ''.join([t_el.text for t_el in child.findall('.//' + qn('w:t')) if t_el.text]).strip()
+                    if link_txt and url:
+                        links.append({'text': link_txt, 'url': url})
+                except Exception:
+                    pass
+
+    for run in paragraph.runs:
+        txt = run.text.strip()
+        if txt:
+            tags = []
+            if run.bold:
+                tags.append('b')
+            if run.italic:
+                tags.append('i')
+            if run.underline:
+                tags.append('u')
+            for tag in tags:
+                emphasized_contents.append(txt)
+                emphasized_tags.append(tag)
+
+    return emphasized_contents, emphasized_tags, links
+
+def get_heading_level(paragraph):
+    """Determine heading level from paragraph style"""
+    style_name = paragraph.style.name.lower()
+    if 'heading' in style_name:
+        match = re.search(r'heading\s*(\d+)', style_name)
+        if match:
+            return int(match.group(1))
+    return 0
+
+def build_numbering_map(doc):
+    """
+    Parse numbering.xml in docx part to map numId -> { ilvl: numFmt }
+    e.g. {'1': {'0': 'bullet'}, '5': {'0': 'decimal'}}
+    """
+    numbering_map = {}
+    try:
+        num_part = getattr(doc.part, 'numbering_part', None)
+        if num_part is None:
+            return numbering_map
+        num_xml = num_part._element
+
+        num_to_abstract = {}
+        for num in num_xml.findall(qn('w:num')):
+            nid = num.get(qn('w:numId'))
+            abId_elem = num.find(qn('w:abstractNumId'))
+            if abId_elem is not None:
+                num_to_abstract[nid] = abId_elem.get(qn('w:val'))
+
+        abstract_formats = {}
+        for ab in num_xml.findall(qn('w:abstractNum')):
+            abId = ab.get(qn('w:abstractNumId'))
+            fmts = {}
+            for lvl in ab.findall(qn('w:lvl')):
+                ilvl = lvl.get(qn('w:ilvl'))
+                numFmt = lvl.find(qn('w:numFmt'))
+                if numFmt is not None:
+                    fmts[ilvl] = numFmt.get(qn('w:val'))
+            abstract_formats[abId] = fmts
+
+        for nid, abId in num_to_abstract.items():
+            numbering_map[nid] = abstract_formats.get(abId, {})
+    except Exception as e:
+        logger.debug(f"Could not build numbering map: {e}")
+    return numbering_map
+
+def get_list_level(paragraph):
+    """Get list indentation level from XML numPr or style name"""
+    # 1. Check paragraph-level pPr numPr
+    if paragraph._element.pPr is not None:
+        numPr = paragraph._element.pPr.find(qn('w:numPr'))
+        if numPr is not None:
+            ilvl = numPr.find(qn('w:ilvl'))
+            if ilvl is not None:
+                return int(ilvl.get(qn('w:val'), 0))
+    # 2. Check style hierarchy numPr
+    cur = paragraph.style
+    while cur:
+        if cur._element.pPr is not None:
+            numPr = cur._element.pPr.find(qn('w:numPr'))
+            if numPr is not None:
+                ilvl = numPr.find(qn('w:ilvl'))
+                if ilvl is not None:
+                    return int(ilvl.get(qn('w:val'), 0))
+        cur = getattr(cur, 'base_style', None)
+    # 3. Fallback to style name level (e.g., 'List Bullet 2' -> level 1)
+    style_name = paragraph.style.name.lower()
+    m = re.search(r'(\d+)$', style_name)
+    if m:
+        return max(0, int(m.group(1)) - 1)
+    return 0
+
+def is_list_paragraph(paragraph):
+    """Check if paragraph is part of a list from XML numPr, style name, or bullet/number prefix"""
+    # 1. Paragraph numPr
+    if paragraph._element.pPr is not None:
+        numPr = paragraph._element.pPr.find(qn('w:numPr'))
+        if numPr is not None:
+            return True
+    # 2. Style hierarchy numPr
+    cur = paragraph.style
+    while cur:
+        if cur._element.pPr is not None:
+            numPr = cur._element.pPr.find(qn('w:numPr'))
+            if numPr is not None:
+                return True
+        cur = getattr(cur, 'base_style', None)
+    # 3. Style name
+    style_name = paragraph.style.name.lower()
+    if 'list' in style_name or 'bullet' in style_name:
+        return True
+    # 4. Text prefix
+    txt = paragraph.text.strip()
+    if re.match(r'^(?:[•*➢➤➔➜►▶→⇒»›✓✔★✦✧\u25cf\u25a0\u25a1\u25aa\u25ab\u25cb\u25e6\u25c6\u25c7\u25c8\u2756\u2219\uf0b7\u27a2\uf0d8\uf0fc\uf06f\uf0a7\uf0a8\uf075]|[-–—]\s+|o\s+|\d{1,3}[.)]\s+)', txt):
+        return True
+    return False
+
+def is_ordered_list(paragraph, numbering_map=None):
+    """Check if a list paragraph is ordered (numbered) rather than unordered (bulleted)"""
+    # 1. Resolve numPr from paragraph or style hierarchy
+    numPr = None
+    if paragraph._element.pPr is not None:
+        numPr = paragraph._element.pPr.find(qn('w:numPr'))
+    if numPr is None:
+        cur = paragraph.style
+        while cur:
+            if cur._element.pPr is not None:
+                numPr = cur._element.pPr.find(qn('w:numPr'))
+                if numPr is not None:
+                    break
+            cur = getattr(cur, 'base_style', None)
+
+    # 2. Check numbering format from numbering.xml
+    if numPr is not None and numbering_map:
+        numId_elem = numPr.find(qn('w:numId'))
+        ilvl_elem = numPr.find(qn('w:ilvl'))
+        if numId_elem is not None:
+            nid = numId_elem.get(qn('w:val'))
+            ilvl = ilvl_elem.get(qn('w:val'), '0') if ilvl_elem is not None else '0'
+            fmt = numbering_map.get(nid, {}).get(ilvl) or numbering_map.get(nid, {}).get('0')
+            if fmt:
+                return fmt != 'bullet'
+
+    # 3. Check style name
+    style_name = paragraph.style.name.lower()
+    if 'number' in style_name or 'decimal' in style_name or 'count' in style_name:
+        return True
+    if 'bullet' in style_name:
+        return False
+
+    # 4. Check paragraph text prefix
+    txt = paragraph.text.strip()
+    if re.match(r'^(?:\d{1,3}[.)]|[a-zA-Z][.)]|[ivxlcdmIVXLCDM]+[.)])\s+', txt):
+        return True
+    return False
+
+def extract_table_html(table):
+    """
+    Extract table from the document with full gridSpan (horizontal)
+    and vMerge (vertical) support, formatting cleanly as an HTML table.
+    """
+    tbl_elem = table._element
+    grid = tbl_elem.find(qn('w:tblGrid'))
+    if grid is not None:
+        total_cols = len(grid.findall(qn('w:gridCol')))
+    else:
+        total_cols = max(
+            sum(
+                int(tc.tcPr.find(qn('w:gridSpan')).get(qn('w:val')))
+                if tc.tcPr is not None and tc.tcPr.find(qn('w:gridSpan')) is not None
+                else 1
+                for tc in tr.tc_lst
+            )
+            for tr in tbl_elem.tr_lst
+        ) if tbl_elem.tr_lst else 1
+
+    rows = []
+    vmerge_map = {}
+
+    for tr in tbl_elem.tr_lst:
+        row_cells = [None] * total_cols
+        col_idx = 0
+        for tc in tr.tc_lst:
+            while col_idx < total_cols and row_cells[col_idx] is not None:
+                col_idx += 1
+            if col_idx >= total_cols:
+                break
+
+            tcPr = tc.tcPr
+            gridSpan = tcPr.find(qn('w:gridSpan')) if tcPr is not None else None
+            span_val = int(gridSpan.get(qn('w:val'))) if gridSpan is not None else 1
+
+            vMerge = tcPr.find(qn('w:vMerge')) if tcPr is not None else None
+            v_val = vMerge.get(qn('w:val'), 'continue') if vMerge is not None else None
+
+            p_texts = []
+            for p in tc.iterchildren():
+                if p.tag.endswith('}p'):
+                    txt = ''.join(t.text for t in p.iter() if t.tag.endswith('}t') and t.text).strip()
+                    if txt:
+                        p_texts.append(txt)
+            cell_text = '<br>'.join(p_texts).strip()
+
+            if vMerge is not None:
+                if v_val == 'restart':
+                    vmerge_map[col_idx] = cell_text
+                else:
+                    cell_text = ''
+            else:
+                if col_idx in vmerge_map:
+                    del vmerge_map[col_idx]
+
+            for s in range(span_val):
+                if col_idx + s < total_cols:
+                    row_cells[col_idx + s] = cell_text if s == 0 else ''
+            col_idx += span_val
+
+        rows.append(row_cells)
+
+    html_parts = ['<table>']
+    for i, r in enumerate(rows):
+        html_parts.append('  <tr>')
+        tag = 'th' if i == 0 else 'td'
+        for cell in r:
+            val = cell if cell is not None else ''
+            html_parts.append(f'    <{tag}>{val}</{tag}>')
+        html_parts.append('  </tr>')
+    html_parts.append('</table>')
+    return '\n'.join(html_parts)
+
+def analyze_document_structure(doc):
+    """
+    Analyze document structure to identify where specific images belong.
+    Returns a mapping of block indices to their specific image r:id references.
+    """
+    block_image_map = {}
+    
+    for idx, block in enumerate(doc.element.body):
+        # Look for specific image references in this block
+        image_refs = []
+        
+        # Check for drawing elements with blip r:embed attributes
+        for drawing in block.iter():
+            local_tag = drawing.tag.split("}")[-1]
+            if local_tag == "blip":
+                r_embed = drawing.get(qn('r:embed'))
+                if r_embed:
+                    image_refs.append(r_embed)
+            elif local_tag == "imagedata":
+                r_id = drawing.get(qn('r:id'))
+                if r_id:
+                    image_refs.append(r_id)
+        
+        # If this block contains specific image references, record them
+        if image_refs:
+            block_image_map[idx] = image_refs
+    
+    return block_image_map
+
+def create_image_elements_for_block(block_image_refs, extracted_images, image_desc_cache):
+    """
+    Create image elements for specific images found in a block.
+    Returns a list of DocxElement objects for images, matched by r:id.
+    """
+    image_elements = []
+    
+    # Match images by their r:id references
+    for r_id in block_image_refs:
+        # Find the corresponding image in extracted_images
+        matching_image = None
+        for img_data in extracted_images:
+            if img_data.get('r_id') == r_id:
+                matching_image = img_data
+                break
+        
+        if matching_image:
+            metadata = DocxMetadata()
+            elem = DocxElement("", 'Image', metadata)
+            
+            # Set image attributes
+            elem.image_alt_text = matching_image.get('alt_text', '')
+            elem.image_title = matching_image.get('title', '')
+            elem.image_name = matching_image.get('name', '')
+            elem.image_r_id = matching_image.get('r_id', '')
+            elem.image_data = matching_image.get('data')
+            elem.image_hash = matching_image.get('hash')
+            
+            image_elements.append(elem)
+    
+    return image_elements
+
+def group_consecutive_lists(elements):
+    """Group consecutive list items together"""
+    grouped = []
+    current_list_group = []
+    
+    for element in elements:
+        if element.element_type == 'ListItem':
+            current_list_group.append(element)
+        else:
+            if current_list_group:
+                grouped.append(('LIST_GROUP', current_list_group))
+                current_list_group = []
+            grouped.append(('SINGLE', element))
+    
+    if current_list_group:
+        grouped.append(('LIST_GROUP', current_list_group))
+    
+    return grouped
+            
+def format_text_with_emphasis(text, emphasized_contents, emphasized_tags):
+    """Format text with proper emphasis tags including nested styles."""
+    if not emphasized_contents or not emphasized_tags or not text:
+        return text
+
+    # List of (start_idx, end_idx, tag)
+    emphasis_spans = []
+
+    # Map each content to list of tags
+    for content, tag in zip(emphasized_contents, emphasized_tags):
+        start_idx = text.find(content)
+        while start_idx != -1:
+            end_idx = start_idx + len(content)
+            emphasis_spans.append((start_idx, end_idx, tag))
+            start_idx = text.find(content, start_idx + 1)
+        emphasis_spans.sort()
+
+    # Build final HTML by applying tags in order
+    result = []
+    last_idx = 0
+
+    i = 0
+    while i < len(emphasis_spans):
+        start, end, tag = emphasis_spans[i]
+
+        result.append(text[last_idx:start])
+
+        tags_to_apply = set()
+        while i < len(emphasis_spans) and emphasis_spans[i][0] == start and emphasis_spans[i][1] == end:
+            tags_to_apply.add(emphasis_spans[i][2])
+            i += 1
+
+        # Apply opening tags
+        for tag in sorted(tags_to_apply):
+            result.append(f"<{tag}>")
+
+        result.append(text[start:end])
+
+        for tag in reversed(sorted(tags_to_apply)):
+            result.append(f"</{tag}>")
+
+        last_idx = end
+
+    result.append(text[last_idx:])
+    return ''.join(result)
+
+def process_image_element(element, image_desc_cache, image_counter):
+    """Process a single image element and add it to the collector"""
+    collector.add(f"<-- IMAGE {image_counter} START -->")
+
+    # Extract detailed image metadata
+    alt_text = getattr(element, 'image_alt_text', None)
+    title = getattr(element, 'image_title', None)
+    name = getattr(element, 'image_name', None)
+    r_id = getattr(element, 'image_r_id', None)
+    image_data = getattr(element, 'image_data', None)
+    img_hash = getattr(element, 'image_hash', None)
+
+    # Generate or retrieve cached description
+    if image_data and img_hash:
+        if img_hash in image_desc_cache:
+            generated_desc = image_desc_cache[img_hash]
+        else:
+            try:
+                generated_desc = generate_image_description(image_data)
+                image_desc_cache[img_hash] = generated_desc
+            except Exception as e:
+                generated_desc = f"[Error generating description: {e}]"
+                image_desc_cache[img_hash] = generated_desc
+        collector.add(f"<figcaption>{generated_desc}</figcaption>")
+
+    # Build image attributes
+    img_attrs = []
+    if alt_text:
+        img_attrs.append(f"alt='{alt_text}'")
+    if title:
+        img_attrs.append(f"title='{title}'")
+    if name:
+        img_attrs.append(f"data-name='{name}'")
+    if r_id:
+        img_attrs.append(f"data-r-id='{r_id}'")
+    attrs_str = " " + " ".join(img_attrs) if img_attrs else ""
+    collector.add(f"<img{attrs_str} />")
+    collector.add(f"<-- IMAGE {image_counter} END -->")
+
+def process_docx_file(filename, original_filename=None, process_images=None, model=None):
+    """
+    Extract and display DOCX content in natural document flow with properly positioned images, headers, and footers
+    """
+    try:
+        doc = Document(filename)
+    except Exception as e:
+        logger.error(f"Error reading DOCX '{filename}': {e}")
+        return []
+        
+    display_name = os.path.basename(original_filename) if original_filename else os.path.basename(filename)
+    should_process_images = bool(process_images)
+    has_images = quick_image_check(filename) if should_process_images else False
+    extracted_images = []
+    if has_images:
+        extracted_images = extract_images_from_docx(filename)
+        block_image_map = analyze_document_structure(doc)
+    else:
+        block_image_map = {}
+    
+    elements = []
+    
+    # --- Capture headers and footers for each section ---
+    for section in doc.sections:
+        # Headers
+        for header_type in ['header', 'first_page_header', 'even_page_header']:
+            header = getattr(section, header_type, None)
+            if header and header.is_linked_to_previous is False:
+                for para in header.paragraphs:
+                    text = para.text.strip()
+                    if text:
+                        metadata = DocxMetadata()
+                        metadata.header_footer_type = header_type
+                        metadata.emphasized_text_contents, metadata.emphasized_text_tags, metadata.links = extract_text_formatting(para)
+                        elements.append(DocxElement(text, 'Header', metadata))
+        # Footers
+        for footer_type in ['footer', 'first_page_footer', 'even_page_footer']:
+            footer = getattr(section, footer_type, None)
+            if footer and footer.is_linked_to_previous is False:
+                for para in footer.paragraphs:
+                    text = para.text.strip()
+                    if text:
+                        metadata = DocxMetadata()
+                        metadata.header_footer_type = footer_type
+                        metadata.emphasized_text_contents, metadata.emphasized_text_tags, metadata.links = extract_text_formatting(para)
+                        elements.append(DocxElement(text, 'Footer', metadata))
+    # --- End header/footer capture ---
+
+    # Initialize image description cache
+    image_desc_cache = {}
+    if extracted_images:
+        # Pre-generate descriptions for all images in parallel
+        import concurrent.futures
+        def desc_task(img):
+            try:
+                return img['hash'], generate_image_description(img['data'], model=model)
+            except Exception as e:
+                return img['hash'], f"[Error generating description: {e}]"
+        
+        workers = int(os.environ.get("MAX_WORKERS", 3))
+        with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
+            results = list(executor.map(desc_task, extracted_images))
+        
+        for h, desc in results:
+            image_desc_cache[h] = desc
+    
+    # Build numbering map for resolving ordered vs unordered lists
+    numbering_map = build_numbering_map(doc)
+
+    # Process each block in the document
+    for block_idx, element in enumerate(doc.element.body):
+        tag = element.tag.split('}')[-1] if '}' in element.tag else element.tag
+        
+        # First, add any images that belong to this block
+        if block_idx in block_image_map:
+            block_images = block_image_map[block_idx]
+            image_elements = create_image_elements_for_block(block_images, extracted_images, image_desc_cache)
+            elements.extend(image_elements)
+        
+        # Helper to check if this block has a page break or section break
+        has_break = False
+        if block_idx < len(doc.element.body) - 1:
+            for br in element.iter(qn('w:br')):
+                if br.get(qn('w:type')) == 'page':
+                    has_break = True
+                    break
+            if not has_break:
+                for lr in element.iter(qn('w:lastRenderedPageBreak')):
+                    has_break = True
+                    break
+            if not has_break:
+                pPr = element.find(qn('w:pPr'))
+                if pPr is not None and pPr.find(qn('w:sectPr')) is not None:
+                    has_break = True
+            if not has_break and tag == 'sectPr':
+                has_break = True
+
+        if tag == 'p':  # Paragraph
+            try:
+                para = Paragraph(element, doc)
+            except Exception:
+                para = None
+
+            if para is None:
+                continue
+
+            text = para.text.strip()
+            if not text:
+                # Check if paragraph contains floating text boxes (callout boxes / shape text)
+                has_txbx = False
+                for txbx in element.iter(qn('w:txBxContent')):
+                    for sub_p in txbx.iter(qn('w:p')):
+                        sub_text = ''.join(x.text for x in sub_p.iter() if x.tag.endswith('}t') and x.text).strip()
+                        if sub_text:
+                            sub_meta = DocxMetadata()
+                            elements.append(DocxElement(sub_text, 'NarrativeText', sub_meta))
+                            has_txbx = True
+                if has_txbx:
+                    if has_break:
+                        elements.append(DocxElement("", 'PageBreak', DocxMetadata()))
+                        has_break = False
+                    continue
+
+                # Check if paragraph contains an equation or floating drawing
+                maths = list(element.iter('{http://schemas.openxmlformats.org/officeDocument/2006/math}oMath'))
+                if maths:
+                    m_parts = []
+                    for m in maths:
+                        m_txt = ''.join(x.text for x in m.iter() if x.tag.endswith('}t') and x.text).strip()
+                        if m_txt:
+                            m_parts.append(m_txt)
+                    if m_parts:
+                        text = ' '.join(m_parts)
+                        metadata = DocxMetadata()
+                        elements.append(DocxElement(text, 'Formula', metadata))
+                if has_break:
+                    elements.append(DocxElement("", 'PageBreak', DocxMetadata()))
+                    has_break = False
+                continue
+
+            # Create metadata
+            metadata = DocxMetadata()
+            metadata.emphasized_text_contents, metadata.emphasized_text_tags, metadata.links = extract_text_formatting(para)
+
+            # Determine element type
+            heading_level = get_heading_level(para)
+            if heading_level > 0:
+                metadata.category_depth = heading_level - 1
+                elem = DocxElement(text, 'Title', metadata)
+            elif is_list_paragraph(para):
+                metadata.list_level = get_list_level(para)
+                metadata.is_ordered = is_ordered_list(para, numbering_map)
+                elem = DocxElement(text, 'ListItem', metadata)
+            else:
+                elem = DocxElement(text, 'NarrativeText', metadata)
+
+            elements.append(elem)
+
+            # Also check for floating text boxes nested alongside text
+            for txbx in element.iter(qn('w:txBxContent')):
+                for sub_p in txbx.iter(qn('w:p')):
+                    sub_text = ''.join(x.text for x in sub_p.iter() if x.tag.endswith('}t') and x.text).strip()
+                    if sub_text:
+                        sub_meta = DocxMetadata()
+                        elements.append(DocxElement(sub_text, 'NarrativeText', sub_meta))
+
+        elif tag == 'tbl':  # Table
+            try:
+                table = Table(element, doc)
+            except Exception:
+                table = None
+
+            if table is None:
+                continue
+
+            metadata = DocxMetadata()
+            table_html = extract_table_html(table)
+            metadata.text_as_html = table_html
+            elem = DocxElement(table_html, 'Table', metadata)
+            elements.append(elem)
+
+        if has_break:
+            elements.append(DocxElement("", 'PageBreak', DocxMetadata()))
+    
+    stats = Counter()
+    image_counter = 0
+    page_idx = 1
+    collector.add(f"[Page 1 | File: {display_name}]\n")
+
+    # Group consecutive list items
+    grouped_elements = group_consecutive_lists(elements)
+
+    # Process each element/group in document flow order
+    for group_type, content in grouped_elements:
+        if group_type == 'LIST_GROUP':
+            # Handle list group
+            collector.add("=== List ===")
+            level_counters = {}
+            for list_element in content:
+                element_type = list_element.element_type
+                element_text = list_element.text.strip()
+                metadata = list_element.metadata
+                
+                stats[element_type] += 1
+                
+                # Get list level and ordering flag
+                list_level = getattr(metadata, 'list_level', 0)
+                is_ordered = getattr(metadata, 'is_ordered', False)
+
+                # Determine bullet or numbered marker
+                if is_ordered:
+                    cnt = level_counters.get(list_level, 0) + 1
+                    level_counters[list_level] = cnt
+                    # Reset any deeper nested levels
+                    for k in list(level_counters.keys()):
+                        if k > list_level:
+                            del level_counters[k]
+                    marker = f"{cnt}."
+                else:
+                    level_counters.clear()
+                    marker = "-"
+                
+                # Get emphasized content for list items
+                emphasized_contents = getattr(metadata, 'emphasized_text_contents', [])
+                emphasized_tags = getattr(metadata, 'emphasized_text_tags', [])
+                
+                # Format text with emphasis
+                formatted_text = element_text
+                if emphasized_contents and emphasized_tags:
+                    formatted_text = format_text_with_emphasis(formatted_text, emphasized_contents, emphasized_tags)
+                
+                # Check for links in list items
+                links = getattr(metadata, 'links', [])
+                if links:
+                    for link in links:
+                        link_text = link.get('text', '')
+                        link_url = link.get('url', '')
+                        if link_text and link_url:
+                            formatted_text = formatted_text.replace(link_text, f"<a href='{link_url}'>{link_text}</a>")
+                
+                # Print the list item with proper indentation and marker
+                print_list_item(formatted_text, list_level, marker=marker)
+            
+        else:
+            element = content
+            
+            element_type = element.element_type
+            element_text = element.text.strip()
+            metadata = element.metadata
+            
+            stats[element_type] += 1
+
+            # Skip empty elements (except images and page breaks)
+            if not element_text and element_type not in ('Image', 'PageBreak'):
+                continue
+
+            if element_type == 'Image':
+                image_counter += 1
+                process_image_element(element, image_desc_cache, image_counter)
+                continue
+            
+            if element_type == 'ListItem':
+                # This should not happen since we group them, but just in case
+                continue
+
+            elif element_type == 'PageBreak':
+                page_idx += 1
+                collector.add(f"\n[Page {page_idx} | File: {display_name}]\n")
+                continue
+
+            elif element_type == 'Title':
+                category_depth = getattr(metadata, 'category_depth', 0)
+                heading_level = min(category_depth + 1, 6)  # HTML h1-h6
+                collector.add(f"<h{heading_level}>{element_text}</h{heading_level}>")
+                continue
+            
+            elif element_type == 'Header':
+                header_type = getattr(metadata, 'header_footer_type', 'primary')
+                collector.add(f"<header class='{header_type}'>{element_text}</header>")
+                continue
+            
+            elif element_type == 'Footer':
+                footer_type = getattr(metadata, 'header_footer_type', 'primary')
+                collector.add(f"<footer class='{footer_type}'>{element_text}</footer>")
+                continue
+            
+            elif element_type == 'Table':
+                table_html = getattr(metadata, 'text_as_html', None)
+                if table_html:
+                    collector.add(table_html)
+                elif element_text:
+                    emphasized_contents = getattr(metadata, 'emphasized_text_contents', [])
+                    emphasized_tags = getattr(metadata, 'emphasized_text_tags', [])
+                    formatted_text = format_text_with_emphasis(element_text, emphasized_contents, emphasized_tags)
+                    collector.add(f"<table-text>{formatted_text}</table-text>")
+                continue
+
+            elif element_type in ['NarrativeText', 'Text', 'UncategorizedText']:
+                # Get emphasized content for regular text
+                emphasized_contents = getattr(metadata, 'emphasized_text_contents', [])
+                emphasized_tags = getattr(metadata, 'emphasized_text_tags', [])
+                # Check for links
+                links = getattr(metadata, 'links', [])
+                
+                formatted_text = element_text
+                # Apply emphasis formatting
+                if emphasized_contents and emphasized_tags:
+                    formatted_text = format_text_with_emphasis(formatted_text, emphasized_contents, emphasized_tags)
+                # Apply link formatting
+                if links:
+                    for link in links:
+                        link_text = link.get('text', '')
+                        link_url = link.get('url', '')
+                        if link_text and link_url:
+                            formatted_text = formatted_text.replace(link_text, f"<a href='{link_url}'>{link_text}</a>")
+                
+                collector.add(f"<p>{formatted_text}</p>")
+                continue
+
+            elif element_type == 'Formula':
+                collector.add(f"<formula>{element_text}</formula>")
+                continue
+            
+            else:
+                # Generic element - check for emphasis and links
+                emphasized_contents = getattr(metadata, 'emphasized_text_contents', [])
+                emphasized_tags = getattr(metadata, 'emphasized_text_tags', [])
+                links = getattr(metadata, 'links', [])
+                
+                formatted_text = element_text
+                # Apply emphasis formatting
+                if emphasized_contents and emphasized_tags:
+                    formatted_text = format_text_with_emphasis(formatted_text, emphasized_contents, emphasized_tags)
+                # Apply link formatting
+                if links:
+                    for link in links:
+                        link_text = link.get('text', '')
+                        link_url = link.get('url', '')
+                        if link_text and link_url:
+                            formatted_text = formatted_text.replace(link_text, f"<a href='{link_url}'>{link_text}</a>")
+                
+                collector.add(f"<-- {element_type.upper()} START -->")
+                collector.add(f"<div class='{element_type.lower()}'>{formatted_text}</div>")
+                collector.add(f"<-- {element_type.upper()} END -->")
+        
+    # Extract footnotes and endnotes
+    footnotes = extract_footnotes(filename)
+    endnotes = extract_endnotes(filename)
+
+    # Add footnotes
+    if footnotes:
+        collector.add("\n[Footnotes]")
+        for i, note in enumerate(footnotes, 1):
+            collector.add(f"[{i}] {note}")
+
+    # Add endnotes
+    if endnotes:
+        collector.add("\n[Endnotes]")
+        for i, note in enumerate(endnotes, 1):
+            collector.add(f"[{i}] {note}")
+
+    all_chunks = []
+    chunked_texts = collector.get_chunked_texts()
+    
+    for i, chunk in enumerate(chunked_texts, 1):
+        all_chunks.append({
+            "content": f"--- Chunk {i} ---\n{chunk}",
+            "chunkIndex": i,
+            "filename": display_name
+        })
+
+    collector.clear()
+    return all_chunks