super-document-processor 0.1.3__py3-none-any.whl

super_document_processor/document_processors/process_pdf.py

@@ -0,0 +1,715 @@
+import os
+import re
+import logging
+import threading
+import concurrent.futures
+from collections import defaultdict
+
+logger = logging.getLogger("super_document_processor.pdf")
+
+os.environ["PYMUPDF_SUGGEST_LAYOUT_ANALYZER"] = "0"
+try:
+    import pymupdf
+    fitz = pymupdf
+    try:
+        pymupdf.no_recommend_layout()
+    except Exception:
+        pass
+except ImportError:
+    import fitz
+
+try:
+    fitz.TOOLS.mupdf_display_errors(False)
+except Exception:
+    pass
+try:
+    from super_document_processor.utils.pdf_table_utils import extract_tables_from_pdf_page
+    from super_document_processor.utils.pdf_image_utils import process_image
+except ImportError:
+    from utils.pdf_table_utils import extract_tables_from_pdf_page
+    from utils.pdf_image_utils import process_image
+
+
+def cluster_spans_into_physical_lines(page, tables, links=None):
+    """
+    Extracts text lines and spans from the page, preserving:
+    1. Superscripts and intra-word span connections (e.g. '1' + 'st' -> '1st', '22.' + '5%' -> '22.5%').
+    2. Proper horizontal column segmentation.
+    3. In-place hyperlinks (both external URLs and internal slide/page jumps).
+    """
+    d = page.get_text('dict')
+    raw_lines = []
+
+    for b in d.get('blocks', []):
+        if 'lines' not in b:
+            continue
+
+        b_lines = b['lines']
+        for l in b_lines:
+            spans = l.get('spans', [])
+            if not spans:
+                continue
+
+            # Join spans within the same line while preserving native spaces
+            line_parts = []
+            cur_line_rect = None
+
+            for s in spans:
+                s_txt = s.get('text', '')
+                if not s_txt:
+                    continue
+                s_bbox = fitz.Rect(s['bbox'])
+
+                # Check if this span intersects with an external or internal link
+                if links:
+                    for lk in links:
+                        lk_rect = lk['bbox']
+                        inter_rect = s_bbox & lk_rect
+                        if inter_rect.is_valid and s_bbox.get_area() > 0:
+                            if (inter_rect.get_area() / s_bbox.get_area()) > 0.40:
+                                target = lk.get('target', '')
+                                clean_stxt = s_txt.strip()
+                                if clean_stxt and target:
+                                    if clean_stxt.startswith(('http://', 'https://', 'www.')):
+                                        s_txt = s_txt.replace(clean_stxt, f"[{target}]")
+                                    else:
+                                        s_txt = s_txt.replace(clean_stxt, f"{clean_stxt} → [{target}]")
+                                break
+
+                if cur_line_rect is None:
+                    cur_line_rect = s_bbox
+                    line_parts.append((s_bbox, s_txt))
+                else:
+                    prev_bbox, prev_txt = line_parts[-1]
+                    gap_x = s_bbox.x0 - prev_bbox.x1
+
+                    prev_has_space = prev_txt.endswith((' ', '\t'))
+                    s_has_space = s_txt.startswith((' ', '\t'))
+
+                    clean_s = s_txt.strip()
+                    # Decimal continuation: '22.' + '5%' -> '22.5%'
+                    is_decimal_continuation = bool(re.search(r'\d+\.$', prev_txt.rstrip())) and bool(re.match(r'^\d', clean_s))
+                    # Suffix check (e.g. digit + 'st', 'nd', 'rd', 'th', '%', '™', '®', '©')
+                    is_suffix = (
+                        (bool(re.search(r'\d$', prev_txt.rstrip())) or bool(re.search(r'\d+\.$', prev_txt.rstrip()))) and (
+                            clean_s.lower() in ('st', 'nd', 'rd', 'th', '%', '™', '®', '©', '°c', '°f', 'k', 'm', 'bn', 'cr', 'x') or
+                            bool(re.match(r'^(st|nd|rd|th)[,\.;:\s]', clean_s, re.IGNORECASE)) or
+                            clean_s.startswith(('%', '™', '®', '©'))
+                        )
+                    )
+
+                    if is_decimal_continuation or is_suffix:
+                        combined_txt = prev_txt.rstrip() + s_txt.lstrip()
+                    elif prev_has_space or s_has_space:
+                        combined_txt = prev_txt.rstrip() + " " + s_txt.lstrip()
+                    elif gap_x > 2.0:
+                        combined_txt = prev_txt + " " + s_txt
+                    else:
+                        combined_txt = prev_txt + s_txt
+
+                    combined_bbox = prev_bbox | s_bbox
+                    line_parts[-1] = (combined_bbox, combined_txt)
+                    cur_line_rect = cur_line_rect | s_bbox
+
+            if line_parts:
+                seg_text = line_parts[0][1].strip()
+                if not seg_text:
+                    continue
+
+                # Table containment filter:
+                # If more than 60% of the line is covered by a table, discard it (it belongs inside the table).
+                # If at least 40% of the line is visible outside the table (e.g. headers, captions), preserve and show it.
+                line_area = cur_line_rect.get_area()
+                if line_area > 0:
+                    is_inside_table = any(
+                        (cur_line_rect & t['bbox']).is_valid and
+                        ((cur_line_rect & t['bbox']).get_area() / line_area) > 0.60
+                        for t in tables
+                    )
+                    if is_inside_table:
+                        continue
+
+                # Clean up spaced abbreviations with periods on both letters (e.g. 'e. g.' -> 'e.g.') or 'w.r.t'
+                seg_text = re.sub(r'\b([a-zA-Z]\.)\s+([a-zA-Z]\.)', r'\1\2', seg_text)
+                seg_text = re.sub(r'\b(w\.)\s*(r\.)?\s*(t\b)', lambda m: 'W.R.T' if m.group(0).isupper() else 'w.r.t', seg_text, flags=re.IGNORECASE)
+                seg_text = re.sub(r'(\d+)\s*\.\s*(\d+)', r'\1.\2', seg_text)
+                seg_text = re.sub(r'(\d+)\s+%', r'\1%', seg_text)
+                raw_lines.append({
+                    'bbox': cur_line_rect,
+                    'text': seg_text,
+                    'y_mid': (cur_line_rect.y0 + cur_line_rect.y1) / 2.0
+                })
+
+    if not raw_lines:
+        return []
+
+    # Sort lines by vertical position
+    raw_lines.sort(key=lambda l: (l['y_mid'], l['bbox'].x0))
+
+    # Cluster horizontally aligned lines that share the same baseline into segments
+    clustered_lines = []
+    cur_line_group = [raw_lines[0]]
+    cur_y = raw_lines[0]['y_mid']
+
+    for l in raw_lines[1:]:
+        if abs(l['y_mid'] - cur_y) <= 4.0:
+            cur_line_group.append(l)
+        else:
+            clustered_lines.append(cur_line_group)
+            cur_line_group = [l]
+            cur_y = l['y_mid']
+    if cur_line_group:
+        clustered_lines.append(cur_line_group)
+
+    line_segments = []
+    for line_group in clustered_lines:
+        line_group.sort(key=lambda item: item['bbox'].x0)
+
+        cur_seg = [line_group[0]]
+        cur_right = line_group[0]['bbox'].x1
+
+        for item in line_group[1:]:
+            lead_txt = cur_seg[0]['text'].strip() if cur_seg else ""
+            is_bullet_leader = len(cur_seg) == 1 and bool(
+                re.match(r'^([•*○◦▪▫■□◆◇◈❖➢➤➔➜►▶→⇒»›✓✔★✦✧\-\u2013\u2014\uf0b7\u27a2\u25cf\uf06f\uf0a7\uf0a8\uf0d8\uf0fc\uf075]|o\s+|\d{1,3}[.)]|\([a-zA-Z0-9]{1,3}\)|\[\d{1,3}\]|(?!(?:e\.g\.|i\.e\.|etc\.|vs\.)\b)[a-zA-Z][.)]|(?:[ivxIVX]+)[.)])$', lead_txt)
+            )
+            split_gap = 35.0 if is_bullet_leader else 20.0
+
+            if item['bbox'].x0 - cur_right >= split_gap:
+                seg_rect = cur_seg[0]['bbox']
+                for cs in cur_seg[1:]:
+                    seg_rect = seg_rect | cs['bbox']
+                seg_text = " ".join(cs['text'] for cs in cur_seg)
+                line_segments.append({'bbox': seg_rect, 'text': seg_text})
+                cur_seg = [item]
+                cur_right = item['bbox'].x1
+            else:
+                cur_seg.append(item)
+                cur_right = max(cur_right, item['bbox'].x1)
+
+        if cur_seg:
+            seg_rect = cur_seg[0]['bbox']
+            for cs in cur_seg[1:]:
+                seg_rect = seg_rect | cs['bbox']
+            seg_text = " ".join(cs['text'] for cs in cur_seg)
+            line_segments.append({'bbox': seg_rect, 'text': seg_text})
+
+    return line_segments
+
+
+def group_line_segments_into_paragraphs(line_segments, page_height=None):
+    """
+    Groups vertically stacked line segments in the same column into paragraph blocks,
+    preserving sub-bullet nesting and indentation.
+    """
+    if not line_segments:
+        return []
+
+    line_segments.sort(key=lambda l: (l['bbox'].y0, l['bbox'].x0))
+
+    PUA_GLYPH_MAP = {
+        '\uf0b7': '•',
+        '\uf06f': 'o',
+        '\uf0a7': '▪',
+        '\uf0a8': '▫',
+        '\uf0d8': '➢',
+        '\uf0fc': '✔',
+        '\uf075': '◆',
+        '\u25cf': '•',
+    }
+
+    # Universal bullet pattern for any list marker (symbols, letters, numbers, roman numerals)
+    # Strictly requires whitespace after hyphens/numbers/letters and excludes negative numbers, decimals, metrics, and names
+    ALL_BULLETS_PATTERN = (
+        r'^(?:([•*➢➤➔➜►▶→⇒»›✓✔★✦✧\u25cf\u25a0\u25a1\u25aa\u25ab\u25cb\u25e6\u25c6\u25c7\u25c8\u2756\u2219'
+        r'\uf0b7\u27a2\uf0d8\uf0fc\uf06f\uf0a7\uf0a8\uf075]|[-–—]\s+|o\s+|\d{1,3}[.)]\s+(?!\d|%|k\b|m\b|cr\b|bn\b|lakh\b|x\b|\$)|'
+        r'\([a-zA-Z0-9]{1,3}\)\s+|\[\d{1,3}\]\s+|'
+        r'(?!(?:e\.g\.|i\.e\.|etc\.|vs\.|w\.r\.t\.)\b)[a-z][.)]\s+|[A-Z][.)]\s+(?=[A-Z0-9])|(?:[ivxIVX]+)[.)]\s+)\s*)(.*)$'
+    )
+
+    processed = []
+    for l in line_segments:
+        txt = l['text'].strip()
+        m_bullet = re.match(ALL_BULLETS_PATTERN, txt)
+
+        if m_bullet:
+            raw_glyph = m_bullet.group(1).strip()
+            glyph = PUA_GLYPH_MAP.get(raw_glyph, raw_glyph)
+            rest = m_bullet.group(2).strip()
+            processed.append({
+                'bbox': l['bbox'],
+                'glyph': glyph,
+                'rest': rest,
+                'is_bullet': True,
+                'raw': txt
+            })
+        else:
+            processed.append({
+                'bbox': l['bbox'],
+                'glyph': '',
+                'rest': txt,
+                'is_bullet': False,
+                'raw': txt
+            })
+
+    blocks = []
+    for item in processed:
+        is_bullet = item['is_bullet']
+        raw_txt = item['raw']
+        is_in_margin = (page_height is None) or (item['bbox'].y0 >= page_height * 0.85 or item['bbox'].y1 <= page_height * 0.10)
+        is_page_num = is_in_margin and bool(re.match(r'^(?:\d{1,4}|page\s+\d+(?:\s+of\s+\d+)?|slide\s+\d+(?:\s+of\s+\d+)?)$', raw_txt.strip(), re.IGNORECASE))
+
+        if is_page_num:
+            blocks.append({
+                'bbox': item['bbox'],
+                'base_x0': item['bbox'].x0,
+                'lines': [raw_txt.strip()],
+                'type': 'page_number'
+            })
+            continue
+
+        if not blocks:
+            if is_bullet:
+                glyph, rest = item['glyph'], item['rest']
+                line_str = f"{glyph} {rest}" if rest else glyph
+                bullet_x0 = item['bbox'].x0
+            else:
+                line_str = raw_txt
+                bullet_x0 = None
+
+            blocks.append({
+                'bbox': item['bbox'],
+                'base_x0': item['bbox'].x0,
+                'bullet_x0': bullet_x0,
+                'lines': [line_str],
+                'type': 'text'
+            })
+            continue
+
+        attached = False
+        for b in reversed(blocks):
+            if b.get('type') == 'page_number':
+                continue
+            y_gap = item['bbox'].y0 - b['bbox'].y1
+            x_overlap = min(b['bbox'].x1, item['bbox'].x1) - max(b['bbox'].x0, item['bbox'].x0)
+            starts_within_column = item['bbox'].x0 >= (b['bbox'].x0 - 8.0) and (x_overlap > 0 or item['bbox'].x0 <= b['bbox'].x0 + 50.0)
+            is_left_aligned = abs(b['bbox'].x0 - item['bbox'].x0) <= 25.0
+
+            if is_bullet:
+                # Coordinate-driven bullet nesting: if within column and directly below
+                if -2.0 <= y_gap <= 12.0 and starts_within_column:
+                    if b.get('bullet_x0') is None:
+                        b['bullet_x0'] = item['bbox'].x0
+
+                    diff = item['bbox'].x0 - b['bullet_x0']
+                    if diff < -5.0:
+                        b['bullet_x0'] = item['bbox'].x0
+                        indent_str = ""
+                    elif diff >= 18.0:
+                        indent_str = "        "
+                    elif diff >= 6.0:
+                        indent_str = "    "
+                    else:
+                        indent_str = ""
+
+                    glyph, rest = item['glyph'], item['rest']
+                    formatted_line = f"{indent_str}{glyph} {rest}" if rest else f"{indent_str}{glyph}"
+
+                    b['lines'].append(formatted_line)
+                    b['bbox'] = b['bbox'] | item['bbox']
+                    attached = True
+                    break
+
+            else:
+                # Continuation of contiguous lines within the exact same paragraph (word wraps)
+                max_wrap_gap = 6.0
+                if -2.0 <= y_gap <= max_wrap_gap and starts_within_column and (x_overlap > 0 or is_left_aligned):
+                    b['lines'].append(raw_txt)
+                    b['bbox'] = b['bbox'] | item['bbox']
+                    attached = True
+                    break
+
+        if not attached:
+            if is_bullet:
+                glyph, rest = item['glyph'], item['rest']
+                line_str = f"{glyph} {rest}" if rest else glyph
+                bullet_x0 = item['bbox'].x0
+            else:
+                line_str = raw_txt
+                bullet_x0 = None
+
+            blocks.append({
+                'bbox': item['bbox'],
+                'base_x0': item['bbox'].x0,
+                'bullet_x0': bullet_x0,
+                'lines': [line_str],
+                'type': 'text'
+            })
+
+    for b in blocks:
+        reflowed = []
+        for line in b['lines']:
+            stripped_line = line.strip()
+            is_new_bullet = bool(re.match(ALL_BULLETS_PATTERN, stripped_line))
+            prev_is_heading = reflowed and reflowed[-1].rstrip().endswith(':')
+            is_sub_bullet = line.startswith(('  ', '\t')) and is_new_bullet
+
+            if not reflowed or is_new_bullet or prev_is_heading or is_sub_bullet:
+                reflowed.append(line)
+            else:
+                prev = reflowed[-1]
+                if prev.endswith('-'):
+                    reflowed[-1] = prev[:-1] + stripped_line
+                else:
+                    reflowed[-1] = f"{prev} {stripped_line}"
+
+        b['text'] = "\n".join(reflowed)
+        b['raw_text'] = b['text']
+
+    return blocks
+
+
+def order_layout_elements(elements, min_gutter_x=12.0, min_band_gap_y=16.0):
+    """
+    Robust Hierarchical Layout Ordering (Recursive XY-Cut):
+    1. Separate full-width spanning headers at top and footers/notes at bottom.
+    2. Split distinct vertical columns (Left -> Right) if a gutter exists.
+    3. Split distinct horizontal bands (Rows) if a horizontal valley exists.
+    4. Order top-to-bottom within each final cell.
+    """
+    if len(elements) <= 1:
+        return elements
+
+    min_x = min(e['bbox'].x0 for e in elements)
+    max_x = max(e['bbox'].x1 for e in elements)
+    total_span_x = max_x - min_x
+
+    sorted_by_y = sorted(elements, key=lambda e: (e['bbox'].y0, e['bbox'].x0))
+
+    # 1a. Check for spanning Header at the top
+    top_elem = sorted_by_y[0]
+    top_elem_width = top_elem['bbox'].width
+    if top_elem_width >= 0.4 * total_span_x and len(sorted_by_y) > 1:
+        y_gap = sorted_by_y[1]['bbox'].y0 - top_elem['bbox'].y1
+        if y_gap >= 8.0:
+            rest = [e for e in elements if e is not top_elem]
+            return [top_elem] + order_layout_elements(rest, min_gutter_x, min_band_gap_y)
+
+    # 1b. Check for spanning Footer / bottom footnote
+    bottom_elem = sorted_by_y[-1]
+    bottom_elem_width = bottom_elem['bbox'].width
+    if bottom_elem_width >= 0.4 * total_span_x and len(sorted_by_y) > 1:
+        other_max_y1 = max(e['bbox'].y1 for e in sorted_by_y[:-1])
+        y_gap = bottom_elem['bbox'].y0 - other_max_y1
+        if y_gap >= 8.0:
+            rest = [e for e in elements if e is not bottom_elem]
+            return order_layout_elements(rest, min_gutter_x, min_band_gap_y) + [bottom_elem]
+
+    # 2. Check for vertical column split (gutters across the current region)
+    sorted_by_x = sorted(elements, key=lambda e: (e['bbox'].x0, e['bbox'].y0))
+    columns = []
+    current_col = [sorted_by_x[0]]
+    current_right = sorted_by_x[0]['bbox'].x1
+
+    for e in sorted_by_x[1:]:
+        if e['bbox'].x0 >= current_right + min_gutter_x:
+            columns.append(current_col)
+            current_col = [e]
+            current_right = e['bbox'].x1
+        else:
+            current_col.append(e)
+            current_right = max(current_right, e['bbox'].x1)
+    if current_col:
+        columns.append(current_col)
+
+    if len(columns) > 1:
+        ordered = []
+        for col in columns:
+            ordered.extend(order_layout_elements(col, min_gutter_x, min_band_gap_y))
+        return ordered
+
+    # 3. Check for horizontal band split (distinct rows across all columns)
+    y_cuts = []
+    current_bottom = sorted_by_y[0]['bbox'].y1
+    current_group = [sorted_by_y[0]]
+
+    for e in sorted_by_y[1:]:
+        if e['bbox'].y0 >= current_bottom + min_band_gap_y:
+            y_cuts.append(current_group)
+            current_group = [e]
+            current_bottom = e['bbox'].y1
+        else:
+            current_group.append(e)
+            current_bottom = max(current_bottom, e['bbox'].y1)
+    if current_group:
+        y_cuts.append(current_group)
+
+    if len(y_cuts) > 1:
+        result = []
+        for group in y_cuts:
+            result.extend(order_layout_elements(group, min_gutter_x, min_band_gap_y))
+        return result
+
+    return sorted(elements, key=lambda e: (e['bbox'].y0, e['bbox'].x0))
+
+
+def extract_links_from_page(page):
+    """Extract links from a page and return them with bounding boxes for positioning."""
+    links = []
+    try:
+        link_list = page.get_links()
+        for link in link_list:
+            rect = fitz.Rect(link['from'])
+            target = None
+
+            # 1. External Web URL
+            if 'uri' in link and link['uri']:
+                target = link['uri']
+            # 2. Internal Page / Slide Jump
+            elif 'page' in link and link['page'] is not None and link['page'] >= 0:
+                target_page = link['page'] + 1
+                target = f"Slide {target_page}"
+
+            if not target:
+                continue
+
+            link_text = page.get_textbox(rect).strip()
+            if not link_text:
+                expanded_rect = rect + (-5, -5, 5, 5)
+                link_text = page.get_textbox(expanded_rect).strip()
+
+            if not link_text:
+                link_text = "Link"
+
+            if link_text.startswith(('http://', 'https://', 'www.')):
+                formatted_link = f"[{target}]"
+            else:
+                formatted_link = f"{link_text} → [{target}]"
+
+            links.append({
+                'bbox': rect,
+                'target': target,
+                'link_text': link_text,
+                'text': formatted_link,
+                'raw_text': formatted_link,
+                'type': 'link'
+            })
+
+    except Exception as e:
+        logger.warning(f"Could not extract links: {e}")
+
+    return links
+
+
+def get_drawing_boxes(page, tables):
+    """
+    Extracts visual container/card boxes from vector drawings on the page.
+    Filters out full-page backgrounds, tiny decorative lines, and table bounding boxes.
+    """
+    page_rect = page.rect
+    raw_boxes = []
+    for d in page.get_drawings():
+        r = fitz.Rect(d['rect'])
+        if r.width >= page_rect.width - 20 and r.height >= page_rect.height - 20:
+            continue
+        if r.width >= 50 and r.height >= 25:
+            # Exclude boxes that intersect heavily with detected tables
+            if any((r & t['bbox']).is_valid and (r & t['bbox']).get_area() > 0.4 * min(r.get_area(), t['bbox'].get_area()) for t in tables):
+                continue
+            raw_boxes.append(r)
+
+    # Deduplicate nearly identical boxes (e.g. stroke + fill pairs for the same card)
+    unique_boxes = []
+    for b in raw_boxes:
+        if not any(abs(ub.x0 - b.x0) < 3 and abs(ub.y0 - b.y0) < 3 and abs(ub.x1 - b.x1) < 3 and abs(ub.y1 - b.y1) < 3 for ub in unique_boxes):
+            unique_boxes.append(b)
+
+    # If an outer container encloses multiple smaller card boxes, keep the individual card boxes
+    clean_boxes = []
+    for b in unique_boxes:
+        children = [other for other in unique_boxes if other != b and b.contains(other) and other.get_area() < 0.8 * b.get_area()]
+        if len(children) >= 2:
+            continue
+        clean_boxes.append(b)
+
+    return clean_boxes
+
+
+def apply_box_decorations(ordered_elements, drawing_boxes):
+    """
+    Identifies elements enclosed inside drawing boxes / callouts and frames them with '---'
+    without creating consecutive duplicate separators.
+    """
+    if not drawing_boxes:
+        return ordered_elements
+
+    decorated = []
+    current_box_elements = []
+    active_box = None
+
+    for el in ordered_elements:
+        el_rect = el['bbox']
+        matching_box = None
+        # Standalone page numbers belong to the global footer and sit outside card containers
+        if el.get('type') != 'page_number':
+            for box in drawing_boxes:
+                el_center_x = (el_rect.x0 + el_rect.x1) / 2.0
+                el_center_y = (el_rect.y0 + el_rect.y1) / 2.0
+                if box.x0 - 5 <= el_center_x <= box.x1 + 5 and box.y0 - 5 <= el_center_y <= box.y1 + 5:
+                    matching_box = box
+                    break
+
+        if matching_box != active_box:
+            if active_box is not None and current_box_elements:
+                decorated.append({'type': 'separator', 'text': '---', 'bbox': active_box})
+                decorated.extend(current_box_elements)
+                decorated.append({'type': 'separator', 'text': '---', 'bbox': active_box})
+                current_box_elements = []
+            active_box = matching_box
+
+        if active_box is not None:
+            current_box_elements.append(el)
+        else:
+            decorated.append(el)
+
+    if active_box is not None and current_box_elements:
+        decorated.append({'type': 'separator', 'text': '---', 'bbox': active_box})
+        decorated.extend(current_box_elements)
+        decorated.append({'type': 'separator', 'text': '---', 'bbox': active_box})
+
+    # Deduplicate consecutive '---' separators
+    clean_decorated = []
+    for e in decorated:
+        if e.get('text', '').strip() == '---':
+            if clean_decorated and clean_decorated[-1].get('text', '').strip() == '---':
+                continue
+        clean_decorated.append(e)
+
+    return clean_decorated
+
+
+def format_page_elements(page, tables, links, image_pieces=None):
+    """
+    Extracts text spans, tables, links, and image descriptions and arranges them in true 2D reading order.
+    """
+    page_height = page.rect.height if page else None
+    drawing_boxes = get_drawing_boxes(page, tables)
+    segs = cluster_spans_into_physical_lines(page, tables, links=links)
+    text_blocks = group_line_segments_into_paragraphs(segs, page_height=page_height)
+
+    elements = list(text_blocks)
+
+    for table in tables:
+        elements.append({
+            'bbox': table["bbox"],
+            'text': table["text"],
+            'raw_text': table["text"],
+            'type': 'table'
+        })
+
+    # For any standalone links that were NOT covered by existing text spans, append them
+    existing_boxes = [e['bbox'] for e in elements]
+    for link in links:
+        is_covered = any(
+            (link['bbox'] & eb).is_valid and (link['bbox'] & eb).get_area() > 0.40 * link['bbox'].get_area()
+            for eb in existing_boxes
+        )
+        if not is_covered:
+            elements.append(link)
+
+    if image_pieces:
+        for piece in image_pieces:
+            if isinstance(piece, dict) and 'bbox' in piece:
+                elements.append(piece)
+            elif isinstance(piece, tuple) and len(piece) == 2:
+                y0_pos, img_txt = piece
+                width = page.rect.width if page else 600
+                img_bbox = fitz.Rect(0, y0_pos, width, y0_pos + 10)
+                elements.append({'bbox': img_bbox, 'text': img_txt, 'raw_text': img_txt, 'type': 'image'})
+
+    ordered = order_layout_elements(elements)
+    decorated = apply_box_decorations(ordered, drawing_boxes)
+    return "\n\n".join(e['text'] for e in decorated if e.get('text', '').strip()).strip()
+
+
+def process_pdf_file(filename, original_filename=None, process_images=None, model=None):
+    display_name = os.path.basename(original_filename) if original_filename else os.path.basename(filename)
+    should_process_images = bool(process_images)
+
+    if not should_process_images:
+        chunks = []
+        with fitz.open(filename) as doc:
+            for i in range(len(doc)):
+                page = doc.load_page(i)
+                tables = extract_tables_from_pdf_page([], page, i + 1)
+                links = extract_links_from_page(page)
+                formatted_body = format_page_elements(page, tables, links)
+                header = f"[Page {i + 1} | File: {display_name}]"
+                page_content = header + "\n\n" + formatted_body if formatted_body else header
+                if page_content:
+                    chunks.append({
+                        "content": page_content,
+                        "chunkIndex": i + 1,
+                        "filename": display_name
+                    })
+                else:
+                    logger.debug(f"No content extracted for page {i+1}")
+        return chunks
+
+    # Multithreaded Vision Pipeline (when process_images is True)
+    all_chunks = []
+    lock = threading.Lock()
+    seen_image_hashes = {}
+    image_data_store = {}
+    image_page_map = defaultdict(list)
+
+    precomputed_tables = []
+    with fitz.open(filename) as doc:
+        for i in range(len(doc)):
+            page = doc.load_page(i)
+            tables = extract_tables_from_pdf_page([], page, i + 1)
+            precomputed_tables.append(tables)
+
+    def process_page(i):
+        with fitz.open(filename) as doc:
+            page = doc.load_page(i)
+            logger.debug(f"Processing Page {i + 1}")
+
+            image_processing_texts, need_full_page_image_processing = process_image(
+                seen_image_hashes, image_data_store, image_page_map, page, i, doc, lock=lock, model=model
+            )
+
+            if need_full_page_image_processing:
+                header = f"[Page {i + 1} | File: {display_name}]"
+                page_content = header + "\n\n" + "\n\n".join(piece[1] for piece in image_processing_texts).strip()
+                if page_content:
+                    return {
+                        "content": page_content,
+                        "chunkIndex": i + 1,
+                        "filename": display_name
+                    }
+                else:
+                    logger.debug(f"No content extracted for page {i+1}")
+                    return None
+
+            tables = precomputed_tables[i]
+            links = extract_links_from_page(page)
+            formatted_body = format_page_elements(page, tables, links, image_pieces=image_processing_texts)
+            header = f"[Page {i + 1} | File: {display_name}]"
+            page_content = header + "\n\n" + formatted_body if formatted_body else header
+            if page_content:
+                return {
+                    "content": page_content,
+                    "chunkIndex": i + 1,
+                    "filename": display_name
+                }
+            else:
+                logger.debug(f"No content extracted for page {i+1}")
+                return None
+
+    workers = int(os.environ.get("MAX_WORKERS", 3))
+    with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
+        results = list(executor.map(process_page, range(len(precomputed_tables))))
+
+    all_chunks = [chunk for chunk in results if chunk]
+    return all_chunks