super-document-processor 0.1.3__py3-none-any.whl

super_document_processor/document_processors/process_plain_text.py

@@ -0,0 +1,25 @@
+import os
+try:
+    from super_document_processor.utils.chunker import collector
+except ImportError:
+    from utils.chunker import collector
+
+def process_plain_file(filename, original_filename=None):
+    display_name = os.path.basename(original_filename) if original_filename else os.path.basename(filename)
+    with open(filename, "r", encoding="utf-8", errors="replace") as f:
+        md_text = f.read()
+
+    # Add the raw content of the file
+    collector.add(md_text)
+
+    all_chunks = []
+    chunked_texts = collector.get_chunked_texts()
+    for i, chunk in enumerate(chunked_texts, 1):
+        all_chunks.append({
+            "content": f"--- Chunk {i} ---\n{chunk}",
+            "chunkIndex": i,
+            "filename": display_name
+        })
+
+    collector.clear()
+    return all_chunks

super_document_processor/document_processors/process_pptx.py

@@ -0,0 +1,367 @@
+import xml.etree.ElementTree as ET
+import os
+import hashlib
+import base64
+import io
+import re
+import logging
+import gc
+import zipfile
+import threading
+import concurrent.futures
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+from pptx import Presentation
+from pptx.enum.shapes import MSO_SHAPE_TYPE
+from collections import defaultdict
+
+logger = logging.getLogger("super_document_processor.pptx")
+slides = None
+drawing = None
+
+def _ensure_aspose():
+    global slides, drawing
+    if slides is None or drawing is None:
+        import aspose.slides as _slides
+        import aspose.pydrawing as _drawing
+        slides = _slides
+        drawing = _drawing                            
+try:
+    from super_document_processor.utils.vlm import generate_image_description
+    from super_document_processor.utils.pptx_components_extract import (
+        format_table_markdown,
+        extract_text_from_shape,
+        is_slide_header_shape,
+        extract_smartart_texts_per_slide,
+        extract_chart_data_from_pptx,
+        extract_embedded_excel_tables,
+        extract_emf_content_per_slide,
+        extract_emf_tables_and_text,
+        get_spatial_shape_reading_order,
+        get_pptx_card_boxes,
+        apply_pptx_box_decorations
+    )
+    from super_document_processor.utils.aspose_slide_renderer import render_slide_to_png_bytes
+except ImportError:
+    from utils.vlm import generate_image_description
+    from utils.pptx_components_extract import (
+        format_table_markdown,
+        extract_text_from_shape,
+        is_slide_header_shape,
+        extract_smartart_texts_per_slide,
+        extract_chart_data_from_pptx,
+        extract_embedded_excel_tables,
+        extract_emf_content_per_slide,
+        extract_emf_tables_and_text,
+        get_spatial_shape_reading_order,
+        get_pptx_card_boxes,
+        apply_pptx_box_decorations
+    )
+    from utils.aspose_slide_renderer import render_slide_to_png_bytes
+
+def clean_vlm_response(text: str) -> str:
+    """Removes Aspose evaluation watermarks from the text."""
+    if not text:
+        return text
+    patterns = [
+        r"Evaluation only\. Created with Aspose\..*?Copyright.*?\.",
+        r"Created with Aspose\..*?Copyright.*",
+        r"Aspose\.Slides for Python via \.NET.*",
+        r"Aspose\.Slides for \.NET.*",
+        r"Aspose\.Cells for \.NET.*",
+        r"Evaluation only\."
+    ]
+    cleaned = text
+    for pattern in patterns:
+        cleaned = re.sub(pattern, "", cleaned, flags=re.IGNORECASE)
+    
+    cleaned = "\n".join([line for line in cleaned.splitlines() if line.strip()])
+    return cleaned.strip()
+
+def get_slide_image_mapping(pptx_path):
+    prs = Presentation(pptx_path)
+    slide_partnames = [slide.part.partname.split('/')[-1] for slide in prs.slides]
+    
+    slide_image_map = defaultdict(list)
+    with zipfile.ZipFile(pptx_path, 'r') as pptx:
+        for i, partname in enumerate(slide_partnames, start=1):
+            rels_path = f"ppt/slides/_rels/{partname}.rels"
+            slide_key = f"Slide {i}"
+            if rels_path not in pptx.namelist():
+                continue
+            rels_data = pptx.read(rels_path)
+            rels_root = ET.fromstring(rels_data)
+            ns = {"r": "http://schemas.openxmlformats.org/package/2006/relationships"}
+            for rel in rels_root.findall("r:Relationship", ns):
+                rel_type = rel.attrib.get("Type", "")
+                target = rel.attrib.get("Target", "").split("/")[-1]
+                ext = os.path.splitext(target)[1].lower()
+                
+                if "relationships/image" in rel_type or ext in ['.png', '.jpg', '.jpeg', '.emf', '.wmf', '.svg', '.webp']:
+                    slide_image_map[slide_key].append(target)
+    return slide_image_map
+
+@dataclass
+class SlideContext:
+    page_num: int
+    slide: Any
+    aspose_ppt: Any
+    slide_image_map: Dict[str, List[str]]
+    seen_image_hashes: Dict[str, str]
+    image_data_store: Dict[str, str]
+    image_page_map: Dict[str, List[int]]
+    filename: str
+    lock: threading.Lock
+    display_name: str
+    should_process_images: bool = False
+    model: Optional[str] = None
+
+def process_pptx_slide(ctx: SlideContext):
+    slide_key = f"Slide {ctx.page_num}"
+    mapped_images = ctx.slide_image_map.get(slide_key, [])
+
+    # --- IMAGE HANDLING (when vision processing is requested) ---
+    image_descriptions = []
+    if ctx.should_process_images and mapped_images:
+        if len(mapped_images) > 5:
+            try:
+                aspose_slide = ctx.aspose_ppt.slides[ctx.page_num - 1]
+                image_bytes = render_slide_to_png_bytes(aspose_slide, scale=2.0)
+                raw_description = generate_image_description(image_bytes, model=ctx.model)
+                description = clean_vlm_response(raw_description)
+                if description:
+                    image_descriptions.append(f"[Full-slide image description]:\n{description}")
+            except Exception as aspose_err:
+                if isinstance(aspose_err, ValueError):
+                    raise
+                logger.warning(f"Aspose full slide render error on slide {ctx.page_num}: {aspose_err}")
+        else:
+            try:
+                aspose_slide = ctx.aspose_ppt.slides[ctx.page_num - 1]
+            except Exception as e:
+                aspose_slide = None
+                logger.warning(f"Could not load aspose_slide for slide {ctx.page_num}: {e}")
+        
+            if aspose_slide is not None:
+                renderable_shapes = []
+                for shape_index, shape in enumerate(aspose_slide.shapes):
+                    if isinstance(shape, (slides.PictureFrame, slides.OleObjectFrame)):
+                        renderable_shapes.append((shape_index, shape))
+
+                def process_rendered_shape(shape_tuple):
+                    idx, shape = shape_tuple
+                    try:
+                        shape_img = shape.get_image()
+                        with io.BytesIO() as temp_stream:
+                            shape_img.save(temp_stream, slides.ImageFormat.PNG)
+                            temp_stream.seek(0)
+                            
+                            src_img = drawing.Image.from_stream(temp_stream)
+                            bitmap = drawing.Bitmap(src_img.width, src_img.height)
+                            graphics = drawing.Graphics.from_image(bitmap)
+                            graphics.clear(drawing.Color.white)
+                            graphics.draw_image(src_img, 0, 0, src_img.width, src_img.height)
+                            
+                            img_stream = io.BytesIO()
+                            bitmap.save(img_stream, drawing.imaging.ImageFormat.png)
+                            image_bytes = img_stream.getvalue()
+                        
+                        if not image_bytes or len(image_bytes) < 10:
+                            return (idx, None)
+
+                        img_hash = hashlib.sha256(image_bytes).hexdigest()
+                        
+                        with ctx.lock:
+                            if ctx.page_num not in ctx.image_page_map[img_hash]:
+                                ctx.image_page_map[img_hash].append(ctx.page_num)
+                            if img_hash not in ctx.image_data_store:
+                                ctx.image_data_store[img_hash] = base64.b64encode(image_bytes).decode("utf-8")
+                            description = ctx.seen_image_hashes.get(img_hash)
+
+                        if description is None:
+                            try:
+                                raw_description = generate_image_description(image_bytes, model=ctx.model)
+                                description = clean_vlm_response(raw_description)
+                            except Exception as e:
+                                if isinstance(e, ValueError):
+                                    raise
+                                description = f"[Error generating description: {e}]"
+                            with ctx.lock:
+                                ctx.seen_image_hashes[img_hash] = description
+                        
+                        return (idx, description)
+                    except Exception as e:
+                        if isinstance(e, ValueError):
+                            raise
+                        logger.warning(f"Error rendering shape {idx} on slide {ctx.page_num}: {e}")
+                        return (idx, None)
+
+                shape_workers = int(os.environ.get("MAX_WORKERS", 3))
+                with concurrent.futures.ThreadPoolExecutor(max_workers=shape_workers) as img_executor:
+                    results = list(img_executor.map(process_rendered_shape, renderable_shapes))
+                
+                for idx, desc in sorted(results, key=lambda x: x[0]):
+                    if desc:
+                        image_descriptions.append(desc)
+
+    # --- EXTRACT VISUAL CARD CONTAINERS ---
+    slide_h = getattr(ctx.slide, 'height', 6858000) or 6858000
+    slide_w = getattr(ctx.slide, 'width', 12192000) or 12192000
+    card_boxes = get_pptx_card_boxes(ctx.slide)
+
+    # --- 2D SPATIAL READING ORDER & IN-PLACE SHAPE RENDERING ---
+    ordered_shapes = get_spatial_shape_reading_order(ctx.slide.shapes, slide_h, slide_w)
+    raw_elements = []
+    extracted_shape_emfs = set()
+    
+    for shape in ordered_shapes:
+        shape_rect = {
+            'left': int(getattr(shape, 'left', 0) or 0),
+            'top': int(getattr(shape, 'top', 0) or 0),
+            'width': int(getattr(shape, 'width', 0) or 0),
+            'height': int(getattr(shape, 'height', 0) or 0),
+        }
+        is_hdr = is_slide_header_shape(shape, slide_h, slide_w)
+        # Tables rendered in-place as markdown tables (not duplicated)
+        if getattr(shape, 'has_table', False) or getattr(shape, 'shape_type', None) == MSO_SHAPE_TYPE.TABLE:
+            table_md = format_table_markdown(shape.table)
+            if table_md:
+                raw_elements.append({**shape_rect, 'text': table_md, 'type': 'table'})
+        elif getattr(shape, 'has_text_frame', False):
+            extracted_text = extract_text_from_shape(shape, is_header=is_hdr)
+            if extracted_text:
+                raw_elements.append({**shape_rect, 'text': extracted_text, 'type': 'header' if is_hdr else 'text'})
+        elif getattr(shape, 'shape_type', None) == MSO_SHAPE_TYPE.GROUP or hasattr(shape, 'shapes'):
+            extracted_text = extract_text_from_shape(shape, is_header=is_hdr)
+            if extracted_text:
+                raw_elements.append({**shape_rect, 'text': extracted_text, 'type': 'header' if is_hdr else 'text'})
+
+        # Check for embedded EMF/WMF vector drawings/tables inside shape to preserve spatial order
+        shape_el = getattr(shape, '_element', None)
+        if shape_el is not None and hasattr(ctx.slide, 'part') and hasattr(ctx.slide.part, 'rels'):
+            for blip in shape_el.xpath('.//a:blip'):
+                embed = blip.get('{http://schemas.openxmlformats.org/officeDocument/2006/relationships}embed')
+                if embed and embed in ctx.slide.part.rels:
+                    rel = ctx.slide.part.rels[embed]
+                    target_ref = getattr(rel, 'target_ref', '') or ''
+                    target_lower = target_ref.lower()
+                    if target_lower.endswith('.emf') or target_lower.endswith('.wmf'):
+                        media_clean = 'ppt/' + target_ref.replace('../', '') if not target_ref.startswith('ppt/') else target_ref
+                        extracted_shape_emfs.add(media_clean)
+                        try:
+                            emf_blob = rel.target_part.blob
+                            emf_text = extract_emf_tables_and_text(emf_blob)
+                            if emf_text and emf_text.strip():
+                                raw_elements.append({**shape_rect, 'text': emf_text.strip(), 'type': 'emf_table'})
+                        except Exception:
+                            pass
+
+    # --- SmartArt / Diagrams ---
+    slide_diagrams_text = extract_smartart_texts_per_slide(ctx.filename, ctx.page_num)
+    if ctx.page_num in slide_diagrams_text:
+        smartart_items = slide_diagrams_text[ctx.page_num]
+        if smartart_items:
+            formatted_smartart = "\n".join(f"• {item}" if not item.startswith(('•', '-', '*', '#')) else item for item in smartart_items)
+            raw_elements.append({'text': formatted_smartart, 'type': 'smartart'})
+
+    # --- Charts ---
+    chart_data = extract_chart_data_from_pptx(ctx.filename, ctx.page_num)
+    if chart_data and ctx.page_num in chart_data:
+        for chart_md in chart_data[ctx.page_num]:
+            if chart_md:
+                raw_elements.append({'text': chart_md, 'type': 'chart'})
+
+    # --- EMF Vector Graphics Tables / Text (fallback for any slide EMFs not attached to shapes) ---
+    emf_content = extract_emf_content_per_slide(ctx.filename, ctx.page_num)
+    if emf_content and ctx.page_num in emf_content:
+        for emf_md in emf_content[ctx.page_num]:
+            if emf_md and not any(el.get('type') == 'emf_table' and el.get('text', '').strip() == emf_md.strip() for el in raw_elements):
+                raw_elements.append({'text': emf_md, 'type': 'emf_table'})
+
+    # --- Embedded Excel Worksheets (fallback if no EMF vector tables) ---
+    embedded_excel = extract_embedded_excel_tables(ctx.filename, ctx.page_num)
+    if embedded_excel and ctx.page_num in embedded_excel:
+        for excel_md in embedded_excel[ctx.page_num]:
+            if excel_md and not any(el.get('type') == 'emf_table' for el in raw_elements):
+                raw_elements.append({'text': excel_md, 'type': 'embedded_table'})
+
+    # --- Image Descriptions (if any) ---
+    if image_descriptions:
+        for desc in image_descriptions:
+            raw_elements.append({'text': desc, 'type': 'image'})
+
+    # --- Apply Visual Card / Box Decorations (---) ---
+    decorated_elements = apply_pptx_box_decorations(raw_elements, card_boxes)
+
+    # --- Slide Notes (Speaker Notes) ---
+    if ctx.slide.has_notes_slide and ctx.slide.notes_slide.notes_text_frame:
+        notes_text = ctx.slide.notes_slide.notes_text_frame.text.strip()
+        if notes_text:
+            decorated_elements.append(f"Notes:\n{notes_text}")
+
+    # Build clean page header and body
+    header = f"[Slide {ctx.page_num} | File: {ctx.display_name}]"
+    body_text = "\n\n".join(el for el in decorated_elements if el.strip())
+    full_content = header + "\n\n" + body_text if body_text else header
+
+    return {
+        "content": full_content,
+        "chunkIndex": ctx.page_num,
+        "filename": ctx.display_name
+    }
+
+def process_pptx_file(filename, original_filename=None, process_images=None, model=None):
+    display_name = os.path.basename(original_filename) if original_filename else os.path.basename(filename)
+    should_process_images = bool(process_images)
+    ppt = Presentation(filename)
+    seen_image_hashes = {}
+    image_data_store = {}
+    image_page_map = defaultdict(list)
+    
+    lock = threading.Lock()
+    
+    if should_process_images:
+        _ensure_aspose()
+        aspose_ppt = slides.Presentation(filename)
+    else:
+        aspose_ppt = None
+    slide_image_map = get_slide_image_mapping(filename)
+    
+    contexts = [
+        SlideContext(
+            page_num=page_num,
+            slide=slide,
+            aspose_ppt=aspose_ppt,
+            slide_image_map=slide_image_map,
+            seen_image_hashes=seen_image_hashes,
+            image_data_store=image_data_store,
+            image_page_map=image_page_map,
+            filename=filename,
+            lock=lock,
+            display_name=display_name,
+            should_process_images=should_process_images,
+            model=model
+        )
+        for page_num, slide in enumerate(ppt.slides, start=1)
+    ]
+
+    try:
+        workers = int(os.environ.get("MAX_WORKERS", 3))
+        with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
+            results = list(executor.map(process_pptx_slide, contexts))
+        
+        sorted_results = sorted(results, key=lambda x: x["chunkIndex"])
+        return sorted_results
+    finally:
+        if aspose_ppt is not None:
+            try:
+                aspose_ppt.dispose()
+            except Exception:
+                pass
+            aspose_ppt = None
+        del contexts
+        try:
+            del results
+        except NameError:
+            pass
+        gc.collect()