super-document-processor 0.1.3__py3-none-any.whl

super_document_processor/processor.py

@@ -0,0 +1,135 @@
+import os
+import tempfile
+import logging
+import importlib
+from typing import List, Dict, Any, Optional
+from dotenv import load_dotenv
+
+logger = logging.getLogger("super_document_processor")
+
+# Load environment variables from local .env file if present
+load_dotenv()
+
+# LLM / Vision endpoint configuration via environment variables
+OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
+OPENAI_BASE_URL = os.getenv('OPENAI_BASE_URL')
+
+# Flag to process images via VLM (default False; enabled explicitly via CLI/API parameter)
+process_images = False
+
+# Server host & port configuration
+PORT = int(os.getenv('PORT', '8000'))
+HOST = "0.0.0.0"
+
+PLAIN_TEXT_EXTENSIONS = {
+    ".txt", ".md", ".json", ".org", ".rst", ".html", ".xml",
+    ".py", ".js", ".jsx", ".tsx", ".java", ".cpp", ".c", ".h",
+    ".cs", ".rb", ".go", ".php", ".swift", ".ts",
+    ".css", ".sh", ".pl", ".sql", ".yml", ".yaml",
+    ".r", ".kt", ".lua", ".vb", ".d", ".scala", ".m", ".vhd",
+    ".asm", ".tex", ".sass", ".scss", ".less", ".vue", ".erb",
+    ".coffee", ".jade", ".handlebars", ".mustache", ".twig",
+    ".hbs", ".liquid", ".pug", ".aspx", ".cshtml"
+}
+
+IMAGE_EXTENSIONS = {".jpg", ".png", ".jpeg", ".jfif"}
+
+SUPPORTED_EXTENSIONS = {
+    ".pdf", ".docx", ".pptx", ".msg"
+} | PLAIN_TEXT_EXTENSIONS | IMAGE_EXTENSIONS
+
+# Registry mapping file extensions to (module_name, function_name)
+_PROCESSOR_REGISTRY = {
+    ".pdf": ("process_pdf", "process_pdf_file"),
+    ".docx": ("process_docx", "process_docx_file"),
+    ".pptx": ("process_pptx", "process_pptx_file"),
+    ".msg": ("process_msg", "process_msg_file"),
+}
+for _ext in IMAGE_EXTENSIONS:
+    _PROCESSOR_REGISTRY[_ext] = ("process_images", "process_image_file")
+for _ext in PLAIN_TEXT_EXTENSIONS:
+    _PROCESSOR_REGISTRY[_ext] = ("process_plain_text", "process_plain_file")
+
+
+def _get_processor(file_type: str):
+    """Dynamically import document processors on demand."""
+    target = _PROCESSOR_REGISTRY.get(file_type)
+    if not target:
+        return None
+    mod_name, func_name = target
+    try:
+        mod = importlib.import_module(f"super_document_processor.document_processors.{mod_name}")
+    except ImportError:
+        mod = importlib.import_module(f"document_processors.{mod_name}")
+    return getattr(mod, func_name)
+
+
+def process_file(
+    file_path: str,
+    original_filename: Optional[str] = None,
+    process_images: Optional[bool] = None,
+    model: Optional[str] = None
+) -> List[Dict[str, Any]]:
+    """
+    Process any supported document or media file and return a list of extracted chunks.
+    
+    Args:
+        file_path: Path to the file to process.
+        original_filename: Optional original filename.
+        process_images: Optional bool to override default VLM image processing.
+        model: Optional vision model name to use for image extraction.
+        
+    Returns:
+        List of chunk dictionaries with content, chunkIndex, and filename.
+    """
+    if not os.path.exists(file_path):
+        raise FileNotFoundError(f"File not found: {file_path}")
+
+    raw_filename = original_filename or os.path.basename(file_path)
+    display_filename = os.path.basename(raw_filename)
+    file_type = os.path.splitext(display_filename)[1].lower()
+
+    if file_type not in SUPPORTED_EXTENSIONS:
+        raise ValueError(f"Unsupported file type '{file_type}'. Supported extensions: {sorted(SUPPORTED_EXTENSIONS)}")
+
+    if file_type in IMAGE_EXTENSIONS and not process_images:
+        raise ValueError(
+            f"Processing image file '{display_filename}' requires image processing to be enabled.\n"
+            "Please enable image processing via CLI (--process-images) or API ('process_images': True)."
+        )
+
+    if process_images and not model:
+        raise ValueError(
+            "Cannot process images: No vision model specified.\n"
+            "Please specify a vision-capable model via CLI (--model) or API ('model' parameter)."
+        )
+
+    if model and not process_images:
+        raise ValueError(
+            "A vision model was specified, but image processing is not enabled.\n"
+            "Please enable image processing (via --process-images or process_images=True) or remove the model parameter."
+        )
+
+    processor = _get_processor(file_type)
+
+    if file_type == ".pdf":
+        return processor(file_path, original_filename=display_filename, process_images=process_images, model=model)
+
+    elif file_type == ".msg":
+        return processor(file_path, original_filename=display_filename)
+
+    elif file_type == ".docx":
+        return processor(file_path, original_filename=display_filename, process_images=process_images, model=model)
+
+    elif file_type == ".pptx":
+        return processor(file_path, original_filename=display_filename, process_images=process_images, model=model)
+
+    elif file_type in PLAIN_TEXT_EXTENSIONS:
+        return processor(file_path, original_filename=display_filename)
+
+    elif file_type in IMAGE_EXTENSIONS:
+        chunk = processor(file_path, original_filename=display_filename, process_images=process_images, model=model)
+        return [chunk] if chunk else []
+
+    else:
+        raise ValueError(f"Unsupported file type: {file_type}")

super_document_processor/utils/aspose_slide_renderer.py

@@ -0,0 +1,103 @@
+import io
+import os
+import re
+import subprocess
+from pathlib import Path
+from typing import Optional
+
+def fix_svg_dimensions(svg_text: str, scale: float = 2.0) -> str:
+    """
+    Aspose exports SVGs with inch units: width="13.333333in" height="7.5in".
+    resvg requires pixel values matching viewBox.
+    This replaces width/height in the root <svg> tag with scaled pixel dimensions.
+    """
+    svg_start = svg_text.find("<svg")
+    if svg_start != -1:
+        svg_end = svg_text.find(">", svg_start)
+        if svg_end != -1:
+            header = svg_text[svg_start:svg_end+1]
+            vb_match = re.search(r'viewBox=["\']\s*[\d.]+\s+[\d.]+\s+([\d.]+)\s+([\d.]+)\s*["\']', header)
+            if vb_match:
+                w_px = int(float(vb_match.group(1)) * scale)
+                h_px = int(float(vb_match.group(2)) * scale)
+                header = re.sub(r'width=["\'][^"\']*["\']', f'width="{w_px}"', header, count=1)
+                header = re.sub(r'height=["\'][^"\']*["\']', f'height="{h_px}"', header, count=1)
+                svg_text = svg_text[:svg_start] + header + svg_text[svg_end+1:]
+    return svg_text
+
+def strip_aspose_watermark_from_svg(svg_text: str) -> str:
+    """
+    Exact search and removal strictly for the Aspose evaluation watermark block.
+    Validates that the matched <text>...</text> block explicitly contains
+    'Created with Aspose.Slides' or 'Evaluation only' and 'Aspose'.
+    """
+    for marker in ["Created with Aspose.Slides", "Evaluation only"]:
+        while marker in svg_text:
+            idx = svg_text.find(marker)
+            if idx == -1:
+                break
+            start_pos = svg_text.rfind("<text", 0, idx)
+            end_pos = svg_text.find("</text>", idx)
+            if start_pos != -1 and end_pos != -1:
+                block = svg_text[start_pos:end_pos + 7]
+                if "Aspose" in block and ("Evaluation only" in block or "tp_0" in block):
+                    svg_text = svg_text[:start_pos] + svg_text[end_pos + 7:]
+                else:
+                    break
+            else:
+                break
+    return svg_text
+
+def find_resvg_bin() -> Optional[Path]:
+    """Locates the standalone native resvg binary if installed."""
+    import shutil
+    for candidate in ["/usr/local/bin/resvg", "/usr/bin/resvg", "resvg", "resvg.exe"]:
+        found = shutil.which(candidate)
+        if found and os.access(found, os.X_OK):
+            return Path(found)
+    return None
+
+def render_svg_to_png_bytes(svg_string: str, scale: float = 2.0) -> bytes:
+    """
+    Renders an SVG string to PNG bytes using the standalone native resvg CLI.
+    """
+    clean_svg = strip_aspose_watermark_from_svg(svg_string)
+    clean_svg = fix_svg_dimensions(clean_svg, scale=scale)
+    
+    resvg_bin = find_resvg_bin()
+    if not resvg_bin:
+        raise RuntimeError("Standalone native resvg binary not found in PATH or /usr/local/bin/resvg.")
+
+    import tempfile
+    with tempfile.NamedTemporaryFile(suffix=".svg", delete=False) as in_svg, \
+         tempfile.NamedTemporaryFile(suffix=".png", delete=False) as out_png:
+        in_svg.write(clean_svg.encode("utf-8"))
+        in_svg.flush()
+        in_path = in_svg.name
+        out_path = out_png.name
+
+    try:
+        print(f"⚡ [Rasterizer] Using standalone native resvg CLI at scale={scale}", flush=True)
+        cmd = [str(resvg_bin), "-z", str(scale), in_path, out_path]
+        res = subprocess.run(cmd, capture_output=True, timeout=60)
+        if res.returncode == 0 and os.path.exists(out_path):
+            with open(out_path, "rb") as f:
+                return f.read()
+        else:
+            err_msg = res.stderr.decode("utf-8", errors="ignore").strip()
+            raise RuntimeError(f"Native resvg rasterization failed: {err_msg}")
+    finally:
+        if os.path.exists(in_path):
+            os.remove(in_path)
+        if os.path.exists(out_path):
+            os.remove(out_path)
+
+def render_slide_to_png_bytes(aspose_slide, scale: float = 2.0) -> bytes:
+    """
+    Exports an Aspose slide to clean SVG in memory, strips the watermark, and
+    rasterizes it to high-fidelity PNG bytes.
+    """
+    svg_stream = io.BytesIO()
+    aspose_slide.write_as_svg(svg_stream)
+    raw_svg = svg_stream.getvalue().decode("utf-8", errors="ignore")
+    return render_svg_to_png_bytes(raw_svg, scale=scale)

super_document_processor/utils/chunker.py

@@ -0,0 +1,127 @@
+import threading
+
+class ChunkCollector:
+    def __init__(self):
+        self.chunks = []
+        self.max_char_per_chunk = 6000
+        self.chunk_overlap = 0
+
+    def add(self, text=""):
+        self.chunks.append(text)
+
+    def output(self):
+        return '\n'.join(self.chunks)
+
+    def get_all(self):
+        return self.chunks
+
+    def clear(self):
+        self.chunks = []
+
+    def get_chunked_texts(self, max_size=None, overlap=None):
+        if max_size is None:
+            max_size = self.max_char_per_chunk
+        if overlap is None:
+            overlap = self.chunk_overlap
+
+        if max_size <= 0:
+            raise ValueError("max_size must be greater than 0")
+
+        # Cap overlap to ensure strictly valid forward chunking
+        overlap = max(0, min(overlap, max_size - 1))
+
+        full_text = '\n'.join(self.chunks)
+        if not full_text:
+            return []
+
+        text_length = len(full_text)
+        if text_length <= max_size:
+            stripped = full_text.strip()
+            return [stripped] if stripped else []
+
+        chunks = []
+        start = 0
+
+        while start < text_length:
+            # Determine candidate end boundary
+            end = min(start + max_size, text_length)
+
+            # If not at the end of the text, break at a whitespace boundary
+            if end < text_length and full_text[end] not in (' ', '\n', '\t'):
+                walk = end
+                while walk > start and full_text[walk] not in (' ', '\n', '\t'):
+                    walk -= 1
+                if walk > start:
+                    end = walk
+
+            chunk = full_text[start:end].strip()
+            if chunk:
+                chunks.append(chunk)
+
+            if end >= text_length:
+                break
+
+            # Determine next chunk starting position
+            if end - overlap > start:
+                target_start = end - overlap
+
+                # Snap target_start cleanly to word boundary without cutting tokens
+                if full_text[target_start] not in (' ', '\n', '\t') and full_text[target_start - 1] not in (' ', '\n', '\t'):
+                    walk = target_start
+                    while walk > start and full_text[walk - 1] not in (' ', '\n', '\t'):
+                        walk -= 1
+                    if walk > start:
+                        target_start = walk
+                    else:
+                        walk = target_start
+                        while walk < end and full_text[walk] not in (' ', '\n', '\t'):
+                            walk += 1
+                        while walk < end and full_text[walk] in (' ', '\n', '\t'):
+                            walk += 1
+                        target_start = walk
+            else:
+                target_start = end
+
+            # Skip leading inline whitespace
+            while target_start < text_length and full_text[target_start] in (' ', '\t'):
+                target_start += 1
+
+            # Guarantee strict forward progress to completely prevent infinite loops
+            start = max(start + 1, target_start)
+
+        return chunks
+
+class _ThreadLocalChunkCollector:
+    """Thread-isolated proxy for ChunkCollector so concurrent requests never share state."""
+    def __init__(self):
+        self._local = threading.local()
+
+    def _get_instance(self) -> ChunkCollector:
+        if not hasattr(self._local, "instance"):
+            self._local.instance = ChunkCollector()
+        return self._local.instance
+
+    def add(self, text=""):
+        self._get_instance().add(text)
+
+    def output(self):
+        return self._get_instance().output()
+
+    def get_all(self):
+        return self._get_instance().get_all()
+
+    def clear(self):
+        self._get_instance().clear()
+
+    @property
+    def max_char_per_chunk(self):
+        return self._get_instance().max_char_per_chunk
+
+    @property
+    def chunk_overlap(self):
+        return self._get_instance().chunk_overlap
+
+    def get_chunked_texts(self, max_size=None, overlap=None):
+        return self._get_instance().get_chunked_texts(max_size=max_size, overlap=overlap)
+
+collector = _ThreadLocalChunkCollector()

super_document_processor/utils/docx_image_utils.py

@@ -0,0 +1,167 @@
+from lxml import etree
+import zipfile
+import logging
+from typing import Iterator, Optional, Dict, Any
+from docx.text.paragraph import Paragraph
+from dataclasses import dataclass
+import hashlib
+
+logger = logging.getLogger("super_document_processor.docx_images")
+
+@dataclass
+class ElementMetadata:
+    filename: Optional[str] = None
+    page_number: Optional[int] = None
+    last_modified: Optional[str] = None
+
+@dataclass
+class ImageElement:
+    text: str
+    metadata: ElementMetadata
+    detection_origin: str = "docx"
+    image_alt_text: Optional[str] = None
+    image_title: Optional[str] = None
+    image_name: Optional[str] = None
+    image_r_id: Optional[str] = None
+
+class CustomImagePartitioner:
+    """Custom image partitioner to extract images from DOCX paragraphs."""
+
+    @classmethod
+    def iter_elements(cls, paragraph: Paragraph, opts) -> Iterator[ImageElement]:
+        """Generate Image elements for each picture found in the paragraph."""
+        # Find all drawing elements (images) in the paragraph
+        drawing_elements = paragraph._p.xpath('.//w:drawing')
+        for drawing in drawing_elements:
+            try:
+                image_info = cls._extract_image_info(drawing, opts)
+                if image_info:
+                    yield cls._create_image_element(image_info, opts)
+            except Exception as e:
+                # Continue processing even if one image fails
+                logger.warning(f"Could not extract image: {e}")
+                continue
+
+    @classmethod
+    def _extract_image_info(cls, drawing_element, opts) -> Optional[Dict[str, Any]]:
+        """Extract image metadata from drawing element."""
+        # Get image relationship ID
+        blip_elements = drawing_element.xpath('.//a:blip/@r:embed')
+        if not blip_elements:
+            return None
+        r_id = blip_elements[0]
+
+        # Get image description/alt text
+        desc_elements = drawing_element.xpath('.//wp:docPr/@descr')
+        alt_text = desc_elements[0] if desc_elements else None
+
+        # Get image title
+        title_elements = drawing_element.xpath('.//wp:docPr/@title')
+        title = title_elements[0] if title_elements else None
+
+        # Get image name
+        name_elements = drawing_element.xpath('.//wp:docPr/@name')
+        name = name_elements[0] if name_elements else f"Image_{r_id}"
+
+        return {
+            'r_id': r_id,
+            'alt_text': alt_text,
+            'title': title,
+            'name': name
+        }
+
+    @classmethod
+    def _create_image_element(cls, image_info: Dict[str, Any], opts) -> ImageElement:
+        """Create Image element with metadata."""
+
+        text_parts = []
+        if image_info.get('title'):
+            text_parts.append(f"Title: {image_info['title']}")
+        if image_info.get('alt_text'):
+            text_parts.append(f"Description: {image_info['alt_text']}")
+        text_parts.append(f"Name: {image_info['name']}")
+
+        text = " | ".join(text_parts) if text_parts else f"Image: {image_info['name']}"
+
+        # Create metadata with only supported parameters
+        metadata = ElementMetadata(
+            filename=getattr(opts, 'metadata_file_path', None),
+            page_number=getattr(opts, 'metadata_page_number', None),
+            last_modified=getattr(opts, 'last_modified', None)
+        )
+
+        # Create and return the ImageElement dataclass
+        image_element = ImageElement(
+            text=text,
+            metadata=metadata,
+            detection_origin="docx",
+            image_alt_text=image_info.get('alt_text'),
+            image_title=image_info.get('title'),
+            image_name=image_info.get('name'),
+            image_r_id=image_info.get('r_id')
+        )
+
+        return image_element
+
+def quick_image_check(filename):
+    """Quick check to see if DOCX contains images by inspecting media files inside the archive"""
+    try:
+        with zipfile.ZipFile(filename, 'r') as docx_zip:
+            files = docx_zip.namelist()
+            media_files = [f for f in files if f.startswith('word/media/')]
+            image_count = len(media_files)
+
+        return image_count > 0
+
+    except Exception as e:
+        logger.error(f"Error checking images: {e}")
+        return False
+
+def extract_images_from_docx(filename):
+    extracted_images = []
+    seen_image_hashes = {}
+    try:
+        with zipfile.ZipFile(filename, 'r') as docx_zip:
+            # Parse the relationships XML
+            rels_xml = docx_zip.read('word/_rels/document.xml.rels')
+            rels_root = etree.fromstring(rels_xml)
+
+            # Build a map rId -> target path
+            rid_to_target = {}
+            for rel in rels_root.findall('{http://schemas.openxmlformats.org/package/2006/relationships}Relationship'):
+                rId = rel.get('Id')
+                target = rel.get('Target')
+                rid_to_target[rId] = target
+
+            # List media files in docx
+            media_files = [f for f in docx_zip.namelist() if f.startswith('word/media/')]
+
+            # Reverse map target path to rId (normalize with 'word/' prefix removed)
+            target_to_rid = {v.lstrip('/'): k for k, v in rid_to_target.items()}
+
+            for media_file in media_files:
+                img_data = docx_zip.read(media_file)
+                # Compute hash for deduplication
+                img_hash = hashlib.sha256(img_data).hexdigest()
+                if img_hash in seen_image_hashes:
+                    continue  # Skip duplicate image
+                seen_image_hashes[img_hash] = True
+
+                # Normalize relative path (remove leading 'word/')
+                relative_path = media_file[len('word/'):]
+
+                # Lookup rId by target filename
+                r_id = target_to_rid.get(relative_path)
+
+                extracted_images.append({
+                    'filename': media_file,
+                    'data': img_data,
+                    'r_id': r_id,
+                    'hash': img_hash,
+                })
+
+        return extracted_images
+
+    except Exception as e:
+        logger.error(f"Error extracting images: {e}")
+        return []

super_document_processor/utils/docx_xml_parser.py

@@ -0,0 +1,73 @@
+import logging
+
+logger = logging.getLogger("super_document_processor.docx_xml")
+
+try:
+    from super_document_processor.utils.chunker import collector
+except ImportError:
+    from utils.chunker import collector
+
+def print_list_item(text, level, marker=None):
+    """Print list item with proper indentation and marker (ordered number or bullet)"""
+    indent = "  " * level
+    bullet_symbol = marker if marker is not None else "-"
+    collector.add(f"{indent}{bullet_symbol} {text}")
+
+def extract_footnotes(docx_path):
+    """Extracts footnotes from a DOCX file."""
+    import zipfile
+    from xml.etree import ElementTree as ET
+    footnotes = []
+    try:
+        with zipfile.ZipFile(docx_path) as docx:
+            if 'word/footnotes.xml' not in docx.namelist():
+                # print("No footnotes.xml found in DOCX.")
+                return footnotes
+            xml_content = docx.read('word/footnotes.xml')
+            tree = ET.fromstring(xml_content)
+            # WordprocessingML namespace
+            ns = {'w': 'http://schemas.openxmlformats.org/wordprocessingml/2006/main'}
+            for footnote in tree.findall('w:footnote', ns):
+                texts = []
+                for paragraph in footnote.findall('.//w:p', ns):
+                    for run in paragraph.findall('.//w:t', ns):
+                        texts.append(run.text)
+                full_text = ''.join(texts).strip()
+                if full_text:
+                    footnotes.append(full_text)
+        return footnotes
+    except Exception as e:
+        logger.warning(f"Could not extract footnotes: {e}")
+        return []
+
+def extract_endnotes(docx_path):
+    """Extracts endnotes from a DOCX file."""
+    import zipfile
+    from xml.etree import ElementTree as ET
+
+    endnotes = []
+
+    try:
+        with zipfile.ZipFile(docx_path) as docx:
+            if 'word/endnotes.xml' not in docx.namelist():
+                # print("No endnotes.xml found in DOCX.")
+                return endnotes
+
+            xml_content = docx.read('word/endnotes.xml')
+            tree = ET.fromstring(xml_content)
+
+            ns = {'w': 'http://schemas.openxmlformats.org/wordprocessingml/2006/main'}
+
+            for endnote in tree.findall('w:endnote', ns):
+                texts = []
+                for paragraph in endnote.findall('.//w:p', ns):
+                    for run in paragraph.findall('.//w:t', ns):
+                        texts.append(run.text)
+                full_text = ''.join(texts).strip()
+                if full_text:
+                    endnotes.append(full_text)
+
+        return endnotes
+    except Exception as e:
+        logger.warning(f"Could not extract endnotes: {e}")
+        return []

super_document_processor/utils/optimizer.py

@@ -0,0 +1,49 @@
+import sys
+import os
+import logging
+import importlib.util
+
+logger = logging.getLogger("super_document_processor.optimizer")
+
+def prune_environment(verbose: bool = False) -> int:
+    """
+    Scans the installed 'aspose' package in the active environment and deletes:
+    1. Redundant native C-extensions compiled for Python versions other than the current running one (e.g. cp35-cp312 on cp313).
+    2. IDE type stubs (.pyi) and non-matching platform static archives.
+    
+    Returns total bytes freed.
+    """
+    total_freed = 0
+    try:
+        spec = importlib.util.find_spec("aspose")
+        if not spec or not spec.submodule_search_locations:
+            return 0
+        aspose_dir = spec.submodule_search_locations[0]
+        cur_tag = f"cp{sys.version_info.major}{sys.version_info.minor}"
+
+        for root, _, files in os.walk(aspose_dir):
+            for f in files:
+                nl = f.lower()
+                # Check for other python version binaries
+                is_other_py = (
+                    (".pyd" in nl or ".so" in nl)
+                    and "cp" in nl
+                    and cur_tag not in nl
+                )
+                is_stub = nl.endswith(".pyi")
+
+                if is_other_py or is_stub:
+                    fp = os.path.join(root, f)
+                    try:
+                        sz = os.path.getsize(fp)
+                        os.remove(fp)
+                        total_freed += sz
+                    except (OSError, PermissionError):
+                        pass
+
+        if verbose and total_freed > 0:
+            logger.info(f"⚡ [SuperDocProcessor] Automatically reclaimed {total_freed / (1024*1024):.1f} MB of redundant binaries.")
+    except Exception as e:
+        logger.debug(f"Environment auto-prune skipped: {e}")
+
+    return total_freed