content-core 1.10.0__py3-none-any.whl

content_core/processors/office.py

@@ -0,0 +1,331 @@
+import asyncio
+from functools import partial
+
+from docx import Document  # type: ignore
+from openpyxl import load_workbook  # type: ignore
+from pptx import Presentation  # type: ignore
+
+from content_core.common import ProcessSourceState
+from content_core.logging import logger
+
+SUPPORTED_OFFICE_TYPES = [
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+]
+
+
+async def extract_docx_content_detailed(file_path):
+    """Extract content from DOCX file"""
+
+    def _extract():
+        try:
+            doc = Document(file_path)
+            content = []
+
+            for paragraph in doc.paragraphs:
+                if not paragraph.text.strip():
+                    continue
+
+                style = paragraph.style.name if paragraph.style else "Normal"
+                text = paragraph.text.strip()
+
+                # Get paragraph formatting
+                p_format = paragraph.paragraph_format
+                indent = p_format.left_indent or 0
+
+                # Convert indent to spaces (1 level = 4 spaces)
+                indent_level = 0
+                if hasattr(indent, "pt"):
+                    indent_level = int(indent.pt / 72)  # 72 points = 1 inch
+                indent_spaces = " " * (indent_level * 4)
+
+                # Handle different types of formatting
+                if "Heading" in style:
+                    level = style[-1] if style[-1].isdigit() else "1"
+                    heading_marks = "#" * int(level)
+                    content.append(f"\n{heading_marks} {text}\n")
+
+                # Handle bullet points
+                elif (
+                    paragraph.style
+                    and hasattr(paragraph.style, "name")
+                    and paragraph.style.name.startswith("List")
+                ):
+                    # Numbered list
+                    if (
+                        hasattr(paragraph._p, "pPr")
+                        and paragraph._p.pPr is not None
+                        and hasattr(paragraph._p.pPr, "numPr")
+                        and paragraph._p.pPr.numPr is not None
+                    ):
+                        # Try to get the actual number
+                        try:
+                            if (
+                                hasattr(paragraph._p.pPr.numPr, "numId")
+                                and paragraph._p.pPr.numPr.numId is not None
+                                and hasattr(paragraph._p.pPr.numPr.numId, "val")
+                            ):
+                                number = paragraph._p.pPr.numPr.numId.val
+                                content.append(f"{indent_spaces}{number}. {text}")
+                            else:
+                                content.append(f"{indent_spaces}1. {text}")
+                        except Exception:
+                            content.append(f"{indent_spaces}1. {text}")
+                    # Bullet list
+                    else:
+                        content.append(f"{indent_spaces}* {text}")
+
+                else:
+                    # Handle text formatting
+                    formatted_text = []
+                    for run in paragraph.runs:
+                        if run.bold:
+                            formatted_text.append(f"**{run.text}**")
+                        elif run.italic:
+                            formatted_text.append(f"*{run.text}*")
+                        else:
+                            formatted_text.append(run.text)
+
+                    content.append(f"{indent_spaces}{''.join(formatted_text)}")
+
+            return "\n\n".join(content)
+
+        except Exception as e:
+            logger.error(f"Failed to extract DOCX content: {e}")
+            return None
+
+    return await asyncio.get_event_loop().run_in_executor(None, _extract)
+
+
+async def get_docx_info(file_path):
+    """Get DOCX metadata and content"""
+
+    async def _get_info():
+        try:
+            doc = Document(file_path)
+
+            # Extract core properties if available
+            core_props = {
+                "author": doc.core_properties.author,
+                "created": doc.core_properties.created,
+                "modified": doc.core_properties.modified,
+                "title": doc.core_properties.title,
+                "subject": doc.core_properties.subject,
+                "keywords": doc.core_properties.keywords,
+                "category": doc.core_properties.category,
+                "comments": doc.core_properties.comments,
+            }
+
+            # Get document content
+            content = await extract_docx_content_detailed(file_path)
+
+            # Get document statistics
+            stats = {
+                "paragraph_count": len(doc.paragraphs),
+                "word_count": sum(
+                    len(p.text.split()) for p in doc.paragraphs if p.text.strip()
+                ),
+                "character_count": sum(
+                    len(p.text) for p in doc.paragraphs if p.text.strip()
+                ),
+            }
+
+            return {"metadata": core_props, "content": content, "statistics": stats}
+
+        except Exception as e:
+            logger.error(f"Failed to get DOCX info: {e}")
+            return None
+
+    return await _get_info()
+
+
+async def extract_pptx_content(file_path):
+    """Extract content from PPTX file"""
+
+    def _extract():
+        try:
+            prs = Presentation(file_path)
+            content = []
+
+            for slide_number, slide in enumerate(prs.slides, 1):
+                content.append(f"\n# Slide {slide_number}\n")
+
+                # Extract title
+                if slide.shapes.title:
+                    content.append(f"## {slide.shapes.title.text}\n")
+
+                # Extract text from all shapes
+                for shape in slide.shapes:
+                    if hasattr(shape, "text") and shape.text.strip():
+                        if (
+                            shape != slide.shapes.title
+                        ):  # Skip title as it's already added
+                            content.append(shape.text.strip())
+
+            return "\n\n".join(content)
+
+        except Exception as e:
+            logger.error(f"Failed to extract PPTX content: {e}")
+            return None
+
+    return await asyncio.get_event_loop().run_in_executor(None, _extract)
+
+
+async def extract_xlsx_content(file_path, max_rows=10000, max_cols=100):
+    """Extract content from XLSX file"""
+
+    def _extract():
+        try:
+            wb = load_workbook(file_path, data_only=True)
+            content = []
+
+            for sheet in wb.sheetnames:
+                ws = wb[sheet]
+                content.append(f"\n# Sheet: {sheet}\n")
+
+                # Get the maximum row and column with data
+                max_row = min(ws.max_row, max_rows)
+                max_col = min(ws.max_column, max_cols)
+
+                # Create markdown table header
+                headers = []
+                for col in range(1, max_col + 1):
+                    cell_value = ws.cell(row=1, column=col).value
+                    headers.append(str(cell_value) if cell_value is not None else "")
+
+                content.append("| " + " | ".join(headers) + " |")
+                content.append("| " + " | ".join(["---"] * len(headers)) + " |")
+
+                # Add table content
+                for row in range(2, max_row + 1):
+                    row_data = []
+                    for col in range(1, max_col + 1):
+                        cell_value = ws.cell(row=row, column=col).value
+                        row_data.append(
+                            str(cell_value) if cell_value is not None else ""
+                        )
+                    content.append("| " + " | ".join(row_data) + " |")
+
+            return "\n".join(content)
+
+        except Exception as e:
+            logger.error(f"Failed to extract XLSX content: {e}")
+            return None
+
+    return await asyncio.get_event_loop().run_in_executor(None, partial(_extract))
+
+
+async def get_pptx_info(file_path):
+    """Get PPTX metadata and content"""
+
+    def _get_pptx_metadata_sync(file_path):
+        """Synchronous helper to extract metadata using python-pptx."""
+        try:
+            prs = Presentation(file_path)
+            props = {
+                "slide_count": len(prs.slides),
+                "title": "",  # PowerPoint doesn't have built-in metadata like Word
+            }
+            stats = {
+                "slide_count": len(prs.slides),
+                "shape_count": sum(len(slide.shapes) for slide in prs.slides),
+                "text_frame_count": sum(
+                    sum(1 for shape in slide.shapes if hasattr(shape, "text"))
+                    for slide in prs.slides
+                ),
+            }
+            return {"metadata": props, "statistics": stats}
+        except Exception as e:
+            logger.error(f"Failed to get PPTX metadata: {e}")
+            return None
+
+    try:
+        # Run blocking python-pptx operations in executor
+        metadata_info = await asyncio.get_event_loop().run_in_executor(
+            None, _get_pptx_metadata_sync, file_path
+        )
+
+        # Await the async content extraction directly
+        content = await extract_pptx_content(file_path)
+
+        if metadata_info:
+            # Combine results
+            return {**metadata_info, "content": content}
+        else:
+            # Fallback if metadata extraction failed
+            return {"metadata": {}, "statistics": {}, "content": content}
+
+    except Exception as e:
+        logger.error(f"Failed to get PPTX info: {e}")
+        return None
+
+
+async def get_xlsx_info(file_path):
+    """Get XLSX metadata and content"""
+
+    async def _get_info():
+        try:
+            wb = load_workbook(file_path, data_only=True)
+
+            # Extract basic properties
+            props = {
+                "sheet_count": len(wb.sheetnames),
+                "sheets": wb.sheetnames,
+                "title": wb.properties.title,
+                "creator": wb.properties.creator,
+                "created": wb.properties.created,
+                "modified": wb.properties.modified,
+            }
+
+            # Get document content
+            content = await extract_xlsx_content(file_path)
+
+            # Get workbook statistics
+            stats = {
+                "sheet_count": len(wb.sheetnames),
+                "total_rows": sum(sheet.max_row for sheet in wb.worksheets),
+                "total_columns": sum(sheet.max_column for sheet in wb.worksheets),
+            }
+
+            return {"metadata": props, "content": content, "statistics": stats}
+
+        except Exception as e:
+            logger.error(f"Failed to get XLSX info: {e}")
+            return None
+
+    return await _get_info()
+
+
+async def extract_office_content(state: ProcessSourceState):
+    """Universal function to extract content from Office files"""
+    assert state.file_path, "No file path provided"
+    assert state.identified_type in SUPPORTED_OFFICE_TYPES, "Unsupported File Type"
+    file_path = state.file_path
+    doc_type = state.identified_type
+
+    if (
+        doc_type
+        == "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+    ):
+        logger.debug("Extracting content from DOCX file")
+        content = await extract_docx_content_detailed(file_path)
+        info = await get_docx_info(file_path)
+    elif (
+        doc_type
+        == "application/vnd.openxmlformats-officedocument.presentationml.presentation"
+    ):
+        logger.debug("Extracting content from PPTX file")
+        content = await extract_pptx_content(file_path)
+        info = await get_pptx_info(file_path)
+    elif (
+        doc_type == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+    ):
+        logger.debug("Extracting content from XLSX file")
+        content = await extract_xlsx_content(file_path)
+        info = await get_xlsx_info(file_path)
+    else:
+        raise Exception(f"Unsupported file format: {doc_type}")
+
+    del info["content"]
+    return {"content": content, "metadata": info}

content_core/processors/pdf.py

@@ -0,0 +1,292 @@
+import asyncio
+import re
+import unicodedata
+
+import fitz  # type: ignore
+
+from content_core.common import ProcessSourceState
+from content_core.config import CONFIG
+from content_core.logging import logger
+
+def count_formula_placeholders(text):
+    """
+    Count the number of formula placeholders in extracted text.
+    
+    Args:
+        text (str): Extracted text content
+    Returns:
+        int: Number of formula placeholders found
+    """
+    if not text:
+        return 0
+    return text.count('<!-- formula-not-decoded -->')
+
+
+def extract_page_with_ocr(page, page_num):
+    """
+    Extract text from a page using OCR (Tesseract).
+    
+    Args:
+        page: PyMuPDF page object
+        page_num (int): Page number for logging
+    Returns:
+        str: OCR-extracted text or None if OCR fails
+    """
+    try:
+        logger.debug(f"Attempting OCR extraction for page {page_num}")
+        # Create TextPage using OCR
+        textpage = page.get_textpage_ocr()
+        if textpage:
+            # Extract text from the OCR TextPage
+            ocr_text = textpage.extractText()
+            logger.debug(f"OCR successful for page {page_num}, extracted {len(ocr_text)} characters")
+            return ocr_text
+        else:
+            logger.warning(f"OCR TextPage creation failed for page {page_num}")
+            return None
+    except (ImportError, RuntimeError, OSError) as e:
+        # Common errors: Tesseract not installed, OCR failure, file access issues
+        logger.debug(f"OCR extraction failed for page {page_num}: {e}")
+        return None
+    except Exception as e:
+        # Unexpected errors - log as warning for debugging
+        logger.warning(f"Unexpected error during OCR extraction for page {page_num}: {e}")
+        return None
+
+
+def convert_table_to_markdown(table):
+    """
+    Convert a PyMuPDF table to markdown format.
+    
+    Args:
+        table: Table data from PyMuPDF (list of lists)
+    Returns:
+        str: Markdown-formatted table
+    """
+    if not table or not table[0]:
+        return ""
+    
+    # Build markdown table
+    markdown_lines = []
+    
+    # Header row
+    header = table[0]
+    header_row = "| " + " | ".join(str(cell) if cell else "" for cell in header) + " |"
+    markdown_lines.append(header_row)
+    
+    # Separator row
+    separator = "|" + "|".join([" --- " for _ in header]) + "|"
+    markdown_lines.append(separator)
+    
+    # Data rows
+    for row in table[1:]:
+        if row:  # Skip empty rows
+            row_text = "| " + " | ".join(str(cell) if cell else "" for cell in row) + " |"
+            markdown_lines.append(row_text)
+    
+    return "\n".join(markdown_lines) + "\n"
+
+# Configuration constants
+DEFAULT_FORMULA_THRESHOLD = 3
+DEFAULT_OCR_FALLBACK = True
+
+SUPPORTED_FITZ_TYPES = [
+    "application/pdf",
+    "application/epub+zip",
+]
+
+
+def clean_pdf_text(text):
+    """
+    Clean text extracted from PDFs with enhanced space handling.
+    Preserves special characters like (, ), %, = that are valid in code/math.
+
+    Args:
+        text (str): The raw text extracted from a PDF
+    Returns:
+        str: Cleaned text with minimal necessary spacing
+    """
+    if not text:
+        return text
+
+    # Step 1: Normalize Unicode characters
+    text = unicodedata.normalize("NFKC", text)
+
+    # Step 2: Replace common PDF artifacts
+    replacements = {
+        # Common ligatures
+        "fi": "fi",
+        "fl": "fl",
+        "ff": "ff",
+        "ffi": "ffi",
+        "ffl": "ffl",
+        # Quotation marks and apostrophes
+        """: "'",
+        """: "'",
+        '"': '"',
+        "′": "'",
+        "‚": ",",
+        "„": '"',
+        # Dashes and hyphens
+        "‒": "-",
+        "–": "-",
+        "—": "-",
+        "―": "-",
+        # Other common replacements
+        "…": "...",
+        "•": "*",
+        "°": " degrees ",
+        "¹": "1",
+        "²": "2",
+        "³": "3",
+        "©": "(c)",
+        "®": "(R)",
+        "™": "(TM)",
+    }
+    for old, new in replacements.items():
+        text = text.replace(old, new)
+
+    # Step 3: Clean control characters while preserving essential whitespace and special chars
+    text = "".join(
+        char
+        for char in text
+        if unicodedata.category(char)[0] != "C"
+        or char in "\n\t "
+        or char in "()%=[]{}#$@!?.,;:+-*/^<>&|~"
+    )
+
+    # Step 4: Enhanced space cleaning
+    text = re.sub(r"[ \t]+", " ", text)  # Consolidate horizontal whitespace
+    text = re.sub(r" +\n", "\n", text)  # Remove spaces before newlines
+    text = re.sub(r"\n +", "\n", text)  # Remove spaces after newlines
+    text = re.sub(r"\n\t+", "\n", text)  # Remove tabs at start of lines
+    text = re.sub(r"\t+\n", "\n", text)  # Remove tabs at end of lines
+    text = re.sub(r"\t+", " ", text)  # Replace tabs with single space
+
+    # Step 5: Remove empty lines while preserving paragraph structure
+    text = re.sub(r"\n{3,}", "\n\n", text)  # Max two consecutive newlines
+    text = re.sub(r"^\s+", "", text)  # Remove leading whitespace
+    text = re.sub(r"\s+$", "", text)  # Remove trailing whitespace
+
+    # Step 6: Clean up around punctuation
+    text = re.sub(r"\s+([.,;:!?)])", r"\1", text)  # Remove spaces before punctuation
+    text = re.sub(r"(\()\s+", r"\1", text)  # Remove spaces after opening parenthesis
+    text = re.sub(
+        r"\s+([.,])\s+", r"\1 ", text
+    )  # Ensure single space after periods and commas
+
+    # Step 7: Remove zero-width and invisible characters
+    text = re.sub(r"[\u200b\u200c\u200d\ufeff\u200e\u200f]", "", text)
+
+    # Step 8: Fix hyphenation and line breaks
+    text = re.sub(
+        r"(?<=\w)-\s*\n\s*(?=\w)", "", text
+    )  # Remove hyphenation at line breaks
+
+    return text.strip()
+
+
+
+
+async def _extract_text_from_pdf(pdf_path):
+    """Extract text from PDF asynchronously with table detection"""
+
+    def _extract():
+        doc = fitz.open(pdf_path)
+        try:
+            full_text = []
+            logger.debug(f"Found {len(doc)} pages in PDF")
+            
+            # Use quality improvement flags for better text extraction
+            extraction_flags = (
+                fitz.TEXT_PRESERVE_LIGATURES |  # Better character rendering
+                fitz.TEXT_PRESERVE_WHITESPACE |  # Better spacing preservation
+                fitz.TEXT_PRESERVE_IMAGES       # Better image-text integration
+            )
+            
+            # Get OCR configuration
+            ocr_config = CONFIG.get('extraction', {}).get('pymupdf', {})
+            enable_ocr = ocr_config.get('enable_formula_ocr', False)
+            formula_threshold = ocr_config.get('formula_threshold', DEFAULT_FORMULA_THRESHOLD)
+            ocr_fallback = ocr_config.get('ocr_fallback', DEFAULT_OCR_FALLBACK)
+            
+            for page_num, page in enumerate(doc):
+                # Extract regular text with quality flags
+                standard_text = page.get_text(flags=extraction_flags)
+                
+                # Check if we should try OCR for this page
+                formula_count = count_formula_placeholders(standard_text)
+                use_ocr = (enable_ocr and 
+                          formula_count >= formula_threshold and 
+                          formula_count > 0)
+                
+                if use_ocr:
+                    logger.debug(f"Page {page_num + 1} has {formula_count} formulas, attempting OCR")
+                    ocr_text = extract_page_with_ocr(page, page_num + 1)
+                    
+                    if ocr_text and ocr_fallback:
+                        # Use OCR text but preserve table extraction from standard text
+                        page_text = ocr_text
+                        logger.debug(f"Using OCR text for page {page_num + 1}")
+                    else:
+                        # OCR failed, use standard text
+                        page_text = standard_text
+                        if not ocr_text:
+                            logger.debug(f"OCR failed for page {page_num + 1}, using standard extraction")
+                else:
+                    page_text = standard_text
+                
+                # Try to find and extract tables (regardless of OCR)
+                try:
+                    tables = page.find_tables()
+                    if tables:
+                        logger.debug(f"Found {len(tables)} table(s) on page {page_num + 1}")
+                        
+                        # For each table found, convert to markdown and append
+                        for table_num, table in enumerate(tables):
+                            # Extract table data
+                            table_data = table.extract()
+                            # Validate table has actual content (not just empty rows/cells)
+                            if table_data and len(table_data) > 0 and any(
+                                any(str(cell).strip() for cell in row if cell) for row in table_data if row
+                            ):
+                                # Add a marker before the table
+                                page_text += f"\n\n[Table {table_num + 1} from page {page_num + 1}]\n"
+                                # Convert to markdown
+                                markdown_table = convert_table_to_markdown(table_data)
+                                page_text += markdown_table + "\n"
+                except Exception as e:
+                    # If table extraction fails, continue with regular text
+                    logger.debug(f"Table extraction failed on page {page_num + 1}: {e}")
+                
+                full_text.append(page_text)
+            
+            # Join all pages and clean
+            combined_text = "".join(full_text)
+            return clean_pdf_text(combined_text)
+        finally:
+            doc.close()
+
+    # Run CPU-bound PDF processing in a thread pool
+    return await asyncio.get_event_loop().run_in_executor(None, _extract)
+
+
+async def extract_pdf(state: ProcessSourceState):
+    """
+    Parse the PDF file and extract its content asynchronously.
+    """
+    return_dict = {}
+    assert state.file_path, "No file path provided"
+    assert state.identified_type in SUPPORTED_FITZ_TYPES, "Unsupported File Type"
+
+    if state.file_path is not None and state.identified_type in SUPPORTED_FITZ_TYPES:
+        file_path = state.file_path
+        try:
+            text = await _extract_text_from_pdf(file_path)
+            return_dict["content"] = text
+        except FileNotFoundError:
+            raise FileNotFoundError(f"File not found at {file_path}")
+        except Exception as e:
+            raise Exception(f"An error occurred: {e}")
+
+    return return_dict

content_core/processors/text.py

@@ -0,0 +1,36 @@
+import asyncio
+
+from content_core.common import ProcessSourceState
+from content_core.logging import logger
+
+
+async def extract_txt(state: ProcessSourceState):
+    """
+    Parse the text file and extract its content asynchronously.
+    """
+    return_dict = {}
+    if state.file_path is not None and state.identified_type == "text/plain":
+        logger.debug(f"Extracting text from {state.file_path}")
+        file_path = state.file_path
+
+        if file_path is not None:
+            try:
+
+                def _read_file():
+                    with open(file_path, "r", encoding="utf-8") as file:
+                        return file.read()
+
+                # Run file I/O in thread pool
+                content = await asyncio.get_event_loop().run_in_executor(
+                    None, _read_file
+                )
+
+                logger.debug(f"Extracted: {content[:100]}")
+                return_dict["content"] = content
+
+            except FileNotFoundError:
+                raise FileNotFoundError(f"File not found at {file_path}")
+            except Exception as e:
+                raise Exception(f"An error occurred: {e}")
+
+    return return_dict