content-core 1.1.2__py3-none-any.whl → 1.2.1__py3-none-any.whl

content_core/cc_config.yaml

@@ -34,6 +34,10 @@ extraction:
   url_engine: auto  # auto | simple | firecrawl | jina | docling - for URLs
   docling:
     output_format: markdown  # markdown | html | json
+  pymupdf:
+    enable_formula_ocr: false    # Enable OCR for formula-heavy pages (requires Tesseract)
+    formula_threshold: 3         # Minimum formulas per page to trigger OCR
+    ocr_fallback: true          # Gracefully fallback to standard extraction if OCR fails
 
 youtube_transcripts:
   preferred_languages: ["en", "es", "pt"]

content_core/config.py

@@ -6,6 +6,10 @@ from dotenv import load_dotenv
 # Load environment variables from .env file
 load_dotenv()
 
+# Allowed engine values for validation
+ALLOWED_DOCUMENT_ENGINES = {"auto", "simple", "docling"}
+ALLOWED_URL_ENGINES = {"auto", "simple", "firecrawl", "jina"}
+
 
 def load_config():
     config_path = os.environ.get("CCORE_CONFIG_PATH") or os.environ.get("CCORE_MODEL_CONFIG_PATH")
@@ -14,8 +18,8 @@ def load_config():
             with open(config_path, "r") as file:
                 return yaml.safe_load(file)
         except Exception as e:
-            print(f"Erro ao carregar o arquivo de configuração de {config_path}: {e}")
-            print("Usando configurações padrão internas.")
+            print(f"Error loading configuration file from {config_path}: {e}")
+            print("Using internal default settings.")
 
     default_config_data = pkgutil.get_data("content_core", "models_config.yaml")
     if default_config_data:
@@ -33,6 +37,39 @@ def load_config():
 
 CONFIG = load_config()
 
+# Environment variable engine selectors for MCP/Raycast users
+def get_document_engine():
+    """Get document engine with environment variable override and validation."""
+    env_engine = os.environ.get("CCORE_DOCUMENT_ENGINE")
+    if env_engine:
+        if env_engine not in ALLOWED_DOCUMENT_ENGINES:
+            # Import logger here to avoid circular imports
+            from content_core.logging import logger
+            logger.warning(
+                f"Invalid CCORE_DOCUMENT_ENGINE: '{env_engine}'. "
+                f"Allowed values: {', '.join(sorted(ALLOWED_DOCUMENT_ENGINES))}. "
+                f"Using default from config."
+            )
+            return CONFIG.get("extraction", {}).get("document_engine", "auto")
+        return env_engine
+    return CONFIG.get("extraction", {}).get("document_engine", "auto")
+
+def get_url_engine():
+    """Get URL engine with environment variable override and validation."""
+    env_engine = os.environ.get("CCORE_URL_ENGINE")
+    if env_engine:
+        if env_engine not in ALLOWED_URL_ENGINES:
+            # Import logger here to avoid circular imports
+            from content_core.logging import logger
+            logger.warning(
+                f"Invalid CCORE_URL_ENGINE: '{env_engine}'. "
+                f"Allowed values: {', '.join(sorted(ALLOWED_URL_ENGINES))}. "
+                f"Using default from config."
+            )
+            return CONFIG.get("extraction", {}).get("url_engine", "auto")
+        return env_engine
+    return CONFIG.get("extraction", {}).get("url_engine", "auto")
+
 # Programmatic config overrides: use in notebooks or scripts
 def set_document_engine(engine: str):
     """Override the document extraction engine ('auto', 'simple', or 'docling')."""
@@ -47,3 +84,21 @@ def set_docling_output_format(fmt: str):
     extraction = CONFIG.setdefault("extraction", {})
     docling_cfg = extraction.setdefault("docling", {})
     docling_cfg["output_format"] = fmt
+
+def set_pymupdf_ocr_enabled(enabled: bool):
+    """Enable or disable PyMuPDF OCR for formula-heavy pages."""
+    extraction = CONFIG.setdefault("extraction", {})
+    pymupdf_cfg = extraction.setdefault("pymupdf", {})
+    pymupdf_cfg["enable_formula_ocr"] = enabled
+
+def set_pymupdf_formula_threshold(threshold: int):
+    """Set the minimum number of formulas per page to trigger OCR."""
+    extraction = CONFIG.setdefault("extraction", {})
+    pymupdf_cfg = extraction.setdefault("pymupdf", {})
+    pymupdf_cfg["formula_threshold"] = threshold
+
+def set_pymupdf_ocr_fallback(enabled: bool):
+    """Enable or disable fallback to standard extraction when OCR fails."""
+    extraction = CONFIG.setdefault("extraction", {})
+    pymupdf_cfg = extraction.setdefault("pymupdf", {})
+    pymupdf_cfg["ocr_fallback"] = enabled

content_core/content/extraction/graph.py

@@ -12,13 +12,19 @@ from content_core.common import (
     ProcessSourceState,
     UnsupportedTypeException,
 )
-from content_core.config import CONFIG  # type: ignore
+from content_core.config import get_document_engine
 from content_core.logging import logger
 from content_core.processors.audio import extract_audio_data  # type: ignore
-from content_core.processors.docling import (
-    DOCLING_SUPPORTED,  # type: ignore
-    extract_with_docling,
-)
+try:
+    from content_core.processors.docling import (
+        DOCLING_SUPPORTED,  # type: ignore
+        extract_with_docling,
+        DOCLING_AVAILABLE,
+    )
+except ImportError:
+    DOCLING_AVAILABLE = False
+    DOCLING_SUPPORTED = set()
+    extract_with_docling = None
 from content_core.processors.office import (
     SUPPORTED_OFFICE_TYPES,
     extract_office_content,
@@ -126,26 +132,30 @@ async def file_type_router_docling(state: ProcessSourceState) -> str:
     Supports 'auto', 'docling', and 'simple'.
     'auto' tries docling first, then falls back to simple if docling fails.
     """
-    engine = state.document_engine or CONFIG.get("extraction", {}).get("document_engine", "auto")
+    # Use environment-aware engine selection
+    engine = state.document_engine or get_document_engine()
+    
     if engine == "auto":
         logger.debug("Using auto engine")
-        # Try docling first; if it fails or is not supported, fallback to simple
-        if state.identified_type in DOCLING_SUPPORTED:
-            try:
-                logger.debug("Trying docling extraction")
-                return "extract_docling"
-            except Exception as e:
-                logger.warning(
-                    f"Docling extraction failed in 'auto' mode, falling back to simple: {e}"
-                )
+        # Check if docling is available AND supports the file type
+        if DOCLING_AVAILABLE and state.identified_type in DOCLING_SUPPORTED:
+            logger.debug("Using docling extraction (auto mode)")
+            return "extract_docling"
         # Fallback to simple
-        logger.debug("Falling back to simple extraction")
+        logger.debug("Falling back to simple extraction (docling unavailable or unsupported)")
         return await file_type_edge(state)
 
-    if engine == "docling" and state.identified_type in DOCLING_SUPPORTED:
-        logger.debug("Using docling engine")
-        return "extract_docling"
-    # For 'simple', use the default file type edge
+    if engine == "docling":
+        if not DOCLING_AVAILABLE:
+            raise ImportError("Docling engine requested but docling package not installed. Install with: pip install content-core[docling]")
+        if state.identified_type in DOCLING_SUPPORTED:
+            logger.debug("Using docling engine")
+            return "extract_docling"
+        # If docling doesn't support this file type, fall back to simple
+        logger.debug("Docling doesn't support this file type, using simple engine")
+        return await file_type_edge(state)
+    
+    # For 'simple' or any other engine
     logger.debug("Using simple engine")
     return await file_type_edge(state)
 
@@ -168,7 +178,9 @@ workflow.add_node("extract_audio_data", extract_audio_data)
 workflow.add_node("extract_youtube_transcript", extract_youtube_transcript)
 workflow.add_node("delete_file", delete_file)
 workflow.add_node("download_remote_file", download_remote_file)
-workflow.add_node("extract_docling", extract_with_docling)
+# Only add docling node if available
+if DOCLING_AVAILABLE:
+    workflow.add_node("extract_docling", extract_with_docling)
 
 # Add edges
 workflow.add_edge(START, "source")

content_core/notebooks/urls.ipynb

@@ -0,0 +1,154 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "873a872b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from content_core.content.extraction import extract_content\n",
+    "\n",
+    "async def process_url(url):\n",
+    "    print(\"Processing: \", url)\n",
+    "    print(\"Simple: -------\")\n",
+    "    result = await extract_content(dict(url=url, engine=\"simple\"))\n",
+    "    print(result.title[:100])\n",
+    "    print(result.content[:100])\n",
+    "    print(\"Jina: -------\")\n",
+    "    result = await extract_content(dict(url=url, engine=\"jina\"))\n",
+    "    print(result.title[:100])\n",
+    "    print(result.content[:100])\n",
+    "    print(\"Firecrawl: -------\")\n",
+    "    result = await extract_content(dict(url=url, engine=\"firecrawl\"))\n",
+    "    print(result.title[:100])\n",
+    "    print(result.content[:100])\n",
+    "    print(\"=============================\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "263dc3af",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processing:  https://www.supernovalabs.com.br/\n",
+      "Simple: -------\n",
+      "Readability failed: No content extracted by readability\n",
+      "Supernova Labs | AI Consulting\n",
+      "Supernova Labs | AI Consulting\n",
+      "Jina: -------\n",
+      "Supernova Labs | Elite AI Consulting to help you build the Future\n",
+      "URL Source: https://www.supernovalabs.com.br/\n",
+      "\n",
+      "Markdown Content:\n",
+      "Supernova Labs\n",
+      "\n",
+      "[About](https://www\n",
+      "Firecrawl: -------\n",
+      "Supernova Labs | AI Consulting\n",
+      "# Unleash Your AI Edge. Fast.\n",
+      "\n",
+      "We turn your data, tech and capabilities into impact with lean AI sol\n",
+      "=============================\n",
+      "None\n",
+      "Processing:  https://building.nubank.com/fine-tuning-transaction-user-models/\n",
+      "Simple: -------\n",
+      "Fine-Tuning Transaction User Models - Building Nubank\n",
+      "Fine-Tuning Transaction User Models Learn how we combine transaction embeddings with tabular data us\n",
+      "Jina: -------\n",
+      "Fine-Tuning Transaction User Models - Building Nubank\n",
+      "URL Source: https://building.nubank.com/fine-tuning-transaction-user-models/\n",
+      "\n",
+      "Published Time: 2025-0\n",
+      "Firecrawl: -------\n",
+      "Fine-Tuning Transaction User Models - Building Nubank\n",
+      "# Fine-Tuning Transaction User Models\n",
+      "\n",
+      "Learn how we combine transaction embeddings with tabular data\n",
+      "=============================\n",
+      "None\n",
+      "Processing:  https://medium.com/writing-for-profit-with-ai/you-can-make-money-with-ai-without-quitting-your-job-5296bbcb703b\n",
+      "Simple: -------\n",
+      "You Can Make Money With AI Without Quitting Your Job | by Nipuna Maduranga | LearnAIforproft.com | M\n",
+      "Most people think they need to quit their job to build a new life. I thought that too. You scroll th\n",
+      "Jina: -------\n",
+      "You Can Make Money With AI Without Quitting Your Job\n",
+      "URL Source: https://medium.com/writing-for-profit-with-ai/you-can-make-money-with-ai-without-quittin\n",
+      "Firecrawl: -------\n",
+      "You Can Make Money With AI Without Quitting Your Job | by Nipuna Maduranga | LearnAIforproft.com | M\n",
+      "[Sitemap](https://medium.com/sitemap/sitemap.xml)\n",
+      "\n",
+      "[Open in app](https://rsci.app.link/?%24canonical\n",
+      "=============================\n",
+      "None\n",
+      "Processing:  https://github.com/mirkonasato/pyodconverter\n",
+      "Simple: -------\n",
+      "GitHub - mirkonasato/pyodconverter: Python script to automate document conversions using LibreOffice\n",
+      "This repository was archived by the owner on Dec 1, 2023. It is now read-only. mirkonasato/pyodconve\n",
+      "Jina: -------\n",
+      "GitHub - mirkonasato/pyodconverter: Python script to automate document conversions using LibreOffice\n",
+      "URL Source: https://github.com/mirkonasato/pyodconverter\n",
+      "\n",
+      "Markdown Content:\n",
+      "GitHub - mirkonasato/pyo\n",
+      "Firecrawl: -------\n",
+      "GitHub - mirkonasato/pyodconverter: Python script to automate document conversions using LibreOffice\n",
+      "[Skip to content](https://github.com/mirkonasato/pyodconverter#start-of-content)\n",
+      "\n",
+      "You signed in with\n",
+      "=============================\n",
+      "None\n",
+      "Processing:  https://www.amazon.com.br/Ultra-aprendizado-habilidades-valiosas-competi%C3%A7%C3%A3o-carreira/dp/6555110058/ref=asc_df_6555110058?tag=googleshopp00-20&hvadid=709857900630&hvpos=&hvnetw=g&hvrand=17798174883330212364&hvpone=&hvptwo=&hvqmt=&hvdev=c&hvdvcmdl=&hvlocint=&hvlocphy=9195894&hvtargid=pla-1148630207439&psc=1&language=pt_BR\n",
+      "Simple: -------\n",
+      "Error processing URL https://www.amazon.com.br/Ultra-aprendizado-habilidades-valiosas-competi%C3%A7%C3%A3o-carreira/dp/6555110058/ref=asc_df_6555110058?tag=googleshopp00-20&hvadid=709857900630&hvpos=&hvnetw=g&hvrand=17798174883330212364&hvpone=&hvptwo=&hvqmt=&hvdev=c&hvdvcmdl=&hvlocint=&hvlocphy=9195894&hvtargid=pla-1148630207439&psc=1&language=pt_BR: HTTP error: 500\n",
+      "Error\n",
+      "Failed to extract content: HTTP error: 500\n",
+      "Jina: -------\n",
+      "Ultra-aprendizado: domine habilidades valiosas, seja mais esperto que a competição e dê um impulso n\n",
+      "URL Source: https://www.amazon.com.br/Ultra-aprendizado-habilidades-valiosas-competi%C3%A7%C3%A3o-ca\n",
+      "Firecrawl: -------\n",
+      "Amazon.com.br\n",
+      "#### Digite os caracteres que você vê abaixo\n",
+      "\n",
+      "Desculpe pelo inconveniente. Para continuar realizando\n",
+      "=============================\n",
+      "None\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "urls= [\"https://www.supernovalabs.com.br/\", \"https://building.nubank.com/fine-tuning-transaction-user-models/\", \"https://medium.com/writing-for-profit-with-ai/you-can-make-money-with-ai-without-quitting-your-job-5296bbcb703b\", \"https://github.com/mirkonasato/pyodconverter\",  \"https://www.amazon.com.br/Ultra-aprendizado-habilidades-valiosas-competi%C3%A7%C3%A3o-carreira/dp/6555110058/ref=asc_df_6555110058?tag=googleshopp00-20&hvadid=709857900630&hvpos=&hvnetw=g&hvrand=17798174883330212364&hvpone=&hvptwo=&hvqmt=&hvdev=c&hvdvcmdl=&hvlocint=&hvlocphy=9195894&hvtargid=pla-1148630207439&psc=1&language=pt_BR\"]\n",
+    "for url in urls:\n",
+    "    result = await process_url(url=url)\n",
+    "    print(result)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

content_core/processors/docling.py

@@ -2,22 +2,29 @@
 Docling-based document extraction processor.
 """
 
+from content_core.common.state import ProcessSourceState
+from content_core.config import CONFIG
+
+DOCLING_AVAILABLE = False
 try:
     from docling.document_converter import DocumentConverter
+    DOCLING_AVAILABLE = True
 except ImportError:
 
     class DocumentConverter:
         """Stub when docling is not installed."""
 
         def __init__(self):
-            raise ImportError("Docling not installed")
+            raise ImportError(
+                "Docling not installed. Install with: pip install content-core[docling] "
+                "or use CCORE_DOCUMENT_ENGINE=simple to skip docling."
+            )
 
         def convert(self, source: str):
-            raise ImportError("Docling not installed")
-
-
-from content_core.common.state import ProcessSourceState
-from content_core.config import CONFIG
+            raise ImportError(
+                "Docling not installed. Install with: pip install content-core[docling] "
+                "or use CCORE_DOCUMENT_ENGINE=simple to skip docling."
+            )
 
 # Supported MIME types for Docling extraction
 DOCLING_SUPPORTED = {

content_core/processors/pdf.py

@@ -5,20 +5,90 @@ import unicodedata
 import fitz  # type: ignore
 
 from content_core.common import ProcessSourceState
+from content_core.config import CONFIG
 from content_core.logging import logger
 
-# todo: find tables - https://pymupdf.readthedocs.io/en/latest/the-basics.html#extracting-tables-from-a-page
-# todo: what else can we do to make the text more readable?
-# todo: try to fix encoding for some PDF that is still breaking
-# def _extract_text_from_pdf(pdf_path):
-#     doc = fitz.open(pdf_path)
-#     text = ""
-#     logger.debug(f"Found {len(doc)} pages in PDF")
-#     for page in doc:
-#         # Use encode/decode if you need to clean up any encoding issues
-#         text += page.get_text().encode('utf-8').decode('utf-8')
-#     doc.close()
-#     return text
+def count_formula_placeholders(text):
+    """
+    Count the number of formula placeholders in extracted text.
+    
+    Args:
+        text (str): Extracted text content
+    Returns:
+        int: Number of formula placeholders found
+    """
+    if not text:
+        return 0
+    return text.count('<!-- formula-not-decoded -->')
+
+
+def extract_page_with_ocr(page, page_num):
+    """
+    Extract text from a page using OCR (Tesseract).
+    
+    Args:
+        page: PyMuPDF page object
+        page_num (int): Page number for logging
+    Returns:
+        str: OCR-extracted text or None if OCR fails
+    """
+    try:
+        logger.debug(f"Attempting OCR extraction for page {page_num}")
+        # Create TextPage using OCR
+        textpage = page.get_textpage_ocr()
+        if textpage:
+            # Extract text from the OCR TextPage
+            ocr_text = textpage.extractText()
+            logger.debug(f"OCR successful for page {page_num}, extracted {len(ocr_text)} characters")
+            return ocr_text
+        else:
+            logger.warning(f"OCR TextPage creation failed for page {page_num}")
+            return None
+    except (ImportError, RuntimeError, OSError) as e:
+        # Common errors: Tesseract not installed, OCR failure, file access issues
+        logger.debug(f"OCR extraction failed for page {page_num}: {e}")
+        return None
+    except Exception as e:
+        # Unexpected errors - log as warning for debugging
+        logger.warning(f"Unexpected error during OCR extraction for page {page_num}: {e}")
+        return None
+
+
+def convert_table_to_markdown(table):
+    """
+    Convert a PyMuPDF table to markdown format.
+    
+    Args:
+        table: Table data from PyMuPDF (list of lists)
+    Returns:
+        str: Markdown-formatted table
+    """
+    if not table or not table[0]:
+        return ""
+    
+    # Build markdown table
+    markdown_lines = []
+    
+    # Header row
+    header = table[0]
+    header_row = "| " + " | ".join(str(cell) if cell else "" for cell in header) + " |"
+    markdown_lines.append(header_row)
+    
+    # Separator row
+    separator = "|" + "|".join([" --- " for _ in header]) + "|"
+    markdown_lines.append(separator)
+    
+    # Data rows
+    for row in table[1:]:
+        if row:  # Skip empty rows
+            row_text = "| " + " | ".join(str(cell) if cell else "" for cell in row) + " |"
+            markdown_lines.append(row_text)
+    
+    return "\n".join(markdown_lines) + "\n"
+
+# Configuration constants
+DEFAULT_FORMULA_THRESHOLD = 3
+DEFAULT_OCR_FALLBACK = True
 
 SUPPORTED_FITZ_TYPES = [
     "application/pdf",
@@ -116,30 +186,84 @@ def clean_pdf_text(text):
     return text.strip()
 
 
-async def _extract_text_from_pdf(pdf_path):
-    doc = fitz.open(pdf_path)
-    try:
-        text = ""
-        logger.debug(f"Found {len(doc)} pages in PDF")
-        for page in doc:
-            text += page.get_text()
-        normalized_text = clean_pdf_text(text)
-        return normalized_text
-    finally:
-        doc.close()
 
 
 async def _extract_text_from_pdf(pdf_path):
-    """Extract text from PDF asynchronously"""
+    """Extract text from PDF asynchronously with table detection"""
 
     def _extract():
         doc = fitz.open(pdf_path)
         try:
-            text = ""
+            full_text = []
             logger.debug(f"Found {len(doc)} pages in PDF")
-            for page in doc:
-                text += page.get_text()
-            return clean_pdf_text(text)
+            
+            # Use quality improvement flags for better text extraction
+            extraction_flags = (
+                fitz.TEXT_PRESERVE_LIGATURES |  # Better character rendering
+                fitz.TEXT_PRESERVE_WHITESPACE |  # Better spacing preservation
+                fitz.TEXT_PRESERVE_IMAGES       # Better image-text integration
+            )
+            
+            # Get OCR configuration
+            ocr_config = CONFIG.get('extraction', {}).get('pymupdf', {})
+            enable_ocr = ocr_config.get('enable_formula_ocr', False)
+            formula_threshold = ocr_config.get('formula_threshold', DEFAULT_FORMULA_THRESHOLD)
+            ocr_fallback = ocr_config.get('ocr_fallback', DEFAULT_OCR_FALLBACK)
+            
+            for page_num, page in enumerate(doc):
+                # Extract regular text with quality flags
+                standard_text = page.get_text(flags=extraction_flags)
+                
+                # Check if we should try OCR for this page
+                formula_count = count_formula_placeholders(standard_text)
+                use_ocr = (enable_ocr and 
+                          formula_count >= formula_threshold and 
+                          formula_count > 0)
+                
+                if use_ocr:
+                    logger.debug(f"Page {page_num + 1} has {formula_count} formulas, attempting OCR")
+                    ocr_text = extract_page_with_ocr(page, page_num + 1)
+                    
+                    if ocr_text and ocr_fallback:
+                        # Use OCR text but preserve table extraction from standard text
+                        page_text = ocr_text
+                        logger.debug(f"Using OCR text for page {page_num + 1}")
+                    else:
+                        # OCR failed, use standard text
+                        page_text = standard_text
+                        if not ocr_text:
+                            logger.debug(f"OCR failed for page {page_num + 1}, using standard extraction")
+                else:
+                    page_text = standard_text
+                
+                # Try to find and extract tables (regardless of OCR)
+                try:
+                    tables = page.find_tables()
+                    if tables:
+                        logger.debug(f"Found {len(tables)} table(s) on page {page_num + 1}")
+                        
+                        # For each table found, convert to markdown and append
+                        for table_num, table in enumerate(tables):
+                            # Extract table data
+                            table_data = table.extract()
+                            # Validate table has actual content (not just empty rows/cells)
+                            if table_data and len(table_data) > 0 and any(
+                                any(str(cell).strip() for cell in row if cell) for row in table_data if row
+                            ):
+                                # Add a marker before the table
+                                page_text += f"\n\n[Table {table_num + 1} from page {page_num + 1}]\n"
+                                # Convert to markdown
+                                markdown_table = convert_table_to_markdown(table_data)
+                                page_text += markdown_table + "\n"
+                except Exception as e:
+                    # If table extraction fails, continue with regular text
+                    logger.debug(f"Table extraction failed on page {page_num + 1}: {e}")
+                
+                full_text.append(page_text)
+            
+            # Join all pages and clean
+            combined_text = "".join(full_text)
+            return clean_pdf_text(combined_text)
         finally:
             doc.close()
 

content_core/processors/url.py

@@ -5,7 +5,7 @@ from bs4 import BeautifulSoup
 from readability import Document
 
 from content_core.common import ProcessSourceState
-from content_core.config import CONFIG
+from content_core.config import get_url_engine
 from content_core.logging import logger
 from content_core.processors.docling import DOCLING_SUPPORTED
 from content_core.processors.office import SUPPORTED_OFFICE_TYPES
@@ -165,7 +165,8 @@ async def extract_url(state: ProcessSourceState):
     """
     assert state.url, "No URL provided"
     url = state.url
-    engine = state.url_engine or CONFIG.get("extraction", {}).get("url_engine", "auto")
+    # Use environment-aware engine selection
+    engine = state.url_engine or get_url_engine()
     try:
         if engine == "auto":
             if os.environ.get("FIRECRAWL_API_KEY"):

{content_core-1.1.2.dist-info → content_core-1.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: content-core
-Version: 1.1.2
+Version: 1.2.1
 Summary: Extract what matters from any media source. Available as Python Library, macOS Service, CLI and MCP Server
 Author-email: LUIS NOVO <lfnovo@gmail.com>
 License-File: LICENSE
@@ -10,7 +10,6 @@ Requires-Dist: aiohttp>=3.11
 Requires-Dist: asciidoc>=10.2.1
 Requires-Dist: bs4>=0.0.2
 Requires-Dist: dicttoxml>=1.7.16
-Requires-Dist: docling>=2.34.0
 Requires-Dist: esperanto>=1.2.0
 Requires-Dist: firecrawl-py>=2.7.0
 Requires-Dist: jinja2>=3.1.6
@@ -31,6 +30,8 @@ Requires-Dist: pytubefix>=9.1.1
 Requires-Dist: readability-lxml>=0.8.4.1
 Requires-Dist: validators>=0.34.0
 Requires-Dist: youtube-transcript-api>=1.0.3
+Provides-Extra: docling
+Requires-Dist: docling>=2.34.0; extra == 'docling'
 Provides-Extra: mcp
 Requires-Dist: fastmcp>=0.5.0; extra == 'mcp'
 Description-Content-Type: text/markdown
@@ -39,29 +40,70 @@ Description-Content-Type: text/markdown
 
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 
-**Content Core** is a versatile Python library designed to extract and process content from various sources, providing a unified interface for handling text, web pages, and local files.
+**Content Core** is a powerful, AI-powered content extraction and processing platform that transforms any source into clean, structured content. Extract text from websites, transcribe videos, process documents, and generate AI summaries—all through a unified interface with multiple integration options.
 
-## Overview
+## 🚀 What You Can Do
 
-> **Note:** As of v0.8, the default extraction engine is `'auto'`. Content Core will automatically select the best extraction method based on your environment and available API keys, with a smart fallback order for both URLs and files. For files/documents, `'auto'` now tries Docling first, then falls back to simple extraction. You can override the engine if needed, but `'auto'` is recommended for most users.
+**Extract content from anywhere:**
+- 📄 **Documents** - PDF, Word, PowerPoint, Excel, Markdown, HTML, EPUB
+- 🎥 **Media** - Videos (MP4, AVI, MOV) with automatic transcription  
+- 🎵 **Audio** - MP3, WAV, M4A with speech-to-text conversion
+- 🌐 **Web** - Any URL with intelligent content extraction
+- 🖼️ **Images** - JPG, PNG, TIFF with OCR text recognition
+- 📦 **Archives** - ZIP, TAR, GZ with content analysis
 
-The primary goal of Content Core is to simplify the process of ingesting content from diverse origins. Whether you have raw text, a URL pointing to an article, or a local file like a video or markdown document, Content Core aims to extract the meaningful content for further use.
+**Process with AI:**
+- ✨ **Clean & format** extracted content automatically
+- 📝 **Generate summaries** with customizable styles (bullet points, executive summary, etc.)
+- 🎯 **Context-aware processing** - explain to a child, technical summary, action items
+- 🔄 **Smart engine selection** - automatically chooses the best extraction method
 
-## Key Features
+## 🛠️ Multiple Ways to Use
 
-*   **Multi-Source Extraction:** Handles content from:
-    *   Direct text strings.
-    *   Web URLs (using robust extraction methods).
-    *   Local files (including automatic transcription for video/audio files and parsing for text-based formats).
-*   **Intelligent Processing:** Applies appropriate extraction techniques based on the source type. See the [Processors Documentation](./docs/processors.md) for detailed information on how different content types are handled.
-*   **Smart Engine Selection:** By default, Content Core uses the `'auto'` engine, which:
-    * For URLs: Uses Firecrawl if `FIRECRAWL_API_KEY` is set, else tries Jina. Jina might fail because of rate limits, which can be fixed by adding `JINA_API_KEY`. If Jina failes, BeautifulSoup is used as a fallback.
-    * For files: Tries Docling extraction first (for robust document parsing), then falls back to simple extraction if needed.
-    * You can override this by specifying an engine, but `'auto'` is recommended for most users.
-*   **Content Cleaning (Optional):** Likely integrates with LLMs (via `prompter.py` and Jinja templates) to refine and clean the extracted content.
-*   **MCP Server:** Includes a Model Context Protocol (MCP) server for seamless integration with Claude Desktop and other MCP-compatible applications.
-*   **macOS Services:** Right-click context menu integration for Finder (extract and summarize files directly).
-*   **Asynchronous:** Built with `asyncio` for efficient I/O operations.
+### 🖥️ Command Line (Zero Install)
+```bash
+# Extract content from any source
+uvx --from "content-core" ccore https://example.com
+uvx --from "content-core" ccore document.pdf
+
+# Generate AI summaries  
+uvx --from "content-core" csum video.mp4 --context "bullet points"
+```
+
+### 🤖 Claude Desktop Integration
+One-click setup with Model Context Protocol (MCP) - extract content directly in Claude conversations.
+
+### 🔍 Raycast Extension  
+Smart auto-detection commands:
+- **Extract Content** - Full interface with format options
+- **Summarize Content** - 9 summary styles available
+- **Quick Extract** - Instant clipboard extraction
+
+### 🖱️ macOS Right-Click Integration
+Right-click any file in Finder → Services → Extract or Summarize content instantly.
+
+### 🐍 Python Library
+```python
+import content_core as cc
+
+# Extract from any source
+result = await cc.extract("https://example.com/article")
+summary = await cc.summarize_content(result, context="explain to a child")
+```
+
+## ⚡ Key Features
+
+*   **🎯 Intelligent Auto-Detection:** Automatically selects the best extraction method based on content type and available services
+*   **🔧 Smart Engine Selection:** 
+    * **URLs:** Firecrawl → Jina → BeautifulSoup fallback chain
+    * **Documents:** Docling → Enhanced PyMuPDF → Simple extraction fallback  
+    * **Media:** OpenAI Whisper transcription
+    * **Images:** OCR with multiple engine support
+*   **📊 Enhanced PDF Processing:** Advanced PyMuPDF engine with quality flags, table detection, and optional OCR for mathematical formulas
+*   **🌍 Multiple Integrations:** CLI, Python library, MCP server, Raycast extension, macOS Services
+*   **⚡ Zero-Install Options:** Use `uvx` for instant access without installation
+*   **🧠 AI-Powered Processing:** LLM integration for content cleaning and summarization
+*   **🔄 Asynchronous:** Built with `asyncio` for efficient processing
 
 ## Getting Started
 
@@ -70,11 +112,17 @@ The primary goal of Content Core is to simplify the process of ingesting content
 Install Content Core using `pip`:
 
 ```bash
-# Install the package
+# Basic installation (PyMuPDF + BeautifulSoup/Jina extraction)
 pip install content-core
 
-# Install with MCP server support
+# With enhanced document processing (adds Docling)
+pip install content-core[docling]
+
+# With MCP server support
 pip install content-core[mcp]
+
+# Full installation
+pip install content-core[docling,mcp]
 ```
 
 Alternatively, if you’re developing locally:
@@ -245,6 +293,49 @@ Add to your `claude_desktop_config.json`:
 
 For detailed setup instructions, configuration options, and usage examples, see our [MCP Documentation](docs/mcp.md).
 
+## Enhanced PDF Processing
+
+Content Core features an optimized PyMuPDF extraction engine with significant improvements for scientific documents and complex PDFs.
+
+### Key Improvements
+
+- **🔬 Mathematical Formula Extraction**: Enhanced quality flags eliminate `<!-- formula-not-decoded -->` placeholders
+- **📊 Automatic Table Detection**: Tables converted to markdown format for LLM consumption
+- **🔧 Quality Text Rendering**: Better ligature, whitespace, and image-text integration
+- **⚡ Optional OCR Enhancement**: Selective OCR for formula-heavy pages (requires Tesseract)
+
+### Configuration for Scientific Documents
+
+For documents with heavy mathematical content, enable OCR enhancement:
+
+```yaml
+# In cc_config.yaml
+extraction:
+  pymupdf:
+    enable_formula_ocr: true      # Enable OCR for formula-heavy pages
+    formula_threshold: 3          # Min formulas per page to trigger OCR
+    ocr_fallback: true           # Graceful fallback if OCR fails
+```
+
+```python
+# Runtime configuration
+from content_core.config import set_pymupdf_ocr_enabled
+set_pymupdf_ocr_enabled(True)
+```
+
+### Requirements for OCR Enhancement
+
+```bash
+# Install Tesseract OCR (optional, for formula enhancement)
+# macOS
+brew install tesseract
+
+# Ubuntu/Debian
+sudo apt-get install tesseract-ocr
+```
+
+**Note**: OCR is optional - you get improved PDF extraction automatically without any additional setup.
+
 ## macOS Services Integration
 
 Content Core provides powerful right-click integration with macOS Finder, allowing you to extract and summarize content from any file without installation. Choose between clipboard or TextEdit output for maximum flexibility.
@@ -288,6 +379,50 @@ Create **4 convenient services** for different workflows:
 
 For complete setup instructions with copy-paste scripts, see [macOS Services Documentation](docs/macos.md).
 
+## Raycast Extension
+
+Content Core provides a powerful Raycast extension with smart auto-detection that handles both URLs and file paths seamlessly. Extract and summarize content directly from your Raycast interface without switching applications.
+
+### Quick Setup
+
+**From Raycast Store** (coming soon):
+1. Open Raycast and search for "Content Core"
+2. Install the extension by `luis_novo`
+3. Configure API keys in preferences
+
+**Manual Installation**:
+1. Download the extension from the repository
+2. Open Raycast → "Import Extension"
+3. Select the `raycast-content-core` folder
+
+### Commands
+
+**🔍 Extract Content** - Smart URL/file detection with full interface
+- Auto-detects URLs vs file paths in real-time
+- Multiple output formats (Text, JSON, XML)
+- Drag & drop support for files
+- Rich results view with metadata
+
+**📝 Summarize Content** - AI-powered summaries with customizable styles  
+- 9 different summary styles (bullet points, executive summary, etc.)
+- Auto-detects source type with visual feedback
+- One-click snippet creation and quicklinks
+
+**⚡ Quick Extract** - Instant extraction to clipboard
+- Type → Tab → Paste source → Enter
+- No UI, works directly from command bar
+- Perfect for quick workflows
+
+### Features
+
+- **Smart Auto-Detection**: Instantly recognizes URLs vs file paths
+- **Zero Installation**: Uses `uvx` for Content Core execution
+- **Rich Integration**: Keyboard shortcuts, clipboard actions, Raycast snippets
+- **All File Types**: Documents, videos, audio, images, archives
+- **Visual Feedback**: Real-time type detection with icons
+
+For detailed setup, configuration, and usage examples, see [Raycast Extension Documentation](docs/raycast.md).
+
 ## Using with Langchain
 
 For users integrating with the [Langchain](https://python.langchain.com/) framework, `content-core` exposes a set of compatible tools. These tools, located in the `src/content_core/tools` directory, allow you to leverage `content-core` extraction, cleaning, and summarization capabilities directly within your Langchain agents and chains.
@@ -397,8 +532,21 @@ Example `.env`:
 ```plaintext
 OPENAI_API_KEY=your-key-here
 GOOGLE_API_KEY=your-key-here
+
+# Engine Selection (optional)
+CCORE_DOCUMENT_ENGINE=auto  # auto, simple, docling
+CCORE_URL_ENGINE=auto       # auto, simple, firecrawl, jina
 ```
 
+### Engine Selection via Environment Variables
+
+For deployment scenarios like MCP servers or Raycast extensions, you can override the extraction engines using environment variables:
+
+- **`CCORE_DOCUMENT_ENGINE`**: Force document engine (`auto`, `simple`, `docling`)
+- **`CCORE_URL_ENGINE`**: Force URL engine (`auto`, `simple`, `firecrawl`, `jina`)
+
+These variables take precedence over config file settings and provide explicit control for different deployment scenarios.
+
 ### Custom Prompt Templates
 
 Content Core allows you to define custom prompt templates for content processing. By default, the library uses built-in prompts located in the `prompts` directory. However, you can create your own prompt templates and store them in a dedicated directory. To specify the location of your custom prompts, set the `PROMPT_PATH` environment variable in your `.env` file or system environment.

{content_core-1.1.2.dist-info → content_core-1.2.1.dist-info}/RECORD

@@ -1,6 +1,6 @@
 content_core/__init__.py,sha256=t4xFo9f3uB2FD1tdR-7ruhMW9_ciJawQReK6iFXWfR0,6531
-content_core/cc_config.yaml,sha256=gGSPM-oO6GIHyCfDCH-cN72BgPJiRmZMgwPrrLhUmfU,851
-content_core/config.py,sha256=vyx0fioR6r0mcZfVdwAFDhFrRNoG0ZNG8RNxIDnhNlo,1802
+content_core/cc_config.yaml,sha256=hjTt5z1Z9b5LShVIqNT3OiAnTAdmr0LB5y8RTyH-fNA,1119
+content_core/config.py,sha256=3XAsMF3EhDJ6aCpzk1UZG_m3-SFdYe3cHiDPH7eVGwQ,4312
 content_core/logging.py,sha256=oeRdWKknEolptopxF1IvnEGEc0ZUw45QXYUEZ71GcdY,438
 content_core/models.py,sha256=Kt6tWdAX87eQ2tL6eTwcHU7_NIRnN4exP4RzV2WrMig,881
 content_core/models_config.yaml,sha256=Yr-GS94ffxnkaWojUfpErUMM7m_MShsYjR6QuDjMzwo,444
@@ -15,27 +15,28 @@ content_core/content/__init__.py,sha256=7IxfLTUHKyHjoT4MfWM2PX2J3QBeYcuERzE9vFeF
 content_core/content/cleanup/__init__.py,sha256=wymD24WLDDdsZrv-5WhparSiHBK9SJCcqBHmokuZqk4,121
 content_core/content/cleanup/core.py,sha256=AXUGUWxGob8si5uKRnDrreOcHV_gbGJr4YnRsNm2GX0,531
 content_core/content/extraction/__init__.py,sha256=TaYw6CAcG62GZfsJxeZ6VJDLP85BU2a7_G271v6WWPk,446
-content_core/content/extraction/graph.py,sha256=Nn2iaQc6YJ4Qt8WKTolwUQUNNqUlwpV8YnijESGvnD0,7605
+content_core/content/extraction/graph.py,sha256=sjk6NpzOMOzMbUOM0bqrDSlB3cLQzboviLDNbj48pjY,8074
 content_core/content/identification/__init__.py,sha256=x4n8JIjDwmPvAopEEEcmZjlozg-zGbMq_s9VYdBjzYU,169
 content_core/content/summary/__init__.py,sha256=ReKCZWKfDtqlInKeh87Y1DEfiNzVWabGybEz3hS2FrI,114
 content_core/content/summary/core.py,sha256=kEabpETljzUb-yf0NcVWTOuCtayESo74gGBVDX7YTFs,550
 content_core/mcp/__init__.py,sha256=KNZYH4F9AoW1Orw1BtO3n92Cn-127hI7iF9gnGadueU,95
 content_core/mcp/server.py,sha256=ql0uXHkIbZlHQUhUQ4CaRnj19xT6t8ErydWntFgmtUg,7021
 content_core/notebooks/run.ipynb,sha256=WPBNcQUNXR5MldNMghVcU4vE4ibrVmlANa80baQn8TA,371078
+content_core/notebooks/urls.ipynb,sha256=gSmiSzmbol_Li36w8tpUsy5QgRbrnBx94Ry2zHwMvwY,7107
 content_core/processors/audio.py,sha256=Mie20g_2Akhw6BHBVo3sHMpDRYUkqBI72lEDakscx3s,5729
-content_core/processors/docling.py,sha256=dkXehsQdfyWXfrK1K_6Pye50ABM7DxMk6TMguabM9Pc,2151
+content_core/processors/docling.py,sha256=lf_NHh255gn4d2EymJYqyH2QiAgQDiJCY3t6Ne7R9rU,2507
 content_core/processors/office.py,sha256=DXkfmjqUhmhP6rJaO5Z5Y9sv-iK0zaPZ3waynFIPtsk,12153
-content_core/processors/pdf.py,sha256=9jf-eROAqw6yQwdlbsxPXsaJXY26hVG7nSTPH9n4afY,5301
+content_core/processors/pdf.py,sha256=TTDhfV2INtXumFDjLJFNMRfpbJ_tqwIcSBDzuThKxJI,10617
 content_core/processors/text.py,sha256=kKHA60-NYjLmCTYUnk8TdJxQQ0Shkg-K61Ezqaelz7k,1158
-content_core/processors/url.py,sha256=6WT8Sw2VHiKyhgWXi_jZjKjwnT_QPSPcH4P99RKbjgU,7521
+content_core/processors/url.py,sha256=YoWw2CjZbqSKBi1CpY0Qowu4hfqGVGJjLZEXUjz7wxs,7536
 content_core/processors/video.py,sha256=3WnZwTswvTLm8PtQhKwoqJ2BH6YZi62dMUjALwJiebo,5196
 content_core/processors/youtube.py,sha256=MOeZboVfM9_C87L5mnUVvsbQeKoznwJoYn1wP1_hA_U,7869
 content_core/tools/__init__.py,sha256=DuJmd7fE-NpDvLP8IW1XY5MUkAQcdks52rn2jk4N8jQ,231
 content_core/tools/cleanup.py,sha256=5IdKedsFyRQMdYzgFSKtsfyxJldbroXQXHesHICNENI,523
 content_core/tools/extract.py,sha256=-r2_jsuMMXyXxGVqWhh1ilNPo_UMYAbw3Pkp1FzPy5g,577
 content_core/tools/summarize.py,sha256=DPfeglLWB08q8SvHrsKpOKZ35XjduUDs2J02ISwjdj0,596
-content_core-1.1.2.dist-info/METADATA,sha256=_0Rg4yeU-05hDB_91dvcMXYKMaKcMcU5C8SpkYhtiRs,15072
-content_core-1.1.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-content_core-1.1.2.dist-info/entry_points.txt,sha256=ifbBxw37b7gAxZXoduS15KtqHuMHuU58STRkEmgM2zA,147
-content_core-1.1.2.dist-info/licenses/LICENSE,sha256=myj0z2T4qIkenCgLsRfx7Wk6UqCQNj5c7O14Qx4zpGg,1066
-content_core-1.1.2.dist-info/RECORD,,
+content_core-1.2.1.dist-info/METADATA,sha256=1LpANnMvECxIekt6kKQr0hnZ1ULGaD2xEmhRh_uzTdk,19676
+content_core-1.2.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+content_core-1.2.1.dist-info/entry_points.txt,sha256=ifbBxw37b7gAxZXoduS15KtqHuMHuU58STRkEmgM2zA,147
+content_core-1.2.1.dist-info/licenses/LICENSE,sha256=myj0z2T4qIkenCgLsRfx7Wk6UqCQNj5c7O14Qx4zpGg,1066
+content_core-1.2.1.dist-info/RECORD,,