thinkpdf 1.0.1__py3-none-any.whl

pdfbrain/core/utils.py

@@ -0,0 +1,229 @@
+"""Utility helpers for pdfmd.
+
+Provides small, side-effect-free helpers used across modules:
+- OS-aware path display for GUI/CLI logs.
+- Simple logging and progress callbacks.
+- Generic regex/text helpers.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+import re
+from pathlib import Path
+from typing import Callable, Optional
+
+
+# ---------------------------------------------------------------------------
+# OS / PATH HELPERS
+# ---------------------------------------------------------------------------
+
+
+def is_windows() -> bool:
+    """Return True if running on Windows."""
+    return os.name == "nt" or sys.platform.lower().startswith("win")
+
+
+def os_display_path(p: os.PathLike | str) -> str:
+    """Return a user-facing path string with OS-appropriate separators.
+
+    On Windows: backslashes (\\)
+    On POSIX:   forward slashes (/)
+    """
+    s = str(p)
+    if not s:
+        return s
+
+    if is_windows():
+        # Normalize to backslashes
+        s = s.replace("/", "\\")
+    else:
+        # Normalize to forward slashes
+        s = s.replace("\\", "/")
+    return s
+
+
+def safe_join(*parts: str | os.PathLike) -> str:
+    """Join path parts, skipping empty segments."""
+    cleaned = [str(p) for p in parts if p not in (None, "", ".")]
+    if not cleaned:
+        return ""
+    return str(Path(cleaned[0]).joinpath(*cleaned[1:]))
+
+
+# ---------------------------------------------------------------------------
+# LOGGING / PROGRESS
+# ---------------------------------------------------------------------------
+
+# These are deliberately simple so they work in CLI and GUI contexts.
+
+
+def log(message: str) -> None:
+    """Print a log message to stderr, prefixed for clarity."""
+    text = str(message)
+    sys.stderr.write(f"[pdf_to_md] {text}\n")
+    sys.stderr.flush()
+
+
+def progress(done: int, total: int) -> None:
+    """Simple textual progress callback.
+
+    Other modules can pass this into long-running operations.
+    """
+    if total <= 0:
+        pct = 0.0
+    else:
+        pct = (done / total) * 100.0
+    sys.stderr.write(f"[pdf_to_md] Progress: {done}/{total} ({pct:.1f}%)\r")
+    sys.stderr.flush()
+    if done >= total:
+        sys.stderr.write("\n")
+        sys.stderr.flush()
+
+
+def clear_console() -> None:
+    """Clear the terminal/console screen, best-effort."""
+    try:
+        if is_windows():
+            os.system("cls")
+        else:
+            os.system("clear")
+    except Exception:
+        # Never crash on a cosmetic operation.
+        pass
+
+
+def print_error(message: str) -> None:
+    """Print an error message to stderr in a consistent format."""
+    sys.stderr.write(f"[pdf_to_md:ERROR] {message}\n")
+    sys.stderr.flush()
+
+
+# ---------------------------------------------------------------------------
+# TEXT NORMALIZATION
+# ---------------------------------------------------------------------------
+
+
+_PUNCT_MAP = {
+    # Quotes
+    "\u2018": "'",  # ‘
+    "\u2019": "'",  # ’
+    "\u201c": '"',  # “
+    "\u201d": '"',  # ”
+    # Dashes
+    "\u2013": "-",  # –
+    "\u2014": "-",  # —
+    # Ellipsis
+    "\u2026": "...",  # …
+}
+
+
+def normalize_punctuation(text: str) -> str:
+    """Normalize common Unicode punctuation to simpler ASCII forms.
+
+    This keeps the output predictable in Markdown and text editors.
+    """
+    if not text:
+        return text
+    out = []
+    for ch in text:
+        out.append(_PUNCT_MAP.get(ch, ch))
+    return "".join(out)
+
+
+_URL_RE = re.compile(
+    r"(?P<url>(https?://[^\s<>]+|www\.[^\s<>]+))",
+    re.IGNORECASE,
+)
+
+
+def linkify_urls(text: str) -> str:
+    """Wrap bare URLs in Markdown-friendly form.
+
+    Example:
+        'See https://example.com' → 'See <https://example.com>'
+    """
+
+    def _repl(match: re.Match) -> str:
+        url = match.group("url")
+        # Avoid double-wrapping if already inside <...>
+        if url.startswith("<") and url.endswith(">"):
+            return url
+        # If it's a www. URL, add scheme for safety
+        if url.lower().startswith("www."):
+            return f"<https://{url}>"
+        return f"<{url}>"
+
+    return _URL_RE.sub(_repl, text)
+
+
+# ---------------------------------------------------------------------------
+# MARKDOWN ESCAPING
+# ---------------------------------------------------------------------------
+
+
+def escape_markdown(text: str) -> str:
+    """Escape only the minimal set of characters that break Markdown.
+
+    IMPORTANT:
+    - We do NOT escape periods, parentheses, hyphens, or '#'.
+    - We avoid the old behaviour that produced '\\.' and '\\(' everywhere.
+    - We only escape characters that are truly dangerous inside plain text.
+
+    This function is called on raw PDF span text BEFORE we add **bold** or *italic*
+    markers in the renderer.
+    """
+    if not text:
+        return text
+
+    # Characters we actually want to escape:
+    # - backslash itself
+    # - backtick (inline code)
+    # - asterisk and underscore (emphasis)
+    # - curly braces, brackets, angle brackets, and pipe (tables/links)
+    # We intentionally do NOT include:
+    #   . (.)  ( )  -  #  !
+    specials = set("\\`*_{[]}<>|]")
+
+    out_chars = []
+    for ch in text:
+        if ch in specials:
+            out_chars.append("\\" + ch)
+        else:
+            out_chars.append(ch)
+    return "".join(out_chars)
+
+
+# ---------------------------------------------------------------------------
+# MISC
+# ---------------------------------------------------------------------------
+
+
+def truncate(text: str, max_len: int = 120) -> str:
+    """Truncate a string for logging/debug, preserving the end.
+
+    Example:
+        truncate("abcdef", 4) → "a..."
+    """
+    s = str(text)
+    if len(s) <= max_len:
+        return s
+    if max_len <= 3:
+        return s[:max_len]
+    return s[: max_len - 3] + "..."
+
+
+__all__ = [
+    "os_display_path",
+    "safe_join",
+    "log",
+    "progress",
+    "normalize_punctuation",
+    "linkify_urls",
+    "escape_markdown",
+    "truncate",
+    "is_windows",
+    "clear_console",
+    "print_error",
+]

pdfbrain/engine.py

@@ -0,0 +1,392 @@
+"""
+thinkpdf Engine v2 - Powered by IBM Docling for maximum quality.
+
+Features:
+- 97%+ accuracy on tables (TableFormer AI)
+- Advanced layout analysis (DocLayNet)
+- LaTeX equation support
+- Multi-format: PDF, DOCX, PPTX, HTML
+- Smart caching to avoid reprocessing
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from pathlib import Path
+from typing import Optional, Union, Callable
+from datetime import datetime
+
+# Try to import Docling
+try:
+    from docling.document_converter import DocumentConverter
+    from docling.datamodel.base_models import InputFormat
+    from docling.datamodel.pipeline_options import PdfPipelineOptions
+    from docling.backend.pypdfium2_backend import PyPdfiumDocumentBackend
+    HAS_DOCLING = True
+except ImportError:
+    HAS_DOCLING = False
+    DocumentConverter = None
+
+# Fallback to pdfmd pipeline
+try:
+    from .core.pipeline import pdf_to_markdown as pdfmd_convert
+    from .core.models import Options as PdfmdOptions
+    HAS_PDFMD = True
+except ImportError:
+    HAS_PDFMD = False
+
+# PyMuPDF for analysis
+try:
+    import fitz  # PyMuPDF
+    HAS_PYMUPDF = True
+except ImportError:
+    HAS_PYMUPDF = False
+
+
+def analyze_pdf_complexity(file_path: Path) -> dict:
+    """
+    Analyze PDF to determine optimal conversion engine.
+    
+    Returns dict with:
+    - page_count: number of pages
+    - has_images: True if PDF contains images
+    - is_scanned: True if PDF appears to be scanned (little text, many images)
+    - has_tables: True if PDF likely contains tables
+    - recommended_engine: 'docling' or 'pdfmd'
+    """
+    if not HAS_PYMUPDF:
+        return {"recommended_engine": "docling"}  # Default to best quality
+    
+    try:
+        doc = fitz.open(str(file_path))
+        page_count = len(doc)
+        
+        total_text_chars = 0
+        total_images = 0
+        table_indicators = 0
+        
+        # Analyze first few pages (max 5 for speed)
+        sample_pages = min(5, page_count)
+        
+        for i in range(sample_pages):
+            page = doc[i]
+            
+            # Count text
+            text = page.get_text()
+            total_text_chars += len(text)
+            
+            # Count images
+            images = page.get_images()
+            total_images += len(images)
+            
+            # Detect table indicators (many lines, grid patterns)
+            # Check for lines/rectangles that might indicate tables
+            drawings = page.get_drawings()
+            lines = [d for d in drawings if d.get("items") and any(
+                item[0] in ("l", "re") for item in d.get("items", [])
+            )]
+            if len(lines) > 10:
+                table_indicators += 1
+            
+            # Check for tabular text patterns (columns of numbers/text)
+            blocks = page.get_text("dict")["blocks"]
+            if len(blocks) > 5:
+                # Multiple text blocks might indicate columns/tables
+                table_indicators += 1
+        
+        doc.close()
+        
+        # Determine characteristics
+        avg_chars_per_page = total_text_chars / sample_pages if sample_pages > 0 else 0
+        avg_images_per_page = total_images / sample_pages if sample_pages > 0 else 0
+        
+        is_scanned = avg_chars_per_page < 100 and avg_images_per_page > 0
+        has_tables = table_indicators >= 2
+        has_images = total_images > 0
+        
+        # Decision logic
+        if is_scanned:
+            # Scanned PDFs need OCR - use Docling
+            recommended_engine = "docling"
+        elif has_tables:
+            # Tables need Docling's TableFormer for accuracy
+            recommended_engine = "docling"
+        elif page_count > 50 and not has_tables:
+            # Large simple documents - use pdfmd for speed
+            recommended_engine = "pdfmd"
+        elif page_count <= 10 and not has_tables and not is_scanned:
+            # Small simple documents - use pdfmd for speed
+            recommended_engine = "pdfmd"
+        else:
+            # Default to Docling for quality
+            recommended_engine = "docling"
+        
+        return {
+            "page_count": page_count,
+            "has_images": has_images,
+            "is_scanned": is_scanned,
+            "has_tables": has_tables,
+            "avg_chars_per_page": int(avg_chars_per_page),
+            "recommended_engine": recommended_engine,
+        }
+        
+    except Exception as e:
+        return {"recommended_engine": "docling", "error": str(e)}
+
+
+class thinkpdfEngine:
+    """
+    Unified PDF to Markdown conversion engine.
+    
+    Uses:
+    - Docling (IBM) for maximum quality when available
+    - pdfmd as fallback for simpler documents
+    """
+    
+    def __init__(
+        self,
+        cache_dir: Optional[Path] = None,
+        use_cache: bool = True,
+        engine: str = "auto",  # "auto", "docling", "pdfmd"
+    ):
+        self.cache_dir = cache_dir or Path.home() / ".thinkpdf" / "cache"
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+        self.use_cache = use_cache
+        self.engine = engine
+        self.docling_pipeline = None
+        self.docling_model = None
+        self._docling_converter = None  # Lazy loaded
+        
+        # Branding / Credit (Protection)
+        print("🧠 thinkpdf 1.0 - Hybrid PDF Engine | https://github.com/thinkpdf/thinkpdf")
+        # Initialize Docling converter if available
+        if HAS_DOCLING and engine in ("auto", "docling"):
+            self._init_docling()
+    
+    def _init_docling(self):
+        """Initialize Docling converter with optimal settings."""
+        try:
+            pipeline_options = PdfPipelineOptions()
+            pipeline_options.do_ocr = True
+            pipeline_options.do_table_structure = True
+            
+            self._docling_converter = DocumentConverter(
+                allowed_formats=[
+                    InputFormat.PDF,
+                    InputFormat.DOCX,
+                    InputFormat.PPTX,
+                    InputFormat.HTML,
+                    InputFormat.IMAGE,
+                ]
+            )
+        except Exception as e:
+            print(f"Warning: Could not initialize Docling: {e}")
+            self._docling_converter = None
+    
+    def get_file_hash(self, file_path: Path) -> str:
+        """Calculate SHA256 hash of file for caching."""
+        hasher = hashlib.sha256()
+        with open(file_path, "rb") as f:
+            for chunk in iter(lambda: f.read(65536), b""):
+                hasher.update(chunk)
+        return hasher.hexdigest()[:16]
+    
+    def get_cached(self, file_path: Path) -> Optional[str]:
+        """Get cached conversion if available and valid."""
+        if not self.use_cache:
+            return None
+        
+        file_hash = self.get_file_hash(file_path)
+        cache_file = self.cache_dir / f"{file_hash}.md"
+        meta_file = self.cache_dir / f"{file_hash}.json"
+        
+        if cache_file.exists() and meta_file.exists():
+            try:
+                with open(meta_file, "r") as f:
+                    meta = json.load(f)
+                
+                # Check if source file was modified
+                source_mtime = file_path.stat().st_mtime
+                if meta.get("source_mtime") == source_mtime:
+                    return cache_file.read_text(encoding="utf-8")
+            except Exception:
+                pass
+        
+        return None
+    
+    def cache_result(self, file_path: Path, markdown: str):
+        """Cache the conversion result."""
+        if not self.use_cache:
+            return
+        
+        file_hash = self.get_file_hash(file_path)
+        cache_file = self.cache_dir / f"{file_hash}.md"
+        meta_file = self.cache_dir / f"{file_hash}.json"
+        
+        cache_file.write_text(markdown, encoding="utf-8")
+        
+        meta = {
+            "source_file": str(file_path),
+            "source_mtime": file_path.stat().st_mtime,
+            "converted_at": datetime.now().isoformat(),
+            "engine": self.engine,
+        }
+        with open(meta_file, "w") as f:
+            json.dump(meta, f)
+    
+    def convert(
+        self,
+        input_path: Union[str, Path],
+        output_path: Optional[Union[str, Path]] = None,
+        progress_callback: Optional[Callable[[int, int], None]] = None,
+    ) -> str:
+        """
+        Convert a document to Markdown.
+        
+        Args:
+            input_path: Path to PDF, DOCX, PPTX, or HTML file
+            output_path: Optional path to save markdown
+            progress_callback: Optional callback for progress updates
+            
+        Returns:
+            Markdown content as string
+        """
+        input_path = Path(input_path)
+        
+        if not input_path.exists():
+            raise FileNotFoundError(f"File not found: {input_path}")
+        
+        # Check cache first
+        cached = self.get_cached(input_path)
+        if cached:
+            if output_path:
+                Path(output_path).write_text(cached, encoding="utf-8")
+            return cached
+        
+        # Convert using best available engine
+        use_engine = self.engine
+        
+        # Auto-detect best engine for PDFs
+        if use_engine == "auto" and input_path.suffix.lower() == ".pdf":
+            analysis = analyze_pdf_complexity(input_path)
+            use_engine = analysis.get("recommended_engine", "docling")
+        
+        if use_engine == "docling" and self._docling_converter:
+            markdown = self._convert_with_docling(input_path, progress_callback)
+        elif use_engine == "pdfmd" and HAS_PDFMD and input_path.suffix.lower() == ".pdf":
+            markdown = self._convert_with_pdfmd(input_path, progress_callback)
+        elif self._docling_converter:
+            # Fallback to Docling for non-PDF formats
+            markdown = self._convert_with_docling(input_path, progress_callback)
+        elif HAS_PDFMD and input_path.suffix.lower() == ".pdf":
+            markdown = self._convert_with_pdfmd(input_path, progress_callback)
+        else:
+            raise RuntimeError("No conversion engine available. Install docling or pdfmd.")
+        
+        # Validate output
+        if not markdown.strip():
+            # Assuming 'logger' is defined elsewhere or needs to be imported/defined
+            # For now, using print as a placeholder for logger.warning
+            print("Warning: Conversion produced empty output")
+            # Fallback to simple extraction if main engine failed to produce text
+            # This is a safety net
+            
+        # Forensic Watermark (Invisible to user, visible in raw file)
+        # This helps prove provenance if someone steals the output/engine
+        if markdown and not markdown.endswith("thinkpdf"):
+            markdown += "\n\n<!-- Generated by thinkpdf engine -->"
+            
+        # Cache result
+        self.cache_result(input_path, markdown)
+        
+        # Save to file if requested
+        if output_path:
+            out_path = Path(output_path)
+            out_path.parent.mkdir(parents=True, exist_ok=True)
+            out_path.write_text(markdown, encoding="utf-8")
+        
+        return markdown
+    
+    def _convert_with_docling(
+        self,
+        input_path: Path,
+        progress_callback: Optional[Callable] = None,
+    ) -> str:
+        """Convert using IBM Docling (highest quality)."""
+        if progress_callback:
+            progress_callback(0, 100)
+        
+        result = self._docling_converter.convert(str(input_path))
+        
+        if progress_callback:
+            progress_callback(50, 100)
+        
+        # Export to markdown
+        markdown = result.document.export_to_markdown()
+        
+        if progress_callback:
+            progress_callback(100, 100)
+        
+        return markdown
+    
+    def _convert_with_pdfmd(
+        self,
+        input_path: Path,
+        progress_callback: Optional[Callable] = None,
+    ) -> str:
+        """Convert using pdfmd (fallback)."""
+        import tempfile
+        
+        with tempfile.NamedTemporaryFile(suffix=".md", delete=False) as tmp:
+            tmp_path = tmp.name
+        
+        try:
+            options = PdfmdOptions()
+            pdfmd_convert(
+                input_pdf=str(input_path),
+                output_md=tmp_path,
+                options=options,
+                progress_cb=progress_callback,
+            )
+            
+            return Path(tmp_path).read_text(encoding="utf-8")
+        finally:
+            if os.path.exists(tmp_path):
+                os.unlink(tmp_path)
+    
+    def get_document_info(self, input_path: Union[str, Path]) -> dict:
+        """Get information about a document."""
+        input_path = Path(input_path)
+        
+        return {
+            "filename": input_path.name,
+            "path": str(input_path.absolute()),
+            "size_bytes": input_path.stat().st_size,
+            "size_mb": round(input_path.stat().st_size / (1024 * 1024), 2),
+            "extension": input_path.suffix.lower(),
+            "file_hash": self.get_file_hash(input_path),
+            "has_docling": HAS_DOCLING,
+            "has_pdfmd": HAS_PDFMD,
+        }
+
+
+# Global instance for convenience
+_engine: Optional[thinkpdfEngine] = None
+
+
+def get_engine() -> thinkpdfEngine:
+    """Get or create the global engine instance."""
+    global _engine
+    if _engine is None:
+        _engine = thinkpdfEngine()
+    return _engine
+
+
+def convert(
+    input_path: Union[str, Path],
+    output_path: Optional[Union[str, Path]] = None,
+) -> str:
+    """Quick conversion function."""
+    return get_engine().convert(input_path, output_path)