dforge-cli 1.0.1__py3-none-any.whl

dforge/converter.py

@@ -0,0 +1,167 @@
+"""
+DForge Conversion Module
+Handles: document format conversion (docx, pdf, md, html, txt), img2pdf, pdf2img
+"""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+from typing import List, Optional
+
+from rich.progress import Progress, SpinnerColumn, TextColumn
+from dforge.config_manager import get_tool_path
+from dforge.utils import (
+    abort, console, ensure_parent, info, require_pandoc, success, warn,
+)
+from dforge.config import DEFAULT_IMAGE_DPI, SUPPORTED_IMAGE_EXTS
+
+
+PANDOC_FORMAT_MAP = {
+    "pdf": "pdf",
+    "docx": "docx",
+    "md": "markdown",
+    "markdown": "markdown",
+    "html": "html",
+    "txt": "plain",
+    "text": "plain",
+    "rst": "rst",
+    "odt": "odt",
+    "epub": "epub",
+}
+
+
+def _ext_to_pandoc_format(ext: str) -> str:
+    """Convert a file extension to a Pandoc format name."""
+    cleaned = ext.lstrip(".").lower()
+    return PANDOC_FORMAT_MAP.get(cleaned, cleaned)
+
+
+# ---------------------------------------------------------------------------
+# Universal convert
+# ---------------------------------------------------------------------------
+
+def convert(
+    input_path: Path,
+    target_format: str,
+    output: Optional[Path] = None,
+) -> None:
+    """
+    Convert a document to the target format using Pandoc.
+
+    Supports: pdf, docx, md, html, txt, rst, odt, epub
+    """
+    require_pandoc()
+
+    if not input_path.exists():
+        abort(f"File not found: {input_path}")
+
+    target_ext = f".{target_format.lstrip('.')}"
+    out = output or input_path.with_suffix(target_ext)
+    ensure_parent(out)
+
+    pandoc_to = _ext_to_pandoc_format(target_format)
+    pandoc_from = _ext_to_pandoc_format(input_path.suffix)
+
+    cmd = ["pandoc", str(input_path), "-f", pandoc_from, "-t", pandoc_to, "-o", str(out)]
+
+    # PDF requires a PDF engine
+    if pandoc_to == "pdf":
+        xelatex = get_tool_path("xelatex")
+
+        if xelatex:
+            cmd += [f"--pdf-engine={xelatex}"]
+        else:
+            cmd += ["--pdf-engine=xelatex"]
+
+    info(f"Converting [bold]{input_path.name}[/bold] -> [bold]{pandoc_to.upper()}[/bold]...")
+    result = subprocess.run(cmd, capture_output=True, text=True)
+    if result.returncode != 0:
+        abort(f"Pandoc error:\n{result.stderr}")
+
+    success(f"Converted -> [bold]{out}[/bold]")
+
+
+# ---------------------------------------------------------------------------
+# Images -> PDF
+# ---------------------------------------------------------------------------
+
+def images_to_pdf(
+    source: Path,
+    output: Optional[Path] = None,
+    sort_files: bool = True,
+) -> None:
+    """
+    Combine images from a directory (or a single image) into a PDF.
+
+    source can be a directory of images or a single image file.
+    """
+    try:
+        import img2pdf
+    except ImportError:
+        abort("img2pdf is required. Run: pip install img2pdf")
+
+    images: List[Path] = []
+
+    if source.is_dir():
+        for ext in SUPPORTED_IMAGE_EXTS:
+            images.extend(source.glob(f"*{ext}"))
+        if sort_files:
+            images = sorted(images)
+        if not images:
+            abort(f"No image files found in {source}")
+        out = output or source.parent / (source.name + ".pdf")
+    elif source.is_file():
+        if source.suffix.lower() not in SUPPORTED_IMAGE_EXTS:
+            abort(f"Not a supported image format: {source.suffix}")
+        images = [source]
+        out = output or source.with_suffix(".pdf")
+    else:
+        abort(f"Path not found: {source}")
+
+    ensure_parent(out)
+
+    info(f"Combining {len(images)} image(s) into PDF...")
+    with open(out, "wb") as fh:
+        fh.write(img2pdf.convert([str(img) for img in images]))
+
+    success(f"{len(images)} image(s) -> [bold]{out}[/bold]")
+
+
+# ---------------------------------------------------------------------------
+# PDF -> Images
+# ---------------------------------------------------------------------------
+
+def pdf_to_images(
+    input_path: Path,
+    output_dir: Optional[Path] = None,
+    dpi: int = DEFAULT_IMAGE_DPI,
+    fmt: str = "png",
+) -> None:
+    """
+    Convert each page of a PDF to an image file.
+
+    fmt: png, jpeg, tiff
+    """
+    try:
+        from pdf2image import convert_from_path
+    except ImportError:
+        abort("pdf2image is required. Run: pip install pdf2image")
+
+    if not input_path.exists():
+        abort(f"File not found: {input_path}")
+
+    dest = output_dir or input_path.parent / (input_path.stem + "_images")
+    dest.mkdir(parents=True, exist_ok=True)
+
+    info(f"Converting PDF pages to {fmt.upper()} images (DPI={dpi})...")
+    pages = convert_from_path(str(input_path), dpi=dpi, fmt=fmt)
+
+    with Progress(SpinnerColumn(), TextColumn("[progress.description]{task.description}"), console=console) as progress:
+        task = progress.add_task(f"Exporting {len(pages)} pages...", total=len(pages))
+        for i, page in enumerate(pages, start=1):
+            out_file = dest / f"{input_path.stem}_page_{i:04d}.{fmt}"
+            page.save(str(out_file), fmt.upper())
+            progress.advance(task)
+
+    success(f"Exported {len(pages)} page(s) -> [bold]{dest}/[/bold]")

dforge/dependencies.py

@@ -0,0 +1,98 @@
+from pathlib import Path
+
+from rich.console import Console
+from rich.table import Table
+
+from dforge.config_manager import get_tool_path
+
+console = Console()
+
+
+def has_poppler():
+    path = get_tool_path("poppler")
+    return path is not None and Path(path).exists()
+
+
+def has_tesseract():
+    path = get_tool_path("tesseract")
+    return path is not None and Path(path).exists()
+
+
+def has_pandoc():
+    path = get_tool_path("pandoc")
+    return path is not None and Path(path).exists()
+
+
+def has_ghostscript():
+    path = get_tool_path("ghostscript")
+    return path is not None and Path(path).exists()
+
+
+def doctor():
+    table = Table(title="DForge System Check")
+
+    table.add_column("Dependency")
+    table.add_column("Status")
+
+    deps = [
+        ("Poppler", has_poppler()),
+        ("Tesseract", has_tesseract()),
+        ("Ghostscript", has_ghostscript()),
+        ("Pandoc", has_pandoc()),
+    ]
+
+    for name, status in deps:
+        table.add_row(
+            name,
+            "✓ Installed" if status else "✗ Missing",
+        )
+
+    console.print(table)
+
+
+def check_poppler():
+    if has_poppler():
+        return True
+
+    console.print(
+        "\n[red]Poppler is required for PDF OCR.[/red]\n"
+        "Run:\n"
+        "[cyan]dforge setup[/cyan]\n"
+    )
+    return False
+
+
+def check_tesseract():
+    if has_tesseract():
+        return True
+
+    console.print(
+        "\n[red]Tesseract OCR is not installed.[/red]\n"
+        "Run:\n"
+        "[cyan]dforge setup[/cyan]\n"
+    )
+    return False
+
+
+def check_ghostscript():
+    if has_ghostscript():
+        return True
+
+    console.print(
+        "\n[red]Ghostscript is not installed.[/red]\n"
+        "Run:\n"
+        "[cyan]dforge setup[/cyan]\n"
+    )
+    return False
+
+
+def check_pandoc():
+    if has_pandoc():
+        return True
+
+    console.print(
+        "\n[red]Pandoc is not installed.[/red]\n"
+        "Run:\n"
+        "[cyan]dforge setup[/cyan]\n"
+    )
+    return False

dforge/engine.py

@@ -0,0 +1,236 @@
+"""
+DForge OCR Module
+Handles: image OCR, PDF OCR, searchable PDF generation, batch OCR
+Output formats: TXT, JSON, Markdown
+"""
+
+from __future__ import annotations
+
+import json
+import tempfile
+from pathlib import Path
+from typing import List, Optional
+
+from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
+
+from dforge.utils import (
+    abort, console, ensure_parent, info, require_tesseract, success, warn,
+)
+from dforge.config import DEFAULT_OCR_LANG, DEFAULT_OCR_DPI, SUPPORTED_IMAGE_EXTS
+
+
+# ---------------------------------------------------------------------------
+# Core OCR helpers
+# ---------------------------------------------------------------------------
+
+def _run_tesseract(image_path: Path, lang: str) -> str:
+    """Run Tesseract on a single image and return extracted text."""
+    require_tesseract()
+    try:
+        import pytesseract
+    except ImportError:
+        abort("pytesseract is required. Run: pip install pytesseract")
+
+    text = pytesseract.image_to_string(str(image_path), lang=lang)
+    return text.strip()
+
+
+def _pdf_to_images(pdf_path: Path, dpi: int = DEFAULT_OCR_DPI) -> List[Path]:
+    """Convert PDF pages to temporary image files for OCR."""
+    try:
+        from pdf2image import convert_from_path
+    except ImportError:
+        abort("pdf2image is required. Run: pip install pdf2image")
+
+    tmp_dir = Path(tempfile.mkdtemp())
+    pages = convert_from_path(str(pdf_path), dpi=dpi, output_folder=str(tmp_dir), fmt="png")
+    paths = []
+    for i, page in enumerate(pages):
+        p = tmp_dir / f"page_{i:04d}.png"
+        page.save(str(p), "PNG")
+        paths.append(p)
+    return paths
+
+
+def _write_output(text: str, output_path: Path, fmt: str, source_name: str = "") -> None:
+    """Write OCR result in the specified format."""
+    ensure_parent(output_path)
+    if fmt == "json":
+        data = {
+            "source": source_name,
+            "text": text,
+            "word_count": len(text.split()),
+            "char_count": len(text),
+        }
+        output_path.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
+    elif fmt == "md":
+        content = f"# OCR Output: {source_name}\n\n{text}\n"
+        output_path.write_text(content, encoding="utf-8")
+    else:  # txt
+        output_path.write_text(text, encoding="utf-8")
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def ocr_image(
+    input_path: Path,
+    output: Optional[Path] = None,
+    lang: str = DEFAULT_OCR_LANG,
+    fmt: str = "txt",
+) -> str:
+    """Run OCR on a single image file."""
+    if not input_path.exists():
+        abort(f"File not found: {input_path}")
+
+    info(f"Running OCR on [bold]{input_path.name}[/bold] (lang: {lang})...")
+    text = _run_tesseract(input_path, lang)
+
+    ext_map = {"txt": ".txt", "json": ".json", "md": ".md"}
+    ext = ext_map.get(fmt, ".txt")
+    out = output or input_path.with_suffix(ext)
+    _write_output(text, out, fmt, input_path.name)
+    success(f"OCR complete -> [bold]{out}[/bold]  ({len(text.split())} words)")
+    return text
+
+
+def ocr_pdf(
+    input_path: Path,
+    output: Optional[Path] = None,
+    lang: str = DEFAULT_OCR_LANG,
+    fmt: str = "txt",
+    dpi: int = DEFAULT_OCR_DPI,
+) -> str:
+    """Run OCR on all pages of a PDF."""
+    if not input_path.exists():
+        abort(f"File not found: {input_path}")
+
+    info(f"Converting PDF pages to images (DPI={dpi})...")
+    images = _pdf_to_images(input_path, dpi)
+    all_text_parts: List[str] = []
+
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(),
+        TextColumn("{task.completed}/{task.total} pages"),
+        console=console,
+    ) as progress:
+        task = progress.add_task("Running OCR...", total=len(images))
+        for img in images:
+            text = _run_tesseract(img, lang)
+            all_text_parts.append(text)
+            progress.advance(task)
+            # Cleanup temp image
+            img.unlink(missing_ok=True)
+
+    full_text = "\n\n---\n\n".join(all_text_parts)
+
+    ext_map = {"txt": ".txt", "json": ".json", "md": ".md"}
+    ext = ext_map.get(fmt, ".txt")
+    out = output or input_path.with_suffix(ext)
+    _write_output(full_text, out, fmt, input_path.name)
+    success(f"OCR complete ({len(images)} pages) -> [bold]{out}[/bold]")
+    return full_text
+
+
+def make_searchable_pdf(
+    input_path: Path,
+    output: Optional[Path] = None,
+    lang: str = DEFAULT_OCR_LANG,
+    dpi: int = DEFAULT_OCR_DPI,
+) -> None:
+    """Create a searchable PDF from a scanned PDF using OCR."""
+    require_tesseract()
+    try:
+        import pytesseract
+        from pdf2image import convert_from_path
+        from PIL import Image
+    except ImportError:
+        abort("pytesseract, pdf2image and Pillow are required.")
+
+    if not input_path.exists():
+        abort(f"File not found: {input_path}")
+
+    out = output or input_path.with_name(input_path.stem + "_searchable.pdf")
+    ensure_parent(out)
+
+    info(f"Converting PDF to images (DPI={dpi})...")
+    pages = convert_from_path(str(input_path), dpi=dpi)
+
+    pdf_pages = []
+    with Progress(SpinnerColumn(), TextColumn("[progress.description]{task.description}"), console=console) as progress:
+        task = progress.add_task(f"Processing {len(pages)} pages...", total=len(pages))
+        for page_img in pages:
+            # pytesseract can produce a searchable PDF per-page
+            pdf_bytes = pytesseract.image_to_pdf_or_hocr(page_img, extension="pdf", lang=lang)
+            pdf_pages.append(pdf_bytes)
+            progress.advance(task)
+
+    # Merge all page PDFs
+    try:
+        from pypdf import PdfWriter, PdfReader
+        import io
+    except ImportError:
+        abort("pypdf is required.")
+
+    writer = PdfWriter()
+    for pdf_bytes in pdf_pages:
+        import io
+        reader = PdfReader(io.BytesIO(pdf_bytes))
+        for page in reader.pages:
+            writer.add_page(page)
+
+    with open(out, "wb") as fh:
+        writer.write(fh)
+
+    success(f"Searchable PDF created -> [bold]{out}[/bold]")
+
+
+def batch_ocr(
+    directory: Path,
+    lang: str = DEFAULT_OCR_LANG,
+    fmt: str = "txt",
+    recursive: bool = True,
+) -> None:
+    """Run OCR on all supported image/PDF files in a directory."""
+    from dforge.utils import collect_files
+
+    if not directory.exists():
+        abort(f"Directory not found: {directory}")
+
+    all_exts = SUPPORTED_IMAGE_EXTS | {".pdf"}
+    files = collect_files(directory, all_exts, recursive=recursive)
+
+    if not files:
+        warn(f"No supported files found in {directory}")
+        return
+
+    info(f"Found {len(files)} files to process...")
+    errors = []
+
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(),
+        TextColumn("{task.completed}/{task.total}"),
+        console=console,
+    ) as progress:
+        task = progress.add_task("Batch OCR...", total=len(files))
+        for f in files:
+            try:
+                if f.suffix.lower() == ".pdf":
+                    ocr_pdf(f, lang=lang, fmt=fmt)
+                else:
+                    ocr_image(f, lang=lang, fmt=fmt)
+            except Exception as exc:
+                errors.append((f, str(exc)))
+            progress.advance(task)
+
+    if errors:
+        warn(f"{len(errors)} file(s) failed:")
+        for f, err in errors:
+            console.print(f"  [red]{f.name}[/red]: {err}")
+
+    success(f"Batch OCR complete. Processed {len(files) - len(errors)}/{len(files)} files.")

dforge/extractor.py

@@ -0,0 +1,201 @@
+"""
+DForge Extraction Module
+Handles: text, images, metadata, and table extraction from PDFs
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Optional
+
+from dforge.utils import abort, console, ensure_parent, info, success, warn
+
+
+# ---------------------------------------------------------------------------
+# Extract Text
+# ---------------------------------------------------------------------------
+
+def extract_text(input_path: Path, output: Optional[Path] = None) -> str:
+    """Extract all text from a PDF."""
+    try:
+        from pypdf import PdfReader
+    except ImportError:
+        abort("pypdf is required.")
+
+    if not input_path.exists():
+        abort(f"File not found: {input_path}")
+
+    reader = PdfReader(str(input_path))
+    parts = []
+    for i, page in enumerate(reader.pages, start=1):
+        text = page.extract_text() or ""
+        parts.append(f"--- Page {i} ---\n{text}")
+
+    full_text = "\n\n".join(parts)
+
+    out = output or input_path.with_suffix(".txt")
+    ensure_parent(out)
+    out.write_text(full_text, encoding="utf-8")
+    success(f"Text extracted ({len(reader.pages)} pages) -> [bold]{out}[/bold]")
+    return full_text
+
+
+# ---------------------------------------------------------------------------
+# Extract Images
+# ---------------------------------------------------------------------------
+
+def extract_images(input_path: Path, output_dir: Optional[Path] = None) -> None:
+    """Extract all embedded images from a PDF."""
+    try:
+        from pypdf import PdfReader
+    except ImportError:
+        abort("pypdf is required.")
+
+    if not input_path.exists():
+        abort(f"File not found: {input_path}")
+
+    dest = output_dir or input_path.parent / (input_path.stem + "_images")
+    dest.mkdir(parents=True, exist_ok=True)
+
+    reader = PdfReader(str(input_path))
+    count = 0
+
+    for page_num, page in enumerate(reader.pages, start=1):
+        if "/XObject" not in page.get("/Resources", {}):
+            continue
+        xobject = page["/Resources"]["/XObject"].get_object()
+        for obj_name, obj_ref in xobject.items():
+            obj = obj_ref.get_object()
+            if obj.get("/Subtype") == "/Image":
+                data = obj.get_data()
+                # Determine extension from color space / filter
+                filters = obj.get("/Filter", "")
+                if isinstance(filters, list):
+                    filters = filters[-1] if filters else ""
+                ext = {
+                    "/DCTDecode": "jpg",
+                    "/JPXDecode": "jp2",
+                    "/FlateDecode": "png",
+                    "/CCITTFaxDecode": "tiff",
+                }.get(str(filters), "bin")
+
+                fname = dest / f"page{page_num:03d}_{obj_name.lstrip('/')}.{ext}"
+                fname.write_bytes(data)
+                count += 1
+
+    if count == 0:
+        warn("No extractable images found in this PDF.")
+    else:
+        success(f"Extracted {count} image(s) -> [bold]{dest}/[/bold]")
+
+
+# ---------------------------------------------------------------------------
+# Extract Metadata
+# ---------------------------------------------------------------------------
+
+def extract_metadata(input_path: Path, output: Optional[Path] = None) -> dict:
+    """Extract PDF metadata and optionally save to JSON."""
+    try:
+        from pypdf import PdfReader
+    except ImportError:
+        abort("pypdf is required.")
+
+    if not input_path.exists():
+        abort(f"File not found: {input_path}")
+
+    reader = PdfReader(str(input_path))
+    meta_raw = reader.metadata or {}
+
+    # Clean metadata keys (strip leading '/')
+    meta = {k.lstrip("/"): str(v) for k, v in meta_raw.items()}
+    meta["PageCount"] = str(len(reader.pages))
+    meta["Encrypted"] = str(reader.is_encrypted)
+
+    # Print to console
+    console.print("\n[bold cyan]PDF Metadata[/bold cyan]")
+    for k, v in meta.items():
+        console.print(f"  [dim]{k}:[/dim] {v}")
+
+    if output:
+        ensure_parent(output)
+        output.write_text(json.dumps(meta, indent=2), encoding="utf-8")
+        success(f"Metadata saved -> [bold]{output}[/bold]")
+
+    return meta
+
+
+# ---------------------------------------------------------------------------
+# Extract Tables
+# ---------------------------------------------------------------------------
+
+def extract_tables(
+    input_path: Path,
+    output: Optional[Path] = None,
+    fmt: str = "csv",
+) -> None:
+    """
+    Extract tables from a PDF and export to CSV, XLSX, or JSON.
+    Uses pdfplumber for table detection.
+    fmt: csv | xlsx | json
+    """
+    try:
+        import pdfplumber
+    except ImportError:
+        abort("pdfplumber is required. Run: pip install pdfplumber")
+
+    if not input_path.exists():
+        abort(f"File not found: {input_path}")
+
+    all_tables = []
+    info(f"Scanning {input_path.name} for tables...")
+
+    with pdfplumber.open(str(input_path)) as pdf:
+        for page_num, page in enumerate(pdf.pages, start=1):
+            tables = page.extract_tables()
+            for t_idx, table in enumerate(tables):
+                all_tables.append({
+                    "page": page_num,
+                    "table_index": t_idx,
+                    "data": table,
+                })
+
+    if not all_tables:
+        warn("No tables found in this PDF.")
+        return
+
+    total_tables = len(all_tables)
+    info(f"Found {total_tables} table(s) across the document.")
+
+    if fmt == "json":
+        out = output or input_path.with_suffix(".tables.json")
+        ensure_parent(out)
+        out.write_text(json.dumps(all_tables, indent=2, ensure_ascii=False), encoding="utf-8")
+        success(f"Tables (JSON) -> [bold]{out}[/bold]")
+
+    elif fmt == "xlsx":
+        try:
+            import pandas as pd
+        except ImportError:
+            abort("pandas is required for XLSX export. Run: pip install pandas openpyxl")
+        out = output or input_path.with_suffix(".tables.xlsx")
+        ensure_parent(out)
+        with pd.ExcelWriter(str(out), engine="openpyxl") as writer:
+            for entry in all_tables:
+                sheet_name = f"P{entry['page']}_T{entry['table_index'] + 1}"[:31]
+                df = pd.DataFrame(entry["data"])
+                df.to_excel(writer, sheet_name=sheet_name, index=False, header=False)
+        success(f"Tables (XLSX, {total_tables} sheets) -> [bold]{out}[/bold]")
+
+    else:  # csv
+        try:
+            import pandas as pd
+        except ImportError:
+            abort("pandas is required for CSV export.")
+        out_dir = output or input_path.parent / (input_path.stem + "_tables")
+        out_dir.mkdir(parents=True, exist_ok=True)
+        for entry in all_tables:
+            csv_file = out_dir / f"page{entry['page']:03d}_table{entry['table_index'] + 1}.csv"
+            df = pd.DataFrame(entry["data"])
+            df.to_csv(str(csv_file), index=False, header=False)
+        success(f"Tables (CSV, {total_tables} files) -> [bold]{out_dir}/[/bold]")