pdf2dotmd 0.0.1__py3-none-any.whl

pdf2dotmd/__init__.py

@@ -0,0 +1,3 @@
+"""pdf2md package."""
+
+__version__ = "0.0.1"

pdf2dotmd/cli.py

@@ -0,0 +1,100 @@
+"""Command line interface for PDF to Markdown converter."""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import os
+import sys
+from glob import glob
+from pathlib import Path
+
+from .converter import PdfToMarkdownConverter
+
+logger = logging.getLogger(__name__)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Convert PDF files to Markdown format",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Example usage:
+  %(prog)s input.pdf                      # Output to stdout
+  %(prog)s input.pdf -o output.md         # Output to file
+  %(prog)s input.pdf --ignore-images      # Skip images, single file output
+  %(prog)s *.pdf -o output_dir/           # Batch conversion
+  %(prog)s input.pdf -p 1-3              # Convert only pages 1-3
+        """,
+    )
+
+    parser.add_argument(
+        "input_files",
+        nargs="+",
+        help="Input PDF file paths (supports wildcards)",
+    )
+    parser.add_argument("-o", "--output", help="Output file or directory path")
+    parser.add_argument(
+        "--ignore-images",
+        "--no-images",
+        action="store_true",
+        dest="ignore_images",
+        help="Ignore all images and output a single Markdown file",
+    )
+    parser.add_argument(
+        "-p",
+        "--pages",
+        help="Page range to convert (e.g., '1-5,8,10-12')",
+    )
+    parser.add_argument("-v", "--verbose", action="store_true", help="Show verbose logs")
+
+    args = parser.parse_args()
+
+    logging.basicConfig(
+        level=logging.DEBUG if args.verbose else logging.INFO,
+        format="%(asctime)s - %(levelname)s - %(message)s",
+    )
+
+    converter = PdfToMarkdownConverter()
+
+    try:
+        for input_pattern in args.input_files:
+            matching_files = glob(input_pattern)
+            if not matching_files:
+                logger.warning("No matching files found: %s", input_pattern)
+                continue
+
+            for file_path in matching_files:
+                if not file_path.lower().endswith(".pdf"):
+                    logger.warning("Skipping non-PDF file: %s", file_path)
+                    continue
+
+                output_path = None
+                if args.output:
+                    if os.path.isdir(args.output) or args.output.endswith("/"):
+                        output_path = os.path.join(
+                            args.output, f"{Path(file_path).stem}.md"
+                        )
+                    else:
+                        output_path = args.output
+
+                markdown_content = converter.convert_file(
+                    file_path,
+                    output_path=output_path,
+                    ignore_images=args.ignore_images,
+                    pages=args.pages,
+                )
+
+                if not output_path:
+                    print(f"\n=== {file_path} ===\n")
+                    print(markdown_content)
+    except KeyboardInterrupt:
+        logger.info("Conversion interrupted by user")
+        sys.exit(1)
+    except Exception as exc:  # pylint: disable=broad-except
+        logger.error("Program execution failed: %s", exc)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

pdf2dotmd/converter.py

@@ -0,0 +1,172 @@
+"""Core converter module for PDF to Markdown conversion."""
+
+from __future__ import annotations
+
+import logging
+import os
+from pathlib import Path
+from typing import Optional
+
+try:
+    import pdfplumber  # type: ignore[import-not-found]
+except ImportError:  # pragma: no cover
+    pdfplumber = None  # type: ignore[assignment]
+
+from .image_extractor import ImageExtractor
+from .layout_analyzer import LayoutAnalyzer
+from .page_processor import PageProcessor
+from .table_processor import TableProcessor
+from .utils import clean_markdown_content
+
+logger = logging.getLogger(__name__)
+
+
+def _parse_page_range(page_spec: str, total_pages: int) -> list[int]:
+    """Parse a page range string like '1-5,8,10-12' into 0-based page indices."""
+    indices: list[int] = []
+    for part in page_spec.split(","):
+        part = part.strip()
+        if "-" in part:
+            start, end = part.split("-", 1)
+            start = int(start.strip())
+            end = int(end.strip())
+            for p in range(start, end + 1):
+                if 1 <= p <= total_pages:
+                    indices.append(p - 1)
+        else:
+            p = int(part)
+            if 1 <= p <= total_pages:
+                indices.append(p - 1)
+    return sorted(set(indices))
+
+
+class PdfToMarkdownConverter:
+    """PDF to Markdown converter."""
+
+    def __init__(self):
+        self.output_folder: str = ""
+        self.assets_dir: str = ""
+
+    def convert_file(
+        self,
+        input_path: str,
+        output_path: Optional[str] = None,
+        ignore_images: bool = False,
+        pages: Optional[str] = None,
+    ) -> str:
+        if not os.path.exists(input_path):
+            raise FileNotFoundError(f"Input file does not exist: {input_path}")
+
+        if not input_path.lower().endswith(".pdf"):
+            raise ValueError(f"Only .pdf is supported: {input_path}")
+
+        if pdfplumber is None:
+            raise RuntimeError(
+                "Missing required dependency 'pdfplumber'. Please run: pip install pdfplumber"
+            )
+
+        self._setup_output_structure(input_path, output_path, ignore_images)
+
+        layout_analyzer = LayoutAnalyzer()
+        table_processor = TableProcessor()
+        image_extractor = (
+            ImageExtractor(self.assets_dir) if not ignore_images and self.assets_dir else None
+        )
+        page_processor = PageProcessor(
+            layout_analyzer=layout_analyzer,
+            table_processor=table_processor,
+            image_extractor=image_extractor,
+            ignore_images=ignore_images,
+        )
+
+        output_lines: list[str] = []
+
+        with pdfplumber.open(input_path) as pdf:
+            total_pages = len(pdf.pages)
+
+            if total_pages == 0:
+                logger.warning("PDF has no pages: %s", input_path)
+                markdown_content = "\n"
+                self._write_output(markdown_content, self._get_final_output_path(input_path, output_path))
+                return markdown_content
+
+            # Determine page indices
+            if pages:
+                page_indices = _parse_page_range(pages, total_pages)
+                if not page_indices:
+                    raise ValueError(f"No valid pages in range '{pages}' (total: {total_pages})")
+            else:
+                page_indices = list(range(total_pages))
+
+            # Check if the PDF is scanned (no text on any page)
+            has_text = False
+            for idx in page_indices:
+                if pdf.pages[idx].chars:
+                    has_text = True
+                    break
+            if not has_text:
+                logger.warning(
+                    "PDF appears to have no text layer (possibly scanned). "
+                    "OCR is not supported. Output may be empty."
+                )
+
+            for idx in page_indices:
+                page = pdf.pages[idx]
+                page_number = idx + 1
+                logger.debug("Processing page %d/%d", page_number, total_pages)
+
+                page_lines = page_processor.process_page(page, page_number)
+                output_lines.extend(page_lines)
+
+        markdown_content = clean_markdown_content(output_lines)
+        final_output_path = self._get_final_output_path(input_path, output_path)
+        self._write_output(markdown_content, final_output_path)
+        self._cleanup_empty_assets_dir()
+
+        logger.info("Conversion completed, output file: %s", final_output_path)
+        return markdown_content
+
+    def _setup_output_structure(
+        self, input_path: str, output_path: Optional[str], ignore_images: bool
+    ):
+        input_stem = Path(input_path).stem
+
+        if output_path:
+            if os.path.isdir(output_path) or output_path.endswith("/"):
+                self.output_folder = os.path.join(output_path, input_stem)
+            else:
+                self.output_folder = os.path.dirname(output_path)
+                if not self.output_folder:
+                    self.output_folder = input_stem
+        else:
+            self.output_folder = input_stem
+
+        os.makedirs(self.output_folder, exist_ok=True)
+
+        if ignore_images:
+            self.assets_dir = ""
+        else:
+            self.assets_dir = os.path.join(self.output_folder, "assets")
+            os.makedirs(self.assets_dir, exist_ok=True)
+
+    def _get_final_output_path(self, input_path: str, output_path: Optional[str]) -> str:
+        input_stem = Path(input_path).stem
+
+        if output_path:
+            if os.path.isdir(output_path) or output_path.endswith("/"):
+                return os.path.join(self.output_folder, f"{input_stem}.md")
+            return output_path
+
+        return os.path.join(self.output_folder, f"{input_stem}.md")
+
+    def _write_output(self, content: str, output_path: str):
+        output_dir = os.path.dirname(output_path)
+        if output_dir and not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+
+        with open(output_path, "w", encoding="utf-8") as f:
+            f.write(content)
+
+    def _cleanup_empty_assets_dir(self):
+        if self.assets_dir and os.path.exists(self.assets_dir) and not os.listdir(self.assets_dir):
+            os.rmdir(self.assets_dir)

pdf2dotmd/image_extractor.py

@@ -0,0 +1,110 @@
+"""Image extraction from PDF pages."""
+
+from __future__ import annotations
+
+import logging
+import os
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+class ImageExtractor:
+    """Extract images from PDF pages into an assets directory."""
+
+    def __init__(self, assets_dir: str):
+        self.assets_dir = assets_dir
+        self._image_count = 0
+
+    def extract_images(self, page, page_number: int) -> list[tuple[str, tuple[float, float, float, float]]]:
+        """Extract images from a pdfplumber page.
+
+        Returns list of (markdown_path, bounding_box) tuples.
+        """
+        results: list[tuple[str, tuple[float, float, float, float]]] = []
+
+        try:
+            images = page.images
+        except Exception as exc:
+            logger.warning("Failed to get images from page %d: %s", page_number, exc)
+            return results
+
+        for img_info in images:
+            self._image_count += 1
+            bbox = (img_info["x0"], img_info["top"], img_info["x1"], img_info["bottom"])
+
+            # Try to extract the actual image data
+            image_path = self._save_image_from_page(page, page_number, self._image_count, img_info)
+            if image_path:
+                results.append((image_path, bbox))
+
+        return results
+
+    def _save_image_from_page(
+        self, page, page_number: int, image_index: int, img_info: dict
+    ) -> str:
+        """Try to extract and save an image, return relative markdown path or empty string."""
+        try:
+            # Access underlying pdfminer page for image XObjects
+            pdfminer_page = page.page
+            resources = pdfminer_page.resources
+
+            if resources is None:
+                return ""
+
+            xobjects = resources.get("XObject", {})
+            if not xobjects:
+                return ""
+
+            # Try to find the image by iterating XObjects
+            for obj_name in xobjects:
+                try:
+                    xobj = xobjects[obj_name].resolve()
+                    if xobj.get("Subtype") != "Image":
+                        continue
+
+                    width = int(xobj.get("Width", 0))
+                    height = int(xobj.get("Height", 0))
+                    if width <= 0 or height <= 0:
+                        continue
+
+                    # Determine format
+                    color_space = xobj.get("ColorSpace", "")
+                    filters = xobj.get("Filter", "")
+
+                    if isinstance(filters, list):
+                        has_jpeg = any("DCT" in str(f) for f in filters)
+                        ext = "jpg" if has_jpeg else "png"
+                    elif "DCT" in str(filters):
+                        ext = "jpg"
+                    else:
+                        ext = "png"
+
+                    filename = f"page{page_number:03d}_img{image_index:02d}.{ext}"
+                    os.makedirs(self.assets_dir, exist_ok=True)
+                    output_path = Path(self.assets_dir) / filename
+
+                    # Extract raw stream data
+                    stream = xobj.get_data()
+                    output_path.write_bytes(stream)
+
+                    return f"assets/{filename}"
+                except Exception:
+                    continue
+
+            # Fallback: just record image position without data
+            logger.debug(
+                "Could not extract image data for image %d on page %d",
+                image_index,
+                page_number,
+            )
+            return ""
+
+        except Exception as exc:
+            logger.warning(
+                "Failed to extract image %d on page %d: %s",
+                image_index,
+                page_number,
+                exc,
+            )
+            return ""

pdf2dotmd/layout_analyzer.py

@@ -0,0 +1,478 @@
+"""Layout analysis for reconstructing reading order from PDF spatial data."""
+
+from __future__ import annotations
+
+import logging
+import re
+from collections import Counter
+from typing import Optional
+
+from .text_block import TextBlock
+
+logger = logging.getLogger(__name__)
+
+# Minimum gap (as fraction of page width) to detect a column boundary
+COLUMN_GAP_MIN_FRACTION = 0.15
+
+# Y-tolerance for grouping characters into lines (points)
+DEFAULT_Y_TOLERANCE = 3.0
+
+# Maximum vertical gap (as multiple of line height) to group lines into a block
+BLOCK_MAX_GAP_LINES = 1.5
+
+# Minimum horizontal overlap ratio for lines to be in the same block
+BLOCK_MIN_OVERLAP = 0.5
+
+
+class _TextLine:
+    """Internal intermediate: a single line of characters on a page."""
+
+    __slots__ = ("chars", "text", "x0", "y0", "x1", "y1", "font_name", "font_size")
+
+    def __init__(self):
+        self.chars: list[dict] = []
+        self.text: str = ""
+        self.x0: float = float("inf")
+        self.y0: float = float("inf")
+        self.x1: float = 0.0
+        self.y1: float = 0.0
+        self.font_name: str = ""
+        self.font_size: float = 0.0
+
+    def add_char(self, char: dict):
+        self.chars.append(char)
+        self.x0 = min(self.x0, char["x0"])
+        self.y0 = min(self.y0, char["y0"])
+        self.x1 = max(self.x1, char["x1"])
+        self.y1 = max(self.y1, char["y1"])
+
+    def finalize(self):
+        if not self.chars:
+            return
+        self.chars.sort(key=lambda c: c["x0"])
+        self.text = "".join(c["text"] for c in self.chars)
+        # Dominant font
+        font_counter = Counter(c.get("fontname", "") for c in self.chars)
+        self.font_name = font_counter.most_common(1)[0][0]
+        size_counter = Counter(c.get("size", 0) for c in self.chars)
+        self.font_size = size_counter.most_common(1)[0][0]
+
+    @property
+    def center_y(self) -> float:
+        return (self.y0 + self.y1) / 2
+
+    @property
+    def width(self) -> float:
+        return self.x1 - self.x0
+
+    @property
+    def height(self) -> float:
+        return self.y1 - self.y0
+
+
+class LayoutAnalyzer:
+    """Analyze PDF page layout to reconstruct reading order."""
+
+    def __init__(self):
+        self._header_texts: list[str] = []
+        self._footer_texts: list[str] = []
+
+    def analyze(self, page, page_number: int) -> list[TextBlock]:
+        """Analyze a pdfplumber page and return TextBlocks in reading order."""
+        chars = page.chars
+        if not chars:
+            logger.debug("Page %d has no text characters", page_number)
+            return []
+
+        page_width = page.width
+        page_height = page.height
+
+        # Step 1: Group characters into text lines
+        lines = self._group_chars_to_lines(chars)
+
+        # Step 2: Group lines into text blocks (paragraphs)
+        blocks = self._group_lines_to_blocks(lines, page_number)
+
+        # Step 3: Detect columns
+        columns = self._detect_columns(blocks, page_width)
+
+        # Step 4: Sort blocks into reading order
+        ordered = self._apply_reading_order(blocks, columns, page_width)
+
+        # Step 5: Detect headers/footers
+        self._detect_headers_footers(ordered, page_height)
+
+        # Step 6: Merge hyphenated line breaks
+        self._merge_hyphenation(ordered)
+
+        return ordered
+
+    def analyze_multi_page(self, pages_with_numbers: list[tuple]) -> list[TextBlock]:
+        """Analyze multiple pages and detect repeated headers/footers.
+
+        pages_with_numbers: list of (page, page_number) tuples
+        """
+        all_blocks: list[TextBlock] = []
+        per_page_top: list[tuple[str, int]] = []
+        per_page_bottom: list[tuple[str, int]] = []
+
+        for page, page_number in pages_with_numbers:
+            blocks = self.analyze(page, page_number)
+            if blocks:
+                # topmost block (highest y1 in PDF coords = visually top)
+                top_block = max(blocks, key=lambda b: b.y1)
+                per_page_top.append((top_block.text.strip(), page_number))
+                # bottommost block
+                bottom_block = min(blocks, key=lambda b: b.y0)
+                per_page_bottom.append((bottom_block.text.strip(), page_number))
+            all_blocks.extend(blocks)
+
+        # Mark repeated headers/footers across pages
+        self._mark_repeated_elements(all_blocks, per_page_top, per_page_bottom)
+
+        return all_blocks
+
+    def _group_chars_to_lines(self, chars: list[dict]) -> list[_TextLine]:
+        """Group characters into text lines by y-coordinate clustering."""
+        if not chars:
+            return []
+
+        # Sort by y center descending (top of page first), then x
+        sorted_chars = sorted(chars, key=lambda c: (-(c["top"] + c["bottom"]) / 2, c["x0"]))
+
+        lines: list[_TextLine] = []
+        current_line = _TextLine()
+
+        for char in sorted_chars:
+            char_cy = (char["top"] + char["bottom"]) / 2
+            # Use adaptive tolerance based on font size
+            tolerance = max(DEFAULT_Y_TOLERANCE, char.get("size", 12) * 0.4)
+
+            if current_line.chars:
+                line_cy = current_line.center_y
+                if abs(char_cy - line_cy) <= tolerance:
+                    current_line.add_char(char)
+                else:
+                    current_line.finalize()
+                    if current_line.text.strip():
+                        lines.append(current_line)
+                    current_line = _TextLine()
+                    current_line.add_char(char)
+            else:
+                current_line.add_char(char)
+
+        if current_line.chars:
+            current_line.finalize()
+            if current_line.text.strip():
+                lines.append(current_line)
+
+        # Sort lines top-to-bottom
+        lines.sort(key=lambda l: -l.center_y)
+        return lines
+
+    def _group_lines_to_blocks(self, lines: list[_TextLine], page_number: int) -> list[TextBlock]:
+        """Group adjacent text lines into TextBlock paragraphs."""
+        if not lines:
+            return []
+
+        blocks: list[TextBlock] = []
+        current_lines: list[_TextLine] = [lines[0]]
+
+        for i in range(1, len(lines)):
+            prev = current_lines[-1]
+            curr = lines[i]
+
+            # Check horizontal overlap
+            overlap = self._horizontal_overlap(prev, curr)
+
+            # Check vertical gap
+            avg_height = (prev.height + curr.height) / 2
+            if avg_height <= 0:
+                avg_height = 12
+            gap = prev.y0 - curr.y1  # positive means curr is below prev
+            if gap < 0:
+                gap = 0
+
+            if overlap >= BLOCK_MIN_OVERLAP and gap < BLOCK_MAX_GAP_LINES * avg_height:
+                current_lines.append(curr)
+            else:
+                blocks.append(self._lines_to_block(current_lines, page_number))
+                current_lines = [curr]
+
+        if current_lines:
+            blocks.append(self._lines_to_block(current_lines, page_number))
+
+        return blocks
+
+    def _horizontal_overlap(self, a: _TextLine, b: _TextLine) -> float:
+        """Calculate horizontal overlap ratio between two lines."""
+        overlap = max(0.0, min(a.x1, b.x1) - max(a.x0, b.x0))
+        min_width = min(a.width, b.width)
+        if min_width <= 0:
+            return 0.0
+        return overlap / min_width
+
+    def _lines_to_block(self, lines: list[_TextLine], page_number: int) -> TextBlock:
+        """Merge multiple lines into a single TextBlock."""
+        if not lines:
+            return TextBlock(page_number=page_number)
+
+        text_parts: list[str] = []
+        for i, line in enumerate(lines):
+            stripped = line.text.strip()
+            if i > 0 and stripped:
+                text_parts.append(" ")
+            text_parts.append(stripped)
+
+        text = "".join(text_parts).strip()
+
+        # Dominant font info
+        font_counter = Counter(l.font_name for l in lines)
+        font_name = font_counter.most_common(1)[0][0]
+        size_counter = Counter(l.font_size for l in lines)
+        font_size = size_counter.most_common(1)[0][0]
+
+        bold = "bold" in font_name.lower()
+
+        return TextBlock(
+            text=text,
+            x0=min(l.x0 for l in lines),
+            y0=min(l.y0 for l in lines),
+            x1=max(l.x1 for l in lines),
+            y1=max(l.y1 for l in lines),
+            font_name=font_name,
+            font_size=font_size,
+            bold=bold,
+            page_number=page_number,
+        )
+
+    def _detect_columns(self, blocks: list[TextBlock], page_width: float) -> list[tuple[float, float]]:
+        """Detect column boundaries. Returns list of (x0, x1) column ranges."""
+        if not blocks or page_width <= 0:
+            return [(0, page_width)]
+
+        # Collect block left edges
+        left_edges = sorted(set(round(b.x0) for b in blocks))
+
+        if len(left_edges) <= 1:
+            return [(0, page_width)]
+
+        # Find the largest horizontal gap between block groups
+        # Divide page into vertical strips and count chars density
+        strip_width = page_width / 40
+        strip_counts: dict[int, int] = {}
+        for b in blocks:
+            center_strip = int(b.center_x / strip_width)
+            strip_counts[center_strip] = strip_counts.get(center_strip, 0) + len(b.text)
+
+        # Find gap ranges with no text
+        all_strips = range(40)
+        gap_ranges: list[tuple[int, int]] = []
+        gap_start = None
+
+        for s in all_strips:
+            if strip_counts.get(s, 0) == 0:
+                if gap_start is None:
+                    gap_start = s
+            else:
+                if gap_start is not None:
+                    gap_ranges.append((gap_start, s - 1))
+                    gap_start = None
+        if gap_start is not None:
+            gap_ranges.append((gap_start, 39))
+
+        # Filter gaps that are wide enough to be column separators
+        min_gap_strips = int(COLUMN_GAP_MIN_FRACTION * 40)
+        significant_gaps = [
+            (s * strip_width, (e + 1) * strip_width)
+            for s, e in gap_ranges
+            if (e - s + 1) >= min_gap_strips
+        ]
+
+        if not significant_gaps:
+            return [(0, page_width)]
+
+        # Build column ranges from gaps
+        columns: list[tuple[float, float]] = []
+        prev_end = 0.0
+        for gap_start, gap_end in significant_gaps:
+            if gap_start > prev_end:
+                columns.append((prev_end, gap_start))
+            prev_end = gap_end
+        if prev_end < page_width:
+            columns.append((prev_end, page_width))
+
+        if len(columns) <= 1:
+            return [(0, page_width)]
+
+        return columns
+
+    def _apply_reading_order(
+        self,
+        blocks: list[TextBlock],
+        columns: list[tuple[float, float]],
+        page_width: float,
+    ) -> list[TextBlock]:
+        """Sort blocks into logical reading order based on detected columns."""
+        if len(columns) <= 1:
+            # Single column: sort top-to-bottom (descending y1 in PDF coords)
+            return sorted(blocks, key=lambda b: -b.y1)
+
+        # Multi-column layout
+        column_blocks: list[list[TextBlock]] = [[] for _ in columns]
+        spanning: list[TextBlock] = []
+
+        for block in blocks:
+            if block.is_spanning(page_width):
+                spanning.append(block)
+            else:
+                # Assign to column by center_x
+                best_col = 0
+                best_dist = float("inf")
+                for i, (cx0, cx1) in enumerate(columns):
+                    col_center = (cx0 + cx1) / 2
+                    dist = abs(block.center_x - col_center)
+                    if dist < best_dist:
+                        best_dist = dist
+                        best_col = i
+                column_blocks[best_col].append(block)
+
+        # Sort each column top-to-bottom
+        for col in column_blocks:
+            col.sort(key=lambda b: -b.y1)
+
+        # Sort spanning blocks top-to-bottom
+        spanning.sort(key=lambda b: -b.y1)
+
+        # Merge: interleaving spanning blocks with column content
+        result: list[TextBlock] = []
+        col_indices = [0] * len(columns)
+        span_idx = 0
+
+        for block in spanning:
+            # Flush column blocks that are above this spanning block
+            for col_i, col in enumerate(column_blocks):
+                while col_indices[col_i] < len(col) and col[col_indices[col_i]].y1 > block.y1:
+                    result.append(col[col_indices[col_i]])
+                    col_indices[col_i] += 1
+            result.append(block)
+
+        # Flush remaining column blocks (left to right, top to bottom within)
+        while any(col_indices[i] < len(column_blocks[i]) for i in range(len(columns))):
+            # Find the topmost remaining block across all columns
+            top_block = None
+            top_col = -1
+            for col_i, col in enumerate(column_blocks):
+                if col_indices[col_i] < len(col):
+                    b = col[col_indices[col_i]]
+                    if top_block is None or b.y1 > top_block.y1:
+                        top_block = b
+                        top_col = col_i
+            if top_block is None:
+                break
+            result.append(top_block)
+            col_indices[top_col] += 1
+
+        return result
+
+    def _detect_headers_footers(self, blocks: list[TextBlock], page_height: float):
+        """Detect header/footer blocks based on position on the page."""
+        if not blocks:
+            return
+
+        for block in blocks:
+            # Top 10% of page → potential header
+            if block.y1 > page_height * 0.92:
+                block.is_header = True
+            # Bottom 8% of page → potential footer
+            if block.y0 < page_height * 0.08:
+                footer_text = block.text.strip()
+                if re.match(r"^[-—\s]*\d+[-—\s]*$", footer_text):
+                    block.is_footer = True
+                elif re.match(r"^\d+\s*/\s*\d+$", footer_text):
+                    block.is_footer = True
+
+    def _mark_repeated_elements(
+        self,
+        all_blocks: list[TextBlock],
+        per_page_top: list[tuple[str, int]],
+        per_page_bottom: list[tuple[str, int]],
+    ):
+        """Mark blocks as headers/footers if they repeat across pages."""
+        if len(per_page_top) < 3:
+            return
+
+        # Check for repeated top texts
+        top_texts = [t for t, _ in per_page_top]
+        top_counter = Counter(top_texts)
+        for text, count in top_counter.items():
+            if count >= 3:
+                for block in all_blocks:
+                    if block.text.strip() == text:
+                        block.is_header = True
+
+        # Check for repeated bottom texts
+        bottom_texts = [t for t, _ in per_page_bottom]
+        bottom_counter = Counter(bottom_texts)
+        for text, count in bottom_counter.items():
+            if count >= 3:
+                for block in all_blocks:
+                    if block.text.strip() == text:
+                        block.is_footer = True
+
+    def _merge_hyphenation(self, blocks: list[TextBlock]):
+        """Merge hyphenated words split across consecutive blocks."""
+        i = 0
+        while i < len(blocks) - 1:
+            current = blocks[i]
+            next_block = blocks[i + 1]
+            if (
+                current.text.endswith("-")
+                and next_block.text
+                and next_block.text[0].islower()
+                and not current.is_header
+                and not current.is_footer
+                and not next_block.is_header
+                and not next_block.is_footer
+            ):
+                # Remove hyphen and join
+                current.text = current.text[:-1] + next_block.text
+                # Expand bounding box
+                current.x1 = max(current.x1, next_block.x1)
+                current.y0 = min(current.y0, next_block.y0)
+                blocks.pop(i + 1)
+                # Don't increment i - check again in case of multi-line hyphenation
+            else:
+                i += 1
+
+    @staticmethod
+    def infer_heading_levels(blocks: list[TextBlock]) -> dict[int, int]:
+        """Infer heading levels (1-6) from font sizes. Returns {block_index: level}."""
+        if not blocks:
+            return {}
+
+        # Collect font sizes from non-header/footer blocks
+        sizes = [b.font_size for b in blocks if not b.is_header and not b.is_footer]
+        if not sizes:
+            return {}
+
+        # Body text = most common size
+        size_counter = Counter(sizes)
+        body_size = size_counter.most_common(1)[0][0]
+
+        # Find unique sizes larger than body
+        larger_sizes = sorted(set(s for s in sizes if s > body_size * 1.1), reverse=True)
+
+        heading_map: dict[int, int] = {}
+        for idx, block in enumerate(blocks):
+            if block.is_header or block.is_footer:
+                continue
+            if block.font_size > body_size * 1.1:
+                try:
+                    level = larger_sizes.index(block.font_size) + 1
+                except ValueError:
+                    level = 6
+                level = min(level, 6)
+                heading_map[idx] = level
+
+        return heading_map

pdf2dotmd/page_processor.py

@@ -0,0 +1,119 @@
+"""Page-level processing: convert analyzed PDF page layout to Markdown lines."""
+
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+from .image_extractor import ImageExtractor
+from .layout_analyzer import LayoutAnalyzer
+from .table_processor import TableProcessor
+from .text_block import TextBlock
+from .utils import escape_markdown
+
+logger = logging.getLogger(__name__)
+
+
+class PageProcessor:
+    """Convert one PDF page into Markdown lines."""
+
+    def __init__(
+        self,
+        layout_analyzer: LayoutAnalyzer,
+        table_processor: TableProcessor,
+        image_extractor: Optional[ImageExtractor],
+        ignore_images: bool = False,
+    ):
+        self.layout_analyzer = layout_analyzer
+        self.table_processor = table_processor
+        self.image_extractor = image_extractor
+        self.ignore_images = ignore_images
+
+    def process_page(self, page, page_number: int) -> list[str]:
+        """Process a pdfplumber page and return Markdown lines."""
+        # Analyze layout to get ordered text blocks
+        blocks = self.layout_analyzer.analyze(page, page_number)
+
+        # Extract tables and their bounding boxes
+        tables_with_bbox = self.table_processor.extract_tables(page)
+        table_bboxes = [bbox for _, bbox in tables_with_bbox]
+        table_lines_by_y: dict[float, list[str]] = {}
+
+        for table_data, bbox in tables_with_bbox:
+            formatted = self.table_processor.format_table(table_data)
+            if formatted:
+                # Key by y-center of table for interleaving with text
+                y_center = (bbox[1] + bbox[3]) / 2
+                table_lines_by_y[y_center] = formatted
+
+        # Extract images (if not ignored)
+        images_with_bbox: list[tuple[str, tuple[float, float, float, float]]] = []
+        if not self.ignore_images and self.image_extractor:
+            images_with_bbox = self.image_extractor.extract_images(page, page_number)
+
+        if not blocks and not tables_with_bbox and not images_with_bbox:
+            return ["<!-- empty page -->"]
+
+        # Filter out blocks that overlap with tables
+        text_blocks = [
+            b for b in blocks
+            if not b.is_header and not b.is_footer
+            and not any(b.overlaps_bbox(tb, threshold=0.6) for tb in table_bboxes)
+        ]
+
+        # Infer heading levels
+        heading_levels = LayoutAnalyzer.infer_heading_levels(blocks)
+
+        # Build output by interleaving text blocks, tables, and images
+        lines: list[str] = []
+
+        # Collect all elements with their y-positions for proper interleaving
+        elements: list[tuple[float, str, list[str]]] = []  # (y1, type, content_lines)
+
+        for idx, block in enumerate(blocks):
+            if block.is_header or block.is_footer:
+                continue
+            if any(block.overlaps_bbox(tb, threshold=0.6) for tb in table_bboxes):
+                continue
+
+            block_lines = self._block_to_markdown(block, idx, heading_levels)
+            if block_lines:
+                elements.append((block.y1, "text", block_lines))
+
+        for y_center, tbl_lines in table_lines_by_y.items():
+            elements.append((y_center, "table", tbl_lines))
+
+        for img_path, bbox in images_with_bbox:
+            elements.append((bbox[3], "image", [f"![]({img_path})", ""]))
+
+        # Sort by y-position (top to bottom = descending y1 in PDF coords)
+        elements.sort(key=lambda e: -e[0])
+
+        for _, _, content_lines in elements:
+            lines.extend(content_lines)
+
+        return lines if lines else ["<!-- empty page -->"]
+
+    def _block_to_markdown(
+        self,
+        block: TextBlock,
+        block_idx: int,
+        heading_levels: dict[int, int],
+    ) -> list[str]:
+        """Convert a single TextBlock to Markdown lines."""
+        text = block.text.strip()
+        if not text:
+            return []
+
+        # Check if this is a heading
+        level = heading_levels.get(block_idx, 0)
+        if level > 0:
+            prefix = "#" * level
+            return [f"{prefix} {escape_markdown(text)}", ""]
+
+        # Bold text
+        if block.bold:
+            return [f"**{escape_markdown(text)}**", ""]
+
+        # Regular paragraph
+        return [escape_markdown(text), ""]

pdf2dotmd/table_processor.py

@@ -0,0 +1,93 @@
+"""Table extraction from PDF pages to Markdown format."""
+
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+from .utils import escape_markdown
+
+logger = logging.getLogger(__name__)
+
+TABLE_SETTINGS = {
+    "vertical_strategy": "lines",
+    "horizontal_strategy": "lines",
+    "snap_tolerance": 5,
+    "join_tolerance": 5,
+    "edge_min_length": 10,
+    "min_words_vertical": 2,
+    "min_words_horizontal": 2,
+}
+
+
+class TableProcessor:
+    """Extract tables from PDF pages and convert to Markdown."""
+
+    def extract_tables(self, page) -> list[tuple[list[list[str]], tuple[float, float, float, float]]]:
+        """Extract tables from a pdfplumber page.
+
+        Returns list of (table_data, bounding_box) tuples.
+        table_data is a list of rows, each row is a list of cell strings.
+        bounding_box is (x0, y0, x1, y1).
+        """
+        try:
+            tables = page.find_tables(table_settings=TABLE_SETTINGS)
+        except Exception as exc:
+            logger.warning("Failed to find tables on page: %s", exc)
+            return []
+
+        results = []
+        for table in tables:
+            try:
+                table_data = table.extract()
+                if not table_data or all(not any(row) for row in table_data):
+                    continue
+                bbox = (table.bbox[0], table.bbox[1], table.bbox[2], table.bbox[3])
+                normalized = self._normalize_table(table_data)
+                results.append((normalized, bbox))
+            except Exception as exc:
+                logger.warning("Failed to extract table: %s", exc)
+                continue
+
+        return results
+
+    def format_table(self, table_data: list[list[str]]) -> list[str]:
+        """Convert table data to Markdown pipe table lines."""
+        if not table_data:
+            return []
+
+        rows = table_data
+        header = rows[0]
+        col_count = len(header)
+        if col_count == 0:
+            return []
+
+        divider = ["---"] * col_count
+        lines = [
+            "| " + " | ".join(header) + " |",
+            "| " + " | ".join(divider) + " |",
+        ]
+
+        for row in rows[1:]:
+            # Pad or truncate to match header
+            if len(row) < col_count:
+                row = row + [""] * (col_count - len(row))
+            elif len(row) > col_count:
+                row = row[:col_count]
+            lines.append("| " + " | ".join(row) + " |")
+
+        return lines
+
+    def _normalize_table(self, table_data: list[list[Optional[str]]]) -> list[list[str]]:
+        """Normalize table cells: replace None with empty string, strip whitespace."""
+        normalized = []
+        for row in table_data:
+            normalized_row = []
+            for cell in row:
+                if cell is None:
+                    normalized_row.append("")
+                else:
+                    text = " ".join(str(cell).split())
+                    normalized_row.append(escape_markdown(text))
+            normalized.append(normalized_row)
+        return normalized

pdf2dotmd/text_block.py

@@ -0,0 +1,63 @@
+"""TextBlock data structure for representing spatial text units on a PDF page."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class TextBlock:
+    """A coherent unit of text extracted from a PDF page with spatial metadata."""
+
+    text: str = ""
+    x0: float = 0.0
+    y0: float = 0.0
+    x1: float = 0.0
+    y1: float = 0.0
+    font_name: str = ""
+    font_size: float = 0.0
+    bold: bool = False
+    page_number: int = 0
+    is_header: bool = False
+    is_footer: bool = False
+
+    @property
+    def width(self) -> float:
+        return self.x1 - self.x0
+
+    @property
+    def height(self) -> float:
+        return self.y1 - self.y0
+
+    @property
+    def center_x(self) -> float:
+        return (self.x0 + self.x1) / 2
+
+    @property
+    def center_y(self) -> float:
+        return (self.y0 + self.y1) / 2
+
+    def horizontal_overlap_ratio(self, other: TextBlock) -> float:
+        """Return the ratio of horizontal overlap to the narrower block's width."""
+        overlap = max(0.0, min(self.x1, other.x1) - max(self.x0, other.x0))
+        min_width = min(self.width, other.width)
+        if min_width <= 0:
+            return 0.0
+        return overlap / min_width
+
+    def overlaps_bbox(self, bbox: tuple[float, float, float, float], threshold: float = 0.5) -> bool:
+        """Check if this block overlaps significantly with a bounding box."""
+        bx0, by0, bx1, by1 = bbox
+        overlap_x = max(0.0, min(self.x1, bx1) - max(self.x0, bx0))
+        overlap_y = max(0.0, min(self.y1, by1) - max(self.y0, by0))
+        if self.width <= 0 or self.height <= 0:
+            return False
+        overlap_area = overlap_x * overlap_y
+        self_area = self.width * self.height
+        return (overlap_area / self_area) > threshold
+
+    def is_spanning(self, page_width: float, ratio: float = 0.8) -> bool:
+        """Check if this block spans most of the page width."""
+        if page_width <= 0:
+            return False
+        return self.width > page_width * ratio

pdf2dotmd/utils.py

@@ -0,0 +1,23 @@
+"""Utility helpers for markdown generation."""
+
+from __future__ import annotations
+
+import re
+from typing import Iterable
+
+
+def escape_markdown(text: str) -> str:
+    """Escape markdown control characters in plain text content."""
+    if not text:
+        return ""
+
+    escaped = text.replace("\\", "\\\\")
+    for ch in ["`", "*", "_", "{", "}", "[", "]", "<", ">", "|"]:
+        escaped = escaped.replace(ch, f"\\{ch}")
+    return escaped
+
+
+def clean_markdown_content(lines: Iterable[str]) -> str:
+    """Join markdown lines and collapse repeated blank lines."""
+    text = "\n".join(lines).rstrip() + "\n"
+    return re.sub(r"\n{3,}", "\n\n", text)

pdf2dotmd-0.0.1.dist-info/METADATA

@@ -0,0 +1,89 @@
+Metadata-Version: 2.4
+Name: pdf2dotmd
+Version: 0.0.1
+Summary: A Python tool for converting PDF files to Markdown
+Author: hnrobert
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/HNRobert/pdf2dotmd
+Project-URL: Repository, https://github.com/HNRobert/pdf2dotmd
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Text Processing :: Markup
+Classifier: Topic :: Utilities
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pdfplumber>=0.11.0
+Dynamic: license-file
+
+# pdf2dotmd
+
+A Python CLI tool that converts PDF files to Markdown format with intelligent layout analysis.
+
+## Features
+
+- **Layout-aware text extraction** — reconstructs logical reading order from PDF spatial data
+- **Multi-column detection** — handles two-column and multi-column layouts
+- **Table extraction** — converts PDF tables to Markdown pipe tables
+- **Heading inference** — detects headings from font size hierarchy
+- **Header/footer filtering** — automatically removes repeated page headers and footers
+- **Image extraction** — extracts embedded images to an `assets/` directory
+- **Ignore images mode** — `--ignore-images` flag for text-only output
+- **Page range selection** — convert specific pages only
+- **Batch conversion** — process multiple PDF files with wildcards
+
+## Installation
+
+```bash
+pip install pdf2dotmd
+```
+
+## Usage
+
+```bash
+# Output to stdout
+pdf2dotmd input.pdf
+
+# Output to file
+pdf2dotmd input.pdf -o output.md
+
+# Skip images, output single Markdown file
+pdf2dotmd input.pdf --ignore-images
+
+# Batch conversion
+pdf2dotmd *.pdf -o output_dir/
+
+# Convert only specific pages
+pdf2dotmd input.pdf -p 1-3
+pdf2dotmd input.pdf -p 1-5,8,10-12
+
+# Verbose logging
+pdf2dotmd input.pdf -v
+```
+
+## How It Works
+
+1. **Character extraction** — uses [pdfplumber](https://github.com/jsvine/pdfplumber) to extract individual characters with position data
+2. **Line grouping** — clusters characters into text lines by y-coordinate proximity
+3. **Block formation** — groups lines into paragraphs based on horizontal alignment and vertical spacing
+4. **Column detection** — identifies multi-column layouts by analyzing horizontal text density gaps
+5. **Reading order** — sorts blocks top-to-bottom, left-to-right, handling spanning titles
+6. **Header/footer removal** — detects repeated elements across pages
+7. **Heading inference** — maps font sizes to heading levels (H1-H6)
+
+## Limitations
+
+- **Scanned PDFs** — OCR is not supported; scanned/image-only PDFs will produce empty output
+- **Encrypted PDFs** — password-protected PDFs are not supported
+- **Complex layouts** — highly irregular layouts may not parse perfectly
+
+## License
+
+MIT

pdf2dotmd-0.0.1.dist-info/RECORD

@@ -0,0 +1,15 @@
+pdf2dotmd/__init__.py,sha256=50eR1xS6VIHHNF2ZUw2yjrBqWFn80nPuIRtn9bVMRAQ,45
+pdf2dotmd/cli.py,sha256=Ni8Fy73uoak10dsXV5lC_OQ6b1TPz8Y0N41R00L6D_0,3182
+pdf2dotmd/converter.py,sha256=a-fXjn_S5RmZJyAHTIQ8ltuwlSwxKpEDGVzQVrkCVjU,6156
+pdf2dotmd/image_extractor.py,sha256=W1e1HySOEH69oHjTqn763AIRBLVmwjT8Z_o1afdWI6A,3726
+pdf2dotmd/layout_analyzer.py,sha256=5k9QogF6wgMrsyx-MBZVY6RetUpO-PN8FOtT_KD_NmI,17193
+pdf2dotmd/page_processor.py,sha256=M5qzTcSp-_khSLQaD0xuwAtSU33pWkg3SsUAuz2glpE,4266
+pdf2dotmd/table_processor.py,sha256=9CyUp45NE9udxO_TBpWmsRK25QnlYfLzwOvc542xnq8,3038
+pdf2dotmd/text_block.py,sha256=MiwC61MT0aY3EoYs3GFNYscl6DrOEvO6xmV9VCPvCMY,2028
+pdf2dotmd/utils.py,sha256=w5deYI99DlsDa1CxfpcGZu2P2k7i28HzIEpB7XnvVq8,653
+pdf2dotmd-0.0.1.dist-info/licenses/LICENSE,sha256=ZfEADodI9tn9tdS_ab8HR0oCeCMl9wTRgwPljxRTsXs,1066
+pdf2dotmd-0.0.1.dist-info/METADATA,sha256=q8ueKEpOQpVGFXmyRVx9eXVxhXaW8TR-w8C2pn2jDj0,3130
+pdf2dotmd-0.0.1.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+pdf2dotmd-0.0.1.dist-info/entry_points.txt,sha256=SQVMAmDp0so-_4paUGfSK6HRGfA4VYxrEEsrEx6TXRA,49
+pdf2dotmd-0.0.1.dist-info/top_level.txt,sha256=fhJsRXlgjF6tT5Wiy50rI5PI66E5uHj_AJ8dR7K-3CI,10
+pdf2dotmd-0.0.1.dist-info/RECORD,,

pdf2dotmd-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

pdf2dotmd-0.0.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+pdf2dotmd = pdf2dotmd.cli:main

pdf2dotmd-0.0.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Robert He
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pdf2dotmd-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+pdf2dotmd