pdf2mj 0.1.0__tar.gz

pdf2mj-0.1.0/.gitignore

@@ -0,0 +1,33 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.egg-info/
+.eggs/
+dist/
+build/
+*.egg
+.venv/
+venv/
+.env
+
+# Testing
+.coverage
+htmlcov/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+
+# Output
+output/
+*.md
+!README.md
+
+# OS
+.DS_Store
+Thumbs.db

pdf2mj-0.1.0/PKG-INFO

@@ -0,0 +1,241 @@
+Metadata-Version: 2.4
+Name: pdf2mj
+Version: 0.1.0
+Summary: Convert PDF documents to Markdown and structured JSON for RAG and LLM pipelines
+Project-URL: Homepage, https://github.com/Ronit-Pai/pdf2mj
+Project-URL: Documentation, https://github.com/Ronit-Pai/pdf2mj#readme
+Project-URL: Repository, https://github.com/Ronit-Pai/pdf2mj
+Author: PDF2MJ Contributors
+License-Expression: MIT
+Keywords: document-conversion,json,llm,markdown,pdf,rag
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Text Processing
+Classifier: Topic :: Utilities
+Requires-Python: >=3.12
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: pillow>=10.0.0
+Requires-Dist: platformdirs>=4.0.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: pymupdf4llm>=0.0.17
+Requires-Dist: pymupdf>=1.24.0
+Requires-Dist: rich>=13.7.0
+Requires-Dist: typer>=0.12.0
+Provides-Extra: dev
+Requires-Dist: pytest-cov>=5.0.0; extra == 'dev'
+Requires-Dist: pytest>=8.0.0; extra == 'dev'
+Provides-Extra: ocr
+Requires-Dist: opencv-python-headless>=4.8.0; extra == 'ocr'
+Requires-Dist: pytesseract>=0.3.10; extra == 'ocr'
+Description-Content-Type: text/markdown
+
+# PDF2MJ
+
+Convert PDF documents to **Markdown** and **structured JSON** for RAG pipelines, LLM preprocessing, and knowledge bases.
+
+## Installation (For Users)
+
+**PyPI:** `pdf2mj` is not published on PyPI yet. Install from source (see [Development Setup](#development-setup-for-contributors)) or publish the package first.
+
+When available on PyPI:
+
+```bash
+pip install pdf2mj
+```
+
+With OCR support:
+
+```bash
+pip install "pdf2mj[ocr]"
+```
+
+### OCR Requirements
+
+OCR is optional and requires:
+
+* Tesseract OCR installed on your system
+* OCR extras installed via:
+
+```bash
+pip install "pdf2mj[ocr]"
+```
+
+### First Run
+
+On the first `pdf2mj` invocation (no arguments), a Rich-powered welcome screen is shown once. State is stored in:
+
+- Linux/macOS: `~/.config/pdf2mj/config.json`
+- Windows: `%APPDATA%\pdf2mj\config.json`
+
+```bash
+pdf2mj welcome   # show the welcome screen again
+pdf2mj doctor    # verify dependencies and environment
+```
+
+### Quick Start
+
+Convert a PDF to Markdown and JSON:
+
+```bash
+pdf2mj document.pdf
+```
+
+Output files are generated next to the source PDF:
+
+```text
+document.md
+document.json
+```
+
+Specify an output directory:
+
+```bash
+pdf2mj document.pdf --output ./output
+```
+
+### Common Examples
+
+Generate all outputs:
+
+```bash
+pdf2mj document.pdf --all --output ./output
+```
+
+Extract images:
+
+```bash
+pdf2mj document.pdf --extract-images
+```
+
+Generate RAG chunks:
+
+```bash
+pdf2mj document.pdf --chunk-size 1000
+```
+
+Use OCR for scanned PDFs:
+
+```bash
+pdf2mj document.pdf --ocr
+```
+
+### CLI Options
+
+| Flag                           | Description                            |
+| ------------------------------ | -------------------------------------- |
+| `--markdown` / `--no-markdown` | Generate Markdown (default: on)        |
+| `--json` / `--no-json`         | Generate structured JSON (default: on) |
+| `--ocr`                        | OCR scanned pages                      |
+| `--extract-images`             | Extract embedded images                |
+| `--figures`                    | Alias for `--extract-images`           |
+| `--chunk-size N`               | Generate RAG chunks                    |
+| `--chunk-overlap N`            | Chunk overlap (default: 200)           |
+| `--output`, `-o`               | Output directory                       |
+| `--verbose`, `-v`              | Detailed logging                       |
+| `--metadata`                   | Export metadata JSON                   |
+| `--tables` / `--no-tables`     | Extract tables                         |
+| `--all`                        | Enable all supported outputs           |
+
+### Utility Commands
+
+| Command           | Description                                      |
+| ----------------- | ------------------------------------------------ |
+| `pdf2mj welcome` | Show the onboarding welcome screen              |
+| `pdf2mj doctor`  | Check Python, dependencies, OCR, and write access |
+
+
+# Development Setup (For Contributors)
+
+## Prerequisites
+
+* Python 3.12+
+* Git
+* Optional: Tesseract OCR
+
+## Clone the Repository
+
+```bash
+git clone https://github.com/Ronit-Pai/pdf2mj.git
+cd pdf2mj
+```
+
+## Create a Development Environment
+
+Using pip:
+
+```bash
+python -m venv .venv
+source .venv/bin/activate  # Linux/macOS
+# .venv\Scripts\activate   # Windows
+
+pip install -e ".[dev]"
+```
+
+Using uv:
+
+```bash
+uv venv
+source .venv/bin/activate
+
+uv pip install -e ".[dev]"
+```
+
+With OCR support:
+
+```bash
+pip install -e ".[dev,ocr]"
+```
+
+## Running Tests
+
+```bash
+pytest
+```
+
+Coverage:
+
+```bash
+pytest --cov=pdf2mj --cov-report=html
+```
+
+## Project Structure
+
+```text
+src/pdf2mj/
+  cli.py
+  config.py
+  welcome.py
+  doctor.py
+  converter.py
+  models.py
+  markdown.py
+  json_export.py
+  metadata.py
+  table_extractor.py
+  image_extractor.py
+  ocr.py
+  chunker.py
+  console_util.py
+
+tests/
+sample_pdfs/
+```
+
+## Local Development
+
+Run directly from source:
+
+```bash
+pdf2mj sample.pdf
+```
+
+or
+
+```bash
+python -m pdf2mj sample.pdf
+```

pdf2mj-0.1.0/README.md

@@ -0,0 +1,205 @@
+# PDF2MJ
+
+Convert PDF documents to **Markdown** and **structured JSON** for RAG pipelines, LLM preprocessing, and knowledge bases.
+
+## Installation (For Users)
+
+**PyPI:** `pdf2mj` is not published on PyPI yet. Install from source (see [Development Setup](#development-setup-for-contributors)) or publish the package first.
+
+When available on PyPI:
+
+```bash
+pip install pdf2mj
+```
+
+With OCR support:
+
+```bash
+pip install "pdf2mj[ocr]"
+```
+
+### OCR Requirements
+
+OCR is optional and requires:
+
+* Tesseract OCR installed on your system
+* OCR extras installed via:
+
+```bash
+pip install "pdf2mj[ocr]"
+```
+
+### First Run
+
+On the first `pdf2mj` invocation (no arguments), a Rich-powered welcome screen is shown once. State is stored in:
+
+- Linux/macOS: `~/.config/pdf2mj/config.json`
+- Windows: `%APPDATA%\pdf2mj\config.json`
+
+```bash
+pdf2mj welcome   # show the welcome screen again
+pdf2mj doctor    # verify dependencies and environment
+```
+
+### Quick Start
+
+Convert a PDF to Markdown and JSON:
+
+```bash
+pdf2mj document.pdf
+```
+
+Output files are generated next to the source PDF:
+
+```text
+document.md
+document.json
+```
+
+Specify an output directory:
+
+```bash
+pdf2mj document.pdf --output ./output
+```
+
+### Common Examples
+
+Generate all outputs:
+
+```bash
+pdf2mj document.pdf --all --output ./output
+```
+
+Extract images:
+
+```bash
+pdf2mj document.pdf --extract-images
+```
+
+Generate RAG chunks:
+
+```bash
+pdf2mj document.pdf --chunk-size 1000
+```
+
+Use OCR for scanned PDFs:
+
+```bash
+pdf2mj document.pdf --ocr
+```
+
+### CLI Options
+
+| Flag                           | Description                            |
+| ------------------------------ | -------------------------------------- |
+| `--markdown` / `--no-markdown` | Generate Markdown (default: on)        |
+| `--json` / `--no-json`         | Generate structured JSON (default: on) |
+| `--ocr`                        | OCR scanned pages                      |
+| `--extract-images`             | Extract embedded images                |
+| `--figures`                    | Alias for `--extract-images`           |
+| `--chunk-size N`               | Generate RAG chunks                    |
+| `--chunk-overlap N`            | Chunk overlap (default: 200)           |
+| `--output`, `-o`               | Output directory                       |
+| `--verbose`, `-v`              | Detailed logging                       |
+| `--metadata`                   | Export metadata JSON                   |
+| `--tables` / `--no-tables`     | Extract tables                         |
+| `--all`                        | Enable all supported outputs           |
+
+### Utility Commands
+
+| Command           | Description                                      |
+| ----------------- | ------------------------------------------------ |
+| `pdf2mj welcome` | Show the onboarding welcome screen              |
+| `pdf2mj doctor`  | Check Python, dependencies, OCR, and write access |
+
+
+# Development Setup (For Contributors)
+
+## Prerequisites
+
+* Python 3.12+
+* Git
+* Optional: Tesseract OCR
+
+## Clone the Repository
+
+```bash
+git clone https://github.com/Ronit-Pai/pdf2mj.git
+cd pdf2mj
+```
+
+## Create a Development Environment
+
+Using pip:
+
+```bash
+python -m venv .venv
+source .venv/bin/activate  # Linux/macOS
+# .venv\Scripts\activate   # Windows
+
+pip install -e ".[dev]"
+```
+
+Using uv:
+
+```bash
+uv venv
+source .venv/bin/activate
+
+uv pip install -e ".[dev]"
+```
+
+With OCR support:
+
+```bash
+pip install -e ".[dev,ocr]"
+```
+
+## Running Tests
+
+```bash
+pytest
+```
+
+Coverage:
+
+```bash
+pytest --cov=pdf2mj --cov-report=html
+```
+
+## Project Structure
+
+```text
+src/pdf2mj/
+  cli.py
+  config.py
+  welcome.py
+  doctor.py
+  converter.py
+  models.py
+  markdown.py
+  json_export.py
+  metadata.py
+  table_extractor.py
+  image_extractor.py
+  ocr.py
+  chunker.py
+  console_util.py
+
+tests/
+sample_pdfs/
+```
+
+## Local Development
+
+Run directly from source:
+
+```bash
+pdf2mj sample.pdf
+```
+
+or
+
+```bash
+python -m pdf2mj sample.pdf
+```

pdf2mj-0.1.0/pyproject.toml

@@ -0,0 +1,70 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "pdf2mj"
+version = "0.1.0"
+description = "Convert PDF documents to Markdown and structured JSON for RAG and LLM pipelines"
+readme = "README.md"
+requires-python = ">=3.12"
+license = "MIT"
+authors = [{ name = "PDF2MJ Contributors" }]
+keywords = ["pdf", "markdown", "json", "rag", "llm", "document-conversion"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Text Processing",
+    "Topic :: Utilities",
+]
+dependencies = [
+    "typer>=0.12.0",
+    "pymupdf>=1.24.0",
+    "pymupdf4llm>=0.0.17",
+    "rich>=13.7.0",
+    "pydantic>=2.0.0",
+    "pandas>=2.0.0",
+    "pillow>=10.0.0",
+    "platformdirs>=4.0.0",
+]
+
+[project.optional-dependencies]
+ocr = ["pytesseract>=0.3.10", "opencv-python-headless>=4.8.0"]
+dev = ["pytest>=8.0.0", "pytest-cov>=5.0.0"]
+
+[project.scripts]
+pdf2mj = "pdf2mj.cli:run"
+
+[project.urls]
+Homepage = "https://github.com/Ronit-Pai/pdf2mj"
+Documentation = "https://github.com/Ronit-Pai/pdf2mj#readme"
+Repository = "https://github.com/Ronit-Pai/pdf2mj"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/pdf2mj"]
+
+[tool.hatch.build.targets.sdist]
+include = ["src/pdf2mj", "tests", "README.md", "plan.md"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]
+addopts = "-v --tb=short"
+
+[tool.coverage.run]
+source = ["pdf2mj"]
+branch = true
+
+[tool.coverage.report]
+fail_under = 73
+show_missing = true
+exclude_lines = [
+    "pragma: no cover",
+    "if TYPE_CHECKING:",
+    "raise NotImplementedError",
+]

pdf2mj-0.1.0/sample_pdfs/README.md

@@ -0,0 +1,22 @@
+# Sample PDFs
+
+Place sample PDF files here for manual testing, for example:
+
+- `text_sample.pdf` — digital text PDF
+- `scan_sample.pdf` — scanned document (use `--ocr`)
+- `tables_sample.pdf` — document with tables
+- `images_sample.pdf` — document with embedded images
+
+Generate a quick test PDF with the CLI after install:
+
+```bash
+python -c "
+import fitz
+doc = fitz.open()
+p = doc.new_page()
+p.insert_text((72, 72), 'Hello PDF2MJ', fontsize=14)
+doc.save('sample_pdfs/demo.pdf')
+doc.close()
+"
+pdf2mj sample_pdfs/demo.pdf --output sample_pdfs/output --all
+```

pdf2mj-0.1.0/src/pdf2mj/__init__.py

@@ -0,0 +1,3 @@
+"""PDF2MJ — convert PDF documents to Markdown and structured JSON."""
+
+__version__ = "0.1.0"

pdf2mj-0.1.0/src/pdf2mj/chunker.py

@@ -0,0 +1,58 @@
+"""Text chunking for RAG pipelines."""
+
+from __future__ import annotations
+
+import re
+
+from pdf2mj.models import Chunk, Document
+
+
+def _estimate_tokens(text: str) -> int:
+    return len(re.findall(r"\S+", text))
+
+
+def chunk_document(
+    document: Document,
+    chunk_size: int = 1000,
+    overlap: int = 200,
+) -> list[Chunk]:
+    """Split document text into overlapping chunks with page references."""
+    chunks: list[Chunk] = []
+    chunk_index = 0
+
+    for page in document.pages:
+        page_text_parts: list[str] = []
+        for block in page.blocks:
+            if block.type in ("image",):
+                continue
+            text = block.text.strip()
+            if text:
+                page_text_parts.append(text)
+
+        full_text = "\n\n".join(page_text_parts)
+        if not full_text.strip():
+            continue
+
+        words = full_text.split()
+        if not words:
+            continue
+
+        start = 0
+        while start < len(words):
+            end = min(start + chunk_size, len(words))
+            chunk_words = words[start:end]
+            chunk_text = " ".join(chunk_words)
+            chunk_index += 1
+            chunks.append(
+                Chunk(
+                    chunk_id=f"c{chunk_index}",
+                    page=page.page_number,
+                    text=chunk_text,
+                    tokens=_estimate_tokens(chunk_text),
+                )
+            )
+            if end >= len(words):
+                break
+            start = max(0, end - overlap)
+
+    return chunks