docmd-cli 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- docmd_cli-0.1.0/.github/workflows/ci.yml +23 -0
- docmd_cli-0.1.0/.github/workflows/release.yml +36 -0
- docmd_cli-0.1.0/.gitignore +10 -0
- docmd_cli-0.1.0/ARCHITECTURE.md +149 -0
- docmd_cli-0.1.0/LICENSE +21 -0
- docmd_cli-0.1.0/PKG-INFO +151 -0
- docmd_cli-0.1.0/README.md +124 -0
- docmd_cli-0.1.0/docmd/__init__.py +83 -0
- docmd_cli-0.1.0/docmd/cli.py +68 -0
- docmd_cli-0.1.0/docmd/config.py +51 -0
- docmd_cli-0.1.0/docmd/converters/__init__.py +0 -0
- docmd_cli-0.1.0/docmd/converters/base.py +36 -0
- docmd_cli-0.1.0/docmd/converters/marker_converter.py +89 -0
- docmd_cli-0.1.0/docmd/converters/registry.py +43 -0
- docmd_cli-0.1.0/docmd/errors.py +60 -0
- docmd_cli-0.1.0/docmd/postprocess/__init__.py +0 -0
- docmd_cli-0.1.0/docmd/postprocess/heading_normalize.py +76 -0
- docmd_cli-0.1.0/docmd/postprocess/image_handling.py +54 -0
- docmd_cli-0.1.0/docmd/postprocess/table_cleanup.py +129 -0
- docmd_cli-0.1.0/pyproject.toml +48 -0
- docmd_cli-0.1.0/tests/fixtures/generate_fixtures.py +100 -0
- docmd_cli-0.1.0/tests/fixtures/sample.docx +0 -0
- docmd_cli-0.1.0/tests/fixtures/sample.pdf +74 -0
- docmd_cli-0.1.0/tests/test_cli.py +32 -0
- docmd_cli-0.1.0/tests/test_converters.py +50 -0
- docmd_cli-0.1.0/tests/test_postprocess.py +100 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
steps:
|
|
12
|
+
- uses: actions/checkout@v4
|
|
13
|
+
- uses: actions/setup-python@v5
|
|
14
|
+
with:
|
|
15
|
+
python-version: "3.11"
|
|
16
|
+
- name: Install weasyprint system dependencies
|
|
17
|
+
# DOCX/PPTX conversion goes through weasyprint, which needs native
|
|
18
|
+
# Pango/GObject/Cairo libraries that pip cannot provide.
|
|
19
|
+
run: sudo apt-get update && sudo apt-get install -y libpango-1.0-0 libpangocairo-1.0-0 libgdk-pixbuf2.0-0 libffi-dev shared-mime-info
|
|
20
|
+
- name: Install docmd
|
|
21
|
+
run: pip install -e ".[full,dev]"
|
|
22
|
+
- name: Run tests
|
|
23
|
+
run: pytest -v
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
release:
|
|
5
|
+
types: [published]
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
build:
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
steps:
|
|
11
|
+
- uses: actions/checkout@v4
|
|
12
|
+
- uses: actions/setup-python@v5
|
|
13
|
+
with:
|
|
14
|
+
python-version: "3.11"
|
|
15
|
+
- name: Build sdist and wheel
|
|
16
|
+
run: |
|
|
17
|
+
python -m pip install --upgrade build
|
|
18
|
+
python -m build
|
|
19
|
+
- uses: actions/upload-artifact@v4
|
|
20
|
+
with:
|
|
21
|
+
name: dist
|
|
22
|
+
path: dist/
|
|
23
|
+
|
|
24
|
+
publish:
|
|
25
|
+
needs: build
|
|
26
|
+
runs-on: ubuntu-latest
|
|
27
|
+
environment: pypi
|
|
28
|
+
permissions:
|
|
29
|
+
id-token: write # required for PyPI trusted publishing (OIDC) - no token stored
|
|
30
|
+
steps:
|
|
31
|
+
- uses: actions/download-artifact@v4
|
|
32
|
+
with:
|
|
33
|
+
name: dist
|
|
34
|
+
path: dist/
|
|
35
|
+
- name: Publish to PyPI
|
|
36
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
# Architecture: Doc-to-Markdown API
|
|
2
|
+
|
|
3
|
+
## The pitch (keep this pinned above your desk)
|
|
4
|
+
We are not competing on conversion quality. We are selling **"hit an endpoint, get clean Markdown back"** — no Python env, no 8GB+ RAM, no GPU, no dependency hell. The open-source core proves the engine works and builds trust. The hosted API sells convenience.
|
|
5
|
+
|
|
6
|
+
## Positioning risk (checked 2026-09-18)
|
|
7
|
+
Marker's *code* is Apache-2.0 (no restriction). Marker's *model weights* use a modified
|
|
8
|
+
Open RAIL-M license: free for research, personal use, and organizations under $5M in
|
|
9
|
+
funding/revenue; beyond that, a commercial license from Datalab is required. This is not
|
|
10
|
+
a launch blocker for a bootstrapped start, but it is a real ceiling to plan around if the
|
|
11
|
+
hosted API grows past that threshold. Separately, Datalab already runs its own hosted
|
|
12
|
+
"upload a file, get Markdown back" API (datalab.to/pricing) with a free tier and
|
|
13
|
+
pay-as-you-go pricing — so "convenience" alone is not a durable moat, since Datalab
|
|
14
|
+
already sells that. The real differentiator has to be the post-processing quality (see
|
|
15
|
+
below), not just "no setup required." Every feature that's free elsewhere (e.g. OCR,
|
|
16
|
+
which Marker already bundles) should stay free in `docmd` too; gating something users
|
|
17
|
+
can trivially get by calling Marker directly doesn't protect revenue, it just damages
|
|
18
|
+
trust.
|
|
19
|
+
|
|
20
|
+
---
|
|
21
|
+
|
|
22
|
+
## System overview
|
|
23
|
+
|
|
24
|
+
```
|
|
25
|
+
+---------------------------+
|
|
26
|
+
| docmd (OSS library) | <- pip install docmd-cli
|
|
27
|
+
| Python CLI + library | <- free, MIT license
|
|
28
|
+
| wraps Marker under |
|
|
29
|
+
| the hood |
|
|
30
|
+
+--------------+--------------+
|
|
31
|
+
| same core logic,
|
|
32
|
+
| imported not duplicated
|
|
33
|
+
v
|
|
34
|
+
+--------------+ +---------------------------+ +-----------------+
|
|
35
|
+
| Client |--->| API Gateway (FastAPI) |--->| Inference |
|
|
36
|
+
| (curl, SDK, | | - auth (API keys) | | Worker (GPU) |
|
|
37
|
+
| your docs) | | - rate limiting | | Modal / RunPod |
|
|
38
|
+
+--------------+ | - usage metering | | runs Marker |
|
|
39
|
+
^ | - job queue | +--------+---------+
|
|
40
|
+
| +--------------+--------------+ |
|
|
41
|
+
| | |
|
|
42
|
+
| v |
|
|
43
|
+
| +---------------------------+ |
|
|
44
|
+
| | Postgres (Supabase) | |
|
|
45
|
+
| | - users, api_keys |<---------------+
|
|
46
|
+
| | - usage_events | writes result +
|
|
47
|
+
| | - jobs (status/result) | usage back
|
|
48
|
+
| +---------------------------+
|
|
49
|
+
| ^
|
|
50
|
+
| |
|
|
51
|
+
| +---------------------------+
|
|
52
|
+
+------------>| Stripe (billing) |
|
|
53
|
+
webhook events | - subscriptions |
|
|
54
|
+
| - usage-based invoicing |
|
|
55
|
+
+---------------------------+
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
**Key architectural decision: the OSS library and the hosted API share the same core conversion code.** Put the actual conversion logic (file -> Markdown) in the open-source package. The API is a thin, closed-source wrapper around it that adds auth, billing, queuing, and GPU orchestration. This way:
|
|
59
|
+
- The OSS repo is genuinely useful standalone (real trust, real stars)
|
|
60
|
+
- You never duplicate logic between "free" and "paid"
|
|
61
|
+
- Improvements to the core benefit both
|
|
62
|
+
|
|
63
|
+
---
|
|
64
|
+
|
|
65
|
+
## Repo structure
|
|
66
|
+
|
|
67
|
+
This repo currently contains **Stage 1 only**: the `docmd/` OSS package. The `api/`
|
|
68
|
+
folder (hosted API) is a later stage, described below for context but not yet built.
|
|
69
|
+
|
|
70
|
+
```
|
|
71
|
+
docmd/
|
|
72
|
+
|-- README.md
|
|
73
|
+
|-- ARCHITECTURE.md
|
|
74
|
+
|-- LICENSE # MIT for the core package
|
|
75
|
+
|-- pyproject.toml # package config for `docmd` on PyPI
|
|
76
|
+
|-- docmd/ # --- OSS PACKAGE (public, pip-installable) ---
|
|
77
|
+
| |-- __init__.py
|
|
78
|
+
| |-- cli.py # `docmd convert file.pdf` entrypoint
|
|
79
|
+
| |-- converters/
|
|
80
|
+
| | |-- __init__.py
|
|
81
|
+
| | |-- base.py # Converter protocol/interface
|
|
82
|
+
| | |-- marker_converter.py # wraps datalab-to/marker
|
|
83
|
+
| | `-- registry.py # maps file type -> converter
|
|
84
|
+
| |-- postprocess/ # THIS is where docmd earns its existence
|
|
85
|
+
| | |-- table_cleanup.py # fix mangled table structure from raw Marker output
|
|
86
|
+
| | |-- heading_normalize.py # consistent heading levels, no orphaned headers
|
|
87
|
+
| | `-- image_handling.py # consistent alt-text / placeholder convention
|
|
88
|
+
| |-- config.py
|
|
89
|
+
| `-- errors.py # error taxonomy
|
|
90
|
+
|-- tests/
|
|
91
|
+
| |-- test_cli.py
|
|
92
|
+
| |-- test_converters.py
|
|
93
|
+
| |-- test_postprocess.py
|
|
94
|
+
| `-- fixtures/ # sample pdf/docx test files (generated)
|
|
95
|
+
`-- .github/
|
|
96
|
+
`-- workflows/
|
|
97
|
+
`-- ci.yml # tests on PR
|
|
98
|
+
|
|
99
|
+
# Not yet built (later stage):
|
|
100
|
+
# api/ - FastAPI hosted service (auth, billing, queue, GPU worker)
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
### Post-processing: the actual product
|
|
104
|
+
Raw Marker output is good but not perfect for RAG use — this is where `docmd` needs to add real value instead of being pure plumbing:
|
|
105
|
+
- **Table cleanup**: fix structure that comes out mangled from complex layouts
|
|
106
|
+
- **Heading normalization**: consistent heading levels, no orphaned/duplicate headers
|
|
107
|
+
- **Image handling**: a consistent convention for alt-text or placeholders (RAG pipelines need to know what to do with images, even if just "skip")
|
|
108
|
+
- **Optional LLM polish pass** (v2, opt-in, costs extra): run a cheap LLM call over the output to fix remaining formatting oddities — this is a legitimate premium feature since it has a real marginal cost
|
|
109
|
+
|
|
110
|
+
Treat this folder as the actual differentiator. A thin wrapper is a weekend project; good post-processing is a product.
|
|
111
|
+
|
|
112
|
+
---
|
|
113
|
+
|
|
114
|
+
## Tech stack (with reasoning)
|
|
115
|
+
|
|
116
|
+
| Layer | Choice | Why |
|
|
117
|
+
|---|---|---|
|
|
118
|
+
| OSS core | Python, wraps `marker-pdf` (pinned `==2.0.0`) | Don't reinvent conversion; Marker has ~40k combined stars (with Surya) and active maintenance |
|
|
119
|
+
| API framework (later stage) | FastAPI | Async-friendly, great for I/O-bound job queuing, auto-generates OpenAPI docs |
|
|
120
|
+
| GPU inference (later stage) | Modal (first choice) or RunPod | Modal has better Python DX and publishes an official Marker deployment guide; RunPod is often cheaper at scale |
|
|
121
|
+
| Database (later stage) | Postgres via Supabase | Free tier is generous, easy to self-host later |
|
|
122
|
+
| Billing (later stage) | Stripe, prepaid credits by default | Avoids surprise invoices |
|
|
123
|
+
| API deploy (later stage) | Railway or Fly.io | Simple deploys, no k8s complexity |
|
|
124
|
+
|
|
125
|
+
---
|
|
126
|
+
|
|
127
|
+
## Packaging: optional dependencies
|
|
128
|
+
|
|
129
|
+
- `pip install docmd-cli` -> core PDF conversion (Marker bundles OCR here already; there is
|
|
130
|
+
no separate lean/no-OCR base install, since Marker's own base dependencies include
|
|
131
|
+
`surya-ocr`)
|
|
132
|
+
- `pip install docmd-cli[full]` -> adds DOCX/PPTX/EPUB/XLSX support via Marker's `full` extra
|
|
133
|
+
|
|
134
|
+
## License clarity
|
|
135
|
+
|
|
136
|
+
MIT for the `docmd` wrapper is fine and expected for an OSS trust play. Marker's code is
|
|
137
|
+
Apache-2.0. Marker's *model weights* are under a modified Open RAIL-M license: free for
|
|
138
|
+
research, personal use, and organizations under $5M funding/revenue; beyond that, a paid
|
|
139
|
+
license from Datalab is required (see datalab.to/pricing). Re-check this before scaling
|
|
140
|
+
the hosted API past that threshold.
|
|
141
|
+
|
|
142
|
+
## What "done" looks like for Stage 1 (this repo, today)
|
|
143
|
+
|
|
144
|
+
- [x] `docmd` installable via `pip install -e .` locally
|
|
145
|
+
- [x] Converts a PDF to Markdown from the CLI in one command
|
|
146
|
+
- [x] Post-processing (table cleanup, heading normalization) visibly improves on raw Marker output
|
|
147
|
+
- [x] README has a compelling before/after example and a 30-second quickstart
|
|
148
|
+
- [ ] Published to PyPI
|
|
149
|
+
- [ ] Hosted API (later stage, not built yet)
|
docmd_cli-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 taherzribi
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
docmd_cli-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: docmd-cli
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Convert PDFs, DOCX, and PPTX to clean, structure-preserving Markdown.
|
|
5
|
+
Project-URL: Homepage, https://github.com/taherzribi/docmd
|
|
6
|
+
Project-URL: Issues, https://github.com/taherzribi/docmd/issues
|
|
7
|
+
Author: taherzribi
|
|
8
|
+
License: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: document-conversion,docx,markdown,pdf,pptx,rag
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Topic :: Text Processing :: Markup :: Markdown
|
|
17
|
+
Requires-Python: <4,>=3.10
|
|
18
|
+
Requires-Dist: click<9,>=8.2.0
|
|
19
|
+
Requires-Dist: marker-pdf==2.0.0
|
|
20
|
+
Provides-Extra: dev
|
|
21
|
+
Requires-Dist: pytest>=8.0.0; extra == 'dev'
|
|
22
|
+
Requires-Dist: python-docx>=1.1.0; extra == 'dev'
|
|
23
|
+
Requires-Dist: reportlab>=4.0.0; extra == 'dev'
|
|
24
|
+
Provides-Extra: full
|
|
25
|
+
Requires-Dist: marker-pdf[full]==2.0.0; extra == 'full'
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
|
|
28
|
+
# docmd
|
|
29
|
+
|
|
30
|
+
**Convert PDFs, DOCX, and PPTX to clean, structure-preserving Markdown — from the CLI or a Python library.**
|
|
31
|
+
|
|
32
|
+
Built for feeding documents into LLM and RAG pipelines, where clean Markdown beats raw text extraction.
|
|
33
|
+
|
|
34
|
+
*(Published on PyPI as `docmd-cli` since `docmd` was already taken by an unrelated
|
|
35
|
+
package — the import (`import docmd`) and CLI command (`docmd convert ...`) are
|
|
36
|
+
unaffected.)*
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
pip install docmd-cli
|
|
40
|
+
docmd convert report.pdf
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
```python
|
|
44
|
+
from docmd import convert
|
|
45
|
+
|
|
46
|
+
markdown = convert("report.pdf")
|
|
47
|
+
print(markdown)
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## Why docmd
|
|
51
|
+
|
|
52
|
+
Great open-source document-to-Markdown converters already exist (docmd is built on
|
|
53
|
+
[Marker](https://github.com/datalab-to/marker)). The problem isn't quality — it's that
|
|
54
|
+
running them yourself means a Python environment, several GB of RAM, ideally a GPU, and
|
|
55
|
+
a non-trivial setup process before you convert your first file.
|
|
56
|
+
|
|
57
|
+
`docmd` wraps that engine with sane defaults and adds real post-processing on top
|
|
58
|
+
(table cleanup, heading normalization) so the output is closer to what a RAG pipeline
|
|
59
|
+
actually wants, not just raw model output.
|
|
60
|
+
|
|
61
|
+
A hosted API (`POST` a file, get Markdown back, no local setup) is planned — see
|
|
62
|
+
[ARCHITECTURE.md](ARCHITECTURE.md). It is not live yet; this repo is the open-source
|
|
63
|
+
core, usable standalone today.
|
|
64
|
+
|
|
65
|
+
## What it handles
|
|
66
|
+
|
|
67
|
+
| Input | Output |
|
|
68
|
+
|---|---|
|
|
69
|
+
| PDF (text-based) | Markdown with preserved headings, lists, tables |
|
|
70
|
+
| PDF (scanned) | Markdown via OCR — bundled by Marker, free |
|
|
71
|
+
| DOCX | Markdown with formatting preserved (`pip install docmd-cli[full]`) |
|
|
72
|
+
| PPTX | Markdown, one section per slide (`pip install docmd-cli[full]`) |
|
|
73
|
+
|
|
74
|
+
## Quickstart
|
|
75
|
+
|
|
76
|
+
**CLI**
|
|
77
|
+
```bash
|
|
78
|
+
pip install docmd-cli
|
|
79
|
+
docmd convert my-file.pdf -o output.md
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
**Python**
|
|
83
|
+
```python
|
|
84
|
+
from docmd import convert
|
|
85
|
+
|
|
86
|
+
# From a file path
|
|
87
|
+
markdown = convert("my-file.pdf")
|
|
88
|
+
|
|
89
|
+
# From bytes
|
|
90
|
+
with open("my-file.pdf", "rb") as f:
|
|
91
|
+
markdown = convert(f.read(), filename="my-file.pdf")
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
## Installing DOCX/PPTX support
|
|
95
|
+
|
|
96
|
+
The base install (`pip install docmd-cli`) covers PDF only and stays lean. DOCX and
|
|
97
|
+
PPTX need Marker's own additional dependencies:
|
|
98
|
+
|
|
99
|
+
```bash
|
|
100
|
+
pip install "docmd-cli[full]"
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
DOCX/PPTX conversion also needs [weasyprint](https://doc.courtbouillon.org/weasyprint/stable/first_steps.html#installation)'s
|
|
104
|
+
native Pango/GObject/Cairo libraries, which `pip` cannot install for you:
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
# macOS
|
|
108
|
+
brew install pango
|
|
109
|
+
|
|
110
|
+
# Debian/Ubuntu
|
|
111
|
+
sudo apt-get install libpango-1.0-0 libpangocairo-1.0-0 libgdk-pixbuf2.0-0 libffi-dev shared-mime-info
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
PDF conversion (the base install) does not need this.
|
|
115
|
+
|
|
116
|
+
## How it works
|
|
117
|
+
|
|
118
|
+
`docmd` wraps [Marker](https://github.com/datalab-to/marker) with sane defaults and a
|
|
119
|
+
clean output format, then runs its own post-processing pass
|
|
120
|
+
(`docmd/postprocess/`) to fix table structure and normalize heading levels — see
|
|
121
|
+
[ARCHITECTURE.md](ARCHITECTURE.md) for why this is the actual differentiation, not
|
|
122
|
+
just a thin wrapper.
|
|
123
|
+
|
|
124
|
+
## License
|
|
125
|
+
|
|
126
|
+
The `docmd` wrapper code is MIT — see [LICENSE](LICENSE).
|
|
127
|
+
|
|
128
|
+
`docmd` depends on [Marker](https://github.com/datalab-to/marker), whose *code* is
|
|
129
|
+
Apache-2.0 and whose *model weights* are licensed under a modified Open RAIL-M license:
|
|
130
|
+
free for research, personal use, and organizations under $5M in funding or revenue.
|
|
131
|
+
Commercial use beyond that threshold requires a license from
|
|
132
|
+
[Datalab](https://www.datalab.to/pricing). This applies to you if you deploy `docmd`
|
|
133
|
+
commercially at scale — check Marker's current license terms directly before doing so.
|
|
134
|
+
|
|
135
|
+
## Roadmap
|
|
136
|
+
|
|
137
|
+
- [x] PDF, DOCX, PPTX → Markdown
|
|
138
|
+
- [x] Table cleanup / heading normalization post-processing
|
|
139
|
+
- [ ] Hosted API
|
|
140
|
+
- [ ] OCR quality tuning for scanned documents
|
|
141
|
+
- [ ] Batch conversion endpoint
|
|
142
|
+
- [ ] HTML output option
|
|
143
|
+
|
|
144
|
+
## Contributing
|
|
145
|
+
|
|
146
|
+
Issues and PRs welcome. If you're hitting a conversion quality issue, please include a
|
|
147
|
+
sample file (or a minimal reproduction) — it makes fixes much faster.
|
|
148
|
+
|
|
149
|
+
---
|
|
150
|
+
|
|
151
|
+
*If docmd saves you the trouble of setting up your own PDF-parsing pipeline, consider starring the repo — it's how other people find it.*
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# docmd
|
|
2
|
+
|
|
3
|
+
**Convert PDFs, DOCX, and PPTX to clean, structure-preserving Markdown — from the CLI or a Python library.**
|
|
4
|
+
|
|
5
|
+
Built for feeding documents into LLM and RAG pipelines, where clean Markdown beats raw text extraction.
|
|
6
|
+
|
|
7
|
+
*(Published on PyPI as `docmd-cli` since `docmd` was already taken by an unrelated
|
|
8
|
+
package — the import (`import docmd`) and CLI command (`docmd convert ...`) are
|
|
9
|
+
unaffected.)*
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
pip install docmd-cli
|
|
13
|
+
docmd convert report.pdf
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
```python
|
|
17
|
+
from docmd import convert
|
|
18
|
+
|
|
19
|
+
markdown = convert("report.pdf")
|
|
20
|
+
print(markdown)
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
## Why docmd
|
|
24
|
+
|
|
25
|
+
Great open-source document-to-Markdown converters already exist (docmd is built on
|
|
26
|
+
[Marker](https://github.com/datalab-to/marker)). The problem isn't quality — it's that
|
|
27
|
+
running them yourself means a Python environment, several GB of RAM, ideally a GPU, and
|
|
28
|
+
a non-trivial setup process before you convert your first file.
|
|
29
|
+
|
|
30
|
+
`docmd` wraps that engine with sane defaults and adds real post-processing on top
|
|
31
|
+
(table cleanup, heading normalization) so the output is closer to what a RAG pipeline
|
|
32
|
+
actually wants, not just raw model output.
|
|
33
|
+
|
|
34
|
+
A hosted API (`POST` a file, get Markdown back, no local setup) is planned — see
|
|
35
|
+
[ARCHITECTURE.md](ARCHITECTURE.md). It is not live yet; this repo is the open-source
|
|
36
|
+
core, usable standalone today.
|
|
37
|
+
|
|
38
|
+
## What it handles
|
|
39
|
+
|
|
40
|
+
| Input | Output |
|
|
41
|
+
|---|---|
|
|
42
|
+
| PDF (text-based) | Markdown with preserved headings, lists, tables |
|
|
43
|
+
| PDF (scanned) | Markdown via OCR — bundled by Marker, free |
|
|
44
|
+
| DOCX | Markdown with formatting preserved (`pip install docmd-cli[full]`) |
|
|
45
|
+
| PPTX | Markdown, one section per slide (`pip install docmd-cli[full]`) |
|
|
46
|
+
|
|
47
|
+
## Quickstart
|
|
48
|
+
|
|
49
|
+
**CLI**
|
|
50
|
+
```bash
|
|
51
|
+
pip install docmd-cli
|
|
52
|
+
docmd convert my-file.pdf -o output.md
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
**Python**
|
|
56
|
+
```python
|
|
57
|
+
from docmd import convert
|
|
58
|
+
|
|
59
|
+
# From a file path
|
|
60
|
+
markdown = convert("my-file.pdf")
|
|
61
|
+
|
|
62
|
+
# From bytes
|
|
63
|
+
with open("my-file.pdf", "rb") as f:
|
|
64
|
+
markdown = convert(f.read(), filename="my-file.pdf")
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## Installing DOCX/PPTX support
|
|
68
|
+
|
|
69
|
+
The base install (`pip install docmd-cli`) covers PDF only and stays lean. DOCX and
|
|
70
|
+
PPTX need Marker's own additional dependencies:
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
pip install "docmd-cli[full]"
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
DOCX/PPTX conversion also needs [weasyprint](https://doc.courtbouillon.org/weasyprint/stable/first_steps.html#installation)'s
|
|
77
|
+
native Pango/GObject/Cairo libraries, which `pip` cannot install for you:
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
# macOS
|
|
81
|
+
brew install pango
|
|
82
|
+
|
|
83
|
+
# Debian/Ubuntu
|
|
84
|
+
sudo apt-get install libpango-1.0-0 libpangocairo-1.0-0 libgdk-pixbuf2.0-0 libffi-dev shared-mime-info
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
PDF conversion (the base install) does not need this.
|
|
88
|
+
|
|
89
|
+
## How it works
|
|
90
|
+
|
|
91
|
+
`docmd` wraps [Marker](https://github.com/datalab-to/marker) with sane defaults and a
|
|
92
|
+
clean output format, then runs its own post-processing pass
|
|
93
|
+
(`docmd/postprocess/`) to fix table structure and normalize heading levels — see
|
|
94
|
+
[ARCHITECTURE.md](ARCHITECTURE.md) for why this is the actual differentiation, not
|
|
95
|
+
just a thin wrapper.
|
|
96
|
+
|
|
97
|
+
## License
|
|
98
|
+
|
|
99
|
+
The `docmd` wrapper code is MIT — see [LICENSE](LICENSE).
|
|
100
|
+
|
|
101
|
+
`docmd` depends on [Marker](https://github.com/datalab-to/marker), whose *code* is
|
|
102
|
+
Apache-2.0 and whose *model weights* are licensed under a modified Open RAIL-M license:
|
|
103
|
+
free for research, personal use, and organizations under $5M in funding or revenue.
|
|
104
|
+
Commercial use beyond that threshold requires a license from
|
|
105
|
+
[Datalab](https://www.datalab.to/pricing). This applies to you if you deploy `docmd`
|
|
106
|
+
commercially at scale — check Marker's current license terms directly before doing so.
|
|
107
|
+
|
|
108
|
+
## Roadmap
|
|
109
|
+
|
|
110
|
+
- [x] PDF, DOCX, PPTX → Markdown
|
|
111
|
+
- [x] Table cleanup / heading normalization post-processing
|
|
112
|
+
- [ ] Hosted API
|
|
113
|
+
- [ ] OCR quality tuning for scanned documents
|
|
114
|
+
- [ ] Batch conversion endpoint
|
|
115
|
+
- [ ] HTML output option
|
|
116
|
+
|
|
117
|
+
## Contributing
|
|
118
|
+
|
|
119
|
+
Issues and PRs welcome. If you're hitting a conversion quality issue, please include a
|
|
120
|
+
sample file (or a minimal reproduction) — it makes fixes much faster.
|
|
121
|
+
|
|
122
|
+
---
|
|
123
|
+
|
|
124
|
+
*If docmd saves you the trouble of setting up your own PDF-parsing pipeline, consider starring the repo — it's how other people find it.*
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""docmd: convert PDF/DOCX/PPTX to clean, structure-preserving Markdown.
|
|
2
|
+
|
|
3
|
+
from docmd import convert
|
|
4
|
+
markdown = convert("report.pdf")
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
import tempfile
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from docmd.config import ConvertConfig
|
|
14
|
+
from docmd.converters.base import ConversionResult
|
|
15
|
+
from docmd.converters.registry import get_converter
|
|
16
|
+
from docmd.postprocess.heading_normalize import normalize_headings
|
|
17
|
+
from docmd.postprocess.image_handling import apply_image_handling
|
|
18
|
+
from docmd.postprocess.table_cleanup import clean_tables
|
|
19
|
+
|
|
20
|
+
__all__ = ["convert", "convert_document", "ConvertConfig", "ConversionResult"]
|
|
21
|
+
__version__ = "0.1.0"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def convert_document(
|
|
25
|
+
source: str | Path | bytes,
|
|
26
|
+
*,
|
|
27
|
+
filename: str | None = None,
|
|
28
|
+
config: ConvertConfig | None = None,
|
|
29
|
+
) -> ConversionResult:
|
|
30
|
+
"""Convert `source` and return the full result (markdown + page count +
|
|
31
|
+
metadata), after docmd's post-processing pass has run.
|
|
32
|
+
|
|
33
|
+
`source` is either a path to a file, or raw bytes - in which case
|
|
34
|
+
`filename` is required so docmd knows the format (its extension is used
|
|
35
|
+
to pick a converter; the content itself is what gets converted).
|
|
36
|
+
"""
|
|
37
|
+
config = config or ConvertConfig()
|
|
38
|
+
|
|
39
|
+
if isinstance(source, (bytes, bytearray)):
|
|
40
|
+
if not filename:
|
|
41
|
+
raise ValueError("filename is required when passing bytes")
|
|
42
|
+
suffix = Path(filename).suffix
|
|
43
|
+
fd, tmp_path = tempfile.mkstemp(suffix=suffix)
|
|
44
|
+
try:
|
|
45
|
+
with os.fdopen(fd, "wb") as tmp:
|
|
46
|
+
tmp.write(source)
|
|
47
|
+
result = _convert_path(tmp_path, config)
|
|
48
|
+
finally:
|
|
49
|
+
os.unlink(tmp_path)
|
|
50
|
+
else:
|
|
51
|
+
result = _convert_path(str(source), config)
|
|
52
|
+
|
|
53
|
+
return result
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def convert(
|
|
57
|
+
source: str | Path | bytes,
|
|
58
|
+
*,
|
|
59
|
+
filename: str | None = None,
|
|
60
|
+
config: ConvertConfig | None = None,
|
|
61
|
+
) -> str:
|
|
62
|
+
"""Convert `source` (a file path, or bytes + filename) and return the
|
|
63
|
+
resulting Markdown as a string."""
|
|
64
|
+
return convert_document(source, filename=filename, config=config).markdown
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _convert_path(filepath: str, config: ConvertConfig) -> ConversionResult:
|
|
68
|
+
converter = get_converter(filepath)
|
|
69
|
+
result = converter.convert(filepath, config)
|
|
70
|
+
|
|
71
|
+
markdown = result.markdown
|
|
72
|
+
if config.clean_tables:
|
|
73
|
+
markdown = clean_tables(markdown)
|
|
74
|
+
if config.normalize_headings:
|
|
75
|
+
markdown = normalize_headings(markdown)
|
|
76
|
+
markdown = apply_image_handling(markdown, result.images, config.image_mode)
|
|
77
|
+
|
|
78
|
+
return ConversionResult(
|
|
79
|
+
markdown=markdown,
|
|
80
|
+
page_count=result.page_count,
|
|
81
|
+
images=result.images,
|
|
82
|
+
metadata=result.metadata,
|
|
83
|
+
)
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""`docmd convert <file> [-o output.md]`"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import sys
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import click
|
|
9
|
+
|
|
10
|
+
from docmd import convert_document
|
|
11
|
+
from docmd.config import ConvertConfig
|
|
12
|
+
from docmd.errors import DocmdError
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@click.group()
|
|
16
|
+
@click.version_option(package_name="docmd")
|
|
17
|
+
def main() -> None:
|
|
18
|
+
"""docmd: convert PDF/DOCX/PPTX to clean, structure-preserving Markdown."""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@main.command()
|
|
22
|
+
@click.argument("file", type=click.Path(exists=True, dir_okay=False, path_type=Path))
|
|
23
|
+
@click.option(
|
|
24
|
+
"-o",
|
|
25
|
+
"--output",
|
|
26
|
+
"output",
|
|
27
|
+
type=click.Path(dir_okay=False, path_type=Path),
|
|
28
|
+
default=None,
|
|
29
|
+
help="Write Markdown to this file instead of stdout.",
|
|
30
|
+
)
|
|
31
|
+
@click.option(
|
|
32
|
+
"--force-ocr",
|
|
33
|
+
is_flag=True,
|
|
34
|
+
default=False,
|
|
35
|
+
help="Force OCR even on pages that already have a text layer.",
|
|
36
|
+
)
|
|
37
|
+
@click.option(
|
|
38
|
+
"--use-llm",
|
|
39
|
+
is_flag=True,
|
|
40
|
+
default=False,
|
|
41
|
+
help="Use an LLM pass for higher-fidelity table/form extraction (needs a provider API key set for Marker; has a real marginal cost).",
|
|
42
|
+
)
|
|
43
|
+
@click.option(
|
|
44
|
+
"--image-mode",
|
|
45
|
+
type=click.Choice(["placeholder", "alt-text", "skip"]),
|
|
46
|
+
default="placeholder",
|
|
47
|
+
show_default=True,
|
|
48
|
+
help="How to represent images in the output.",
|
|
49
|
+
)
|
|
50
|
+
def convert(file: Path, output: Path | None, force_ocr: bool, use_llm: bool, image_mode: str) -> None:
|
|
51
|
+
"""Convert FILE to Markdown."""
|
|
52
|
+
config = ConvertConfig(force_ocr=force_ocr, use_llm=use_llm, image_mode=image_mode)
|
|
53
|
+
|
|
54
|
+
try:
|
|
55
|
+
result = convert_document(str(file), config=config)
|
|
56
|
+
except DocmdError as exc:
|
|
57
|
+
click.echo(f"error: {exc}", err=True)
|
|
58
|
+
sys.exit(1)
|
|
59
|
+
|
|
60
|
+
if output is not None:
|
|
61
|
+
output.write_text(result.markdown, encoding="utf-8")
|
|
62
|
+
click.echo(f"wrote {output} ({result.page_count} page(s))", err=True)
|
|
63
|
+
else:
|
|
64
|
+
click.echo(result.markdown)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
if __name__ == "__main__":
|
|
68
|
+
main()
|