pagemark 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pagemark-0.1.0/.gitignore +228 -0
- pagemark-0.1.0/CHANGELOG.md +47 -0
- pagemark-0.1.0/LICENSE +21 -0
- pagemark-0.1.0/PKG-INFO +226 -0
- pagemark-0.1.0/README.md +185 -0
- pagemark-0.1.0/pyproject.toml +95 -0
- pagemark-0.1.0/src/pagemark/__init__.py +22 -0
- pagemark-0.1.0/src/pagemark/anchor.py +80 -0
- pagemark-0.1.0/src/pagemark/assemble.py +171 -0
- pagemark-0.1.0/src/pagemark/assets.py +76 -0
- pagemark-0.1.0/src/pagemark/backends.py +41 -0
- pagemark-0.1.0/src/pagemark/cli.py +157 -0
- pagemark-0.1.0/src/pagemark/client.py +92 -0
- pagemark-0.1.0/src/pagemark/guardrails.py +198 -0
- pagemark-0.1.0/src/pagemark/models.py +84 -0
- pagemark-0.1.0/src/pagemark/pipeline.py +313 -0
- pagemark-0.1.0/src/pagemark/profiles/__init__.py +41 -0
- pagemark-0.1.0/src/pagemark/profiles/generic.py +65 -0
- pagemark-0.1.0/src/pagemark/py.typed +0 -0
- pagemark-0.1.0/src/pagemark/render.py +88 -0
- pagemark-0.1.0/tests/__init__.py +0 -0
- pagemark-0.1.0/tests/conftest.py +90 -0
- pagemark-0.1.0/tests/test_anchor.py +33 -0
- pagemark-0.1.0/tests/test_assemble.py +102 -0
- pagemark-0.1.0/tests/test_backends.py +24 -0
- pagemark-0.1.0/tests/test_guardrails.py +100 -0
- pagemark-0.1.0/tests/test_models.py +87 -0
- pagemark-0.1.0/tests/test_pipeline.py +117 -0
- pagemark-0.1.0/tests/test_render.py +41 -0
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[codz]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# C extensions
|
|
7
|
+
*.so
|
|
8
|
+
|
|
9
|
+
# Distribution / packaging
|
|
10
|
+
.Python
|
|
11
|
+
build/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
dist/
|
|
14
|
+
downloads/
|
|
15
|
+
eggs/
|
|
16
|
+
.eggs/
|
|
17
|
+
lib/
|
|
18
|
+
lib64/
|
|
19
|
+
parts/
|
|
20
|
+
sdist/
|
|
21
|
+
var/
|
|
22
|
+
wheels/
|
|
23
|
+
share/python-wheels/
|
|
24
|
+
*.egg-info/
|
|
25
|
+
.installed.cfg
|
|
26
|
+
*.egg
|
|
27
|
+
MANIFEST
|
|
28
|
+
|
|
29
|
+
# PyInstaller
|
|
30
|
+
# Usually these files are written by a python script from a template
|
|
31
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
32
|
+
*.manifest
|
|
33
|
+
*.spec
|
|
34
|
+
|
|
35
|
+
# Installer logs
|
|
36
|
+
pip-log.txt
|
|
37
|
+
pip-delete-this-directory.txt
|
|
38
|
+
|
|
39
|
+
# Unit test / coverage reports
|
|
40
|
+
htmlcov/
|
|
41
|
+
.tox/
|
|
42
|
+
.nox/
|
|
43
|
+
.coverage
|
|
44
|
+
.coverage.*
|
|
45
|
+
.cache
|
|
46
|
+
nosetests.xml
|
|
47
|
+
coverage.xml
|
|
48
|
+
*.cover
|
|
49
|
+
*.py.cover
|
|
50
|
+
*.lcov
|
|
51
|
+
.hypothesis/
|
|
52
|
+
.pytest_cache/
|
|
53
|
+
cover/
|
|
54
|
+
|
|
55
|
+
# Translations
|
|
56
|
+
*.mo
|
|
57
|
+
*.pot
|
|
58
|
+
|
|
59
|
+
# Django stuff:
|
|
60
|
+
*.log
|
|
61
|
+
local_settings.py
|
|
62
|
+
db.sqlite3
|
|
63
|
+
db.sqlite3-journal
|
|
64
|
+
|
|
65
|
+
# Flask stuff:
|
|
66
|
+
instance/
|
|
67
|
+
.webassets-cache
|
|
68
|
+
|
|
69
|
+
# Scrapy stuff:
|
|
70
|
+
.scrapy
|
|
71
|
+
|
|
72
|
+
# Sphinx documentation
|
|
73
|
+
docs/_build/
|
|
74
|
+
|
|
75
|
+
# PyBuilder
|
|
76
|
+
.pybuilder/
|
|
77
|
+
target/
|
|
78
|
+
|
|
79
|
+
# Jupyter Notebook
|
|
80
|
+
.ipynb_checkpoints
|
|
81
|
+
|
|
82
|
+
# IPython
|
|
83
|
+
profile_default/
|
|
84
|
+
ipython_config.py
|
|
85
|
+
|
|
86
|
+
# pyenv
|
|
87
|
+
# For a library or package, you might want to ignore these files since the code is
|
|
88
|
+
# intended to run in multiple environments; otherwise, check them in:
|
|
89
|
+
# .python-version
|
|
90
|
+
|
|
91
|
+
# pipenv
|
|
92
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
93
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
94
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
95
|
+
# install all needed dependencies.
|
|
96
|
+
# Pipfile.lock
|
|
97
|
+
|
|
98
|
+
# UV
|
|
99
|
+
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
|
100
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
101
|
+
# commonly ignored for libraries.
|
|
102
|
+
# uv.lock
|
|
103
|
+
|
|
104
|
+
# poetry
|
|
105
|
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
|
106
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
107
|
+
# commonly ignored for libraries.
|
|
108
|
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
|
109
|
+
# poetry.lock
|
|
110
|
+
# poetry.toml
|
|
111
|
+
|
|
112
|
+
# pdm
|
|
113
|
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
|
114
|
+
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
|
|
115
|
+
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
|
|
116
|
+
# pdm.lock
|
|
117
|
+
# pdm.toml
|
|
118
|
+
.pdm-python
|
|
119
|
+
.pdm-build/
|
|
120
|
+
|
|
121
|
+
# pixi
|
|
122
|
+
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
|
|
123
|
+
# pixi.lock
|
|
124
|
+
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
|
|
125
|
+
# in the .venv directory. It is recommended not to include this directory in version control.
|
|
126
|
+
.pixi/*
|
|
127
|
+
!.pixi/config.toml
|
|
128
|
+
|
|
129
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
|
130
|
+
__pypackages__/
|
|
131
|
+
|
|
132
|
+
# Celery stuff
|
|
133
|
+
celerybeat-schedule*
|
|
134
|
+
celerybeat.pid
|
|
135
|
+
|
|
136
|
+
# Redis
|
|
137
|
+
*.rdb
|
|
138
|
+
*.aof
|
|
139
|
+
*.pid
|
|
140
|
+
|
|
141
|
+
# RabbitMQ
|
|
142
|
+
mnesia/
|
|
143
|
+
rabbitmq/
|
|
144
|
+
rabbitmq-data/
|
|
145
|
+
|
|
146
|
+
# ActiveMQ
|
|
147
|
+
activemq-data/
|
|
148
|
+
|
|
149
|
+
# SageMath parsed files
|
|
150
|
+
*.sage.py
|
|
151
|
+
|
|
152
|
+
# Environments
|
|
153
|
+
.env
|
|
154
|
+
.envrc
|
|
155
|
+
.venv
|
|
156
|
+
env/
|
|
157
|
+
venv/
|
|
158
|
+
ENV/
|
|
159
|
+
env.bak/
|
|
160
|
+
venv.bak/
|
|
161
|
+
|
|
162
|
+
# Spyder project settings
|
|
163
|
+
.spyderproject
|
|
164
|
+
.spyproject
|
|
165
|
+
|
|
166
|
+
# Rope project settings
|
|
167
|
+
.ropeproject
|
|
168
|
+
|
|
169
|
+
# mkdocs documentation
|
|
170
|
+
/site
|
|
171
|
+
|
|
172
|
+
# mypy
|
|
173
|
+
.mypy_cache/
|
|
174
|
+
.dmypy.json
|
|
175
|
+
dmypy.json
|
|
176
|
+
|
|
177
|
+
# Pyre type checker
|
|
178
|
+
.pyre/
|
|
179
|
+
|
|
180
|
+
# pytype static type analyzer
|
|
181
|
+
.pytype/
|
|
182
|
+
|
|
183
|
+
# Cython debug symbols
|
|
184
|
+
cython_debug/
|
|
185
|
+
|
|
186
|
+
# PyCharm
|
|
187
|
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
|
188
|
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
|
189
|
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
|
190
|
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
|
191
|
+
.idea/
|
|
192
|
+
|
|
193
|
+
# Abstra
|
|
194
|
+
# Abstra is an AI-powered process automation framework.
|
|
195
|
+
# Ignore directories containing user credentials, local state, and settings.
|
|
196
|
+
# Learn more at https://abstra.io/docs
|
|
197
|
+
.abstra/
|
|
198
|
+
|
|
199
|
+
# Visual Studio Code
|
|
200
|
+
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
|
|
201
|
+
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
|
202
|
+
# and can be added to the global gitignore or merged into this file. However, if you prefer,
|
|
203
|
+
# you could uncomment the following to ignore the entire vscode folder
|
|
204
|
+
# .vscode/
|
|
205
|
+
# Temporary file for partial code execution
|
|
206
|
+
tempCodeRunnerFile.py
|
|
207
|
+
|
|
208
|
+
# Ruff stuff:
|
|
209
|
+
.ruff_cache/
|
|
210
|
+
|
|
211
|
+
# PyPI configuration file
|
|
212
|
+
.pypirc
|
|
213
|
+
|
|
214
|
+
# Marimo
|
|
215
|
+
marimo/_static/
|
|
216
|
+
marimo/_lsp/
|
|
217
|
+
__marimo__/
|
|
218
|
+
|
|
219
|
+
# Streamlit
|
|
220
|
+
.streamlit/secrets.toml
|
|
221
|
+
# Generated test fixtures (rebuilt on demand by tests/conftest.py)
|
|
222
|
+
tests/fixtures/*.pdf
|
|
223
|
+
|
|
224
|
+
# Local tooling and scratch files
|
|
225
|
+
.claude/
|
|
226
|
+
.mcp.json
|
|
227
|
+
plan.md
|
|
228
|
+
test.pdf
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
## [0.1.0] - 2026-10-04
|
|
11
|
+
|
|
12
|
+
Initial release.
|
|
13
|
+
|
|
14
|
+
### Added
|
|
15
|
+
|
|
16
|
+
- `Pagemark` client with `convert()` and `aconvert()`. The async version is the
|
|
17
|
+
implementation; the sync one wraps it in `asyncio.run()`. Pages run
|
|
18
|
+
concurrently behind a semaphore (default 4).
|
|
19
|
+
- Works against any OpenAI-compatible endpoint. No provider constants, base
|
|
20
|
+
URLs, or model defaults are baked in — the caller supplies `base_url`,
|
|
21
|
+
`model`, and `api_key`. Key resolution order: explicit argument,
|
|
22
|
+
`OPENAI_API_KEY`, then an error naming the `base_url`.
|
|
23
|
+
- Page rendering with pypdfium2 under a pixel budget, so `Page.dpi` reports the
|
|
24
|
+
effective DPI after downscaling rather than the requested one.
|
|
25
|
+
- Text-layer anchoring: the PDF text layer and image-object placeholders are
|
|
26
|
+
sent alongside the page image, middle-elided to roughly 6k characters.
|
|
27
|
+
- Five guardrails on every model response — repetition, truncation, coverage,
|
|
28
|
+
hallucination, and refusal.
|
|
29
|
+
- A four-step escalation ladder (normal, reseed, high DPI, fallback prompt) with
|
|
30
|
+
a per-step confidence ceiling, ending in a text-layer fallback. A page that
|
|
31
|
+
fails every step is recorded with warnings instead of aborting the document.
|
|
32
|
+
- Cross-page assembly: running headers and footers are stripped, paragraphs
|
|
33
|
+
split across a page break are rejoined, and hyphenated line breaks are healed.
|
|
34
|
+
- `ModelProfile` protocol with a self-registering profile registry and one
|
|
35
|
+
built-in profile, `generic`.
|
|
36
|
+
- Typed output: `Document`, `Page`, `Block`, `Asset`, and `TokenUsage` pydantic
|
|
37
|
+
models. Every page carries `confidence`, `warnings`, and `source`.
|
|
38
|
+
- Embedded PDF images extracted per page. Binary payloads live in a private
|
|
39
|
+
attribute, so `model_dump_json()` stays free of them.
|
|
40
|
+
- `Document.save(md_path, assets_dir=...)` writes the Markdown and the images.
|
|
41
|
+
- Page selection via `pages`, accepting `"3"`, `"1-5"`, `"1-5,9,12-"`, or a
|
|
42
|
+
`list[int]`. Out-of-range indices raise rather than being clipped silently.
|
|
43
|
+
- Optional CLI (`pagemark[cli]`) with `convert` and `doctor` commands.
|
|
44
|
+
- `py.typed` marker; the package type-checks under mypy strict.
|
|
45
|
+
|
|
46
|
+
[Unreleased]: https://github.com/uts58/pagemark/compare/v0.1.0...HEAD
|
|
47
|
+
[0.1.0]: https://github.com/uts58/pagemark/releases/tag/v0.1.0
|
pagemark-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 pagemark contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
pagemark-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: pagemark
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Local-first PDF-to-Markdown converter using vision-language models
|
|
5
|
+
Project-URL: Homepage, https://github.com/uts58/pagemark
|
|
6
|
+
Project-URL: Repository, https://github.com/uts58/pagemark
|
|
7
|
+
Project-URL: Issues, https://github.com/uts58/pagemark/issues
|
|
8
|
+
Author: Utsha Saha
|
|
9
|
+
Maintainer: Glichin
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: document-conversion,markdown,ocr,pdf,vision-language-model,vlm
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Text Processing :: Markup :: Markdown
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Requires-Python: >=3.10
|
|
24
|
+
Requires-Dist: langchain-openai<2,>=1.0
|
|
25
|
+
Requires-Dist: markdown-it-py>=3
|
|
26
|
+
Requires-Dist: pillow>=10
|
|
27
|
+
Requires-Dist: pydantic>=2
|
|
28
|
+
Requires-Dist: pypdfium2>=4
|
|
29
|
+
Provides-Extra: all
|
|
30
|
+
Requires-Dist: datasets>=2; extra == 'all'
|
|
31
|
+
Requires-Dist: rapidfuzz>=3; extra == 'all'
|
|
32
|
+
Requires-Dist: rich>=13; extra == 'all'
|
|
33
|
+
Requires-Dist: typer>=0.9; extra == 'all'
|
|
34
|
+
Provides-Extra: cli
|
|
35
|
+
Requires-Dist: rich>=13; extra == 'cli'
|
|
36
|
+
Requires-Dist: typer>=0.9; extra == 'cli'
|
|
37
|
+
Provides-Extra: eval
|
|
38
|
+
Requires-Dist: datasets>=2; extra == 'eval'
|
|
39
|
+
Requires-Dist: rapidfuzz>=3; extra == 'eval'
|
|
40
|
+
Description-Content-Type: text/markdown
|
|
41
|
+
|
|
42
|
+
# pagemark
|
|
43
|
+
|
|
44
|
+
[](https://pypi.org/project/pagemark/)
|
|
45
|
+
[](https://pypi.org/project/pagemark/)
|
|
46
|
+
[](https://github.com/uts58/pagemark/blob/main/LICENSE)
|
|
47
|
+
|
|
48
|
+
pagemark converts PDFs to Markdown with vision-language models.
|
|
49
|
+
|
|
50
|
+
Use it with any OpenAI-compatible endpoint. You provide the `base_url`, `model`,
|
|
51
|
+
and `api_key`. pagemark renders each page, grounds the model with the PDF text
|
|
52
|
+
layer, and checks the result for common failures: repetition loops, truncation,
|
|
53
|
+
dropped columns, and hallucinated tables. If a check fails, it retries with a
|
|
54
|
+
stricter strategy and falls back to the text layer if the model still cannot
|
|
55
|
+
recover.
|
|
56
|
+
|
|
57
|
+
If one page fails, the document still finishes. pagemark records warnings for
|
|
58
|
+
that page instead of aborting the whole conversion.
|
|
59
|
+
|
|
60
|
+
## Prerequisites
|
|
61
|
+
|
|
62
|
+
- Python 3.10+
|
|
63
|
+
- An OpenAI-compatible endpoint serving a vision-language model
|
|
64
|
+
(Ollama, vLLM, LM Studio, any hosted provider, etc.)
|
|
65
|
+
|
|
66
|
+
## Install
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
pip install pagemark # library only
|
|
70
|
+
pip install "pagemark[cli]" # with CLI
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
## Quick start
|
|
74
|
+
|
|
75
|
+
```python
|
|
76
|
+
from pagemark import Pagemark
|
|
77
|
+
|
|
78
|
+
client = Pagemark(
|
|
79
|
+
base_url="http://localhost:11434/v1", # OpenAI-compatible API
|
|
80
|
+
api_key="your-key", # API key
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
doc = client.convert("paper.pdf", model="qwen3-vl:8b")
|
|
84
|
+
|
|
85
|
+
print(doc.markdown)
|
|
86
|
+
doc.save("out.md", assets_dir="out_assets/")
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
`api_key` is optional. pagemark resolves it from the explicit argument first,
|
|
90
|
+
then from `OPENAI_API_KEY`. If neither exists, it raises an error that names the
|
|
91
|
+
`base_url`.
|
|
92
|
+
|
|
93
|
+
### Async
|
|
94
|
+
|
|
95
|
+
`aconvert()` is the implementation behind `convert()`. `convert()` wraps it with
|
|
96
|
+
`asyncio.run()`. Pages run concurrently, bounded by a semaphore. The default
|
|
97
|
+
limit is 4.
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
from pagemark import Pagemark
|
|
101
|
+
|
|
102
|
+
client = Pagemark(
|
|
103
|
+
base_url="http://localhost:11434/v1", # OpenAI-compatible API
|
|
104
|
+
api_key="your-key", # API key
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
doc = await client.aconvert(
|
|
108
|
+
"paper.pdf",
|
|
109
|
+
model="qwen3-vl:8b",
|
|
110
|
+
concurrency=8,
|
|
111
|
+
)
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
### Page selection
|
|
115
|
+
|
|
116
|
+
Convert selected pages instead of the whole document:
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
doc = client.convert("paper.pdf", model="qwen3-vl:8b", pages="1-5,9,12-")
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
`pages` accepts `"3"`, `"1-5"`, `"1-5,9,12-"`, or a `list[int]`. Out-of-range
|
|
123
|
+
indices raise an error instead of being silently clipped.
|
|
124
|
+
|
|
125
|
+
### Bring your own model
|
|
126
|
+
|
|
127
|
+
If you need settings the constructor does not expose, pass a pre-configured
|
|
128
|
+
`BaseChatModel`. That covers custom timeouts, proxies, non-OpenAI LangChain
|
|
129
|
+
backends like `ChatAnthropic` or `ChatOllama`, callbacks, and similar options.
|
|
130
|
+
|
|
131
|
+
```python
|
|
132
|
+
from langchain_openai import ChatOpenAI
|
|
133
|
+
from pagemark import Pagemark
|
|
134
|
+
|
|
135
|
+
my_model = ChatOpenAI(
|
|
136
|
+
model="...",
|
|
137
|
+
base_url="...",
|
|
138
|
+
api_key="...",
|
|
139
|
+
timeout=120,
|
|
140
|
+
max_retries=5,
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
client = Pagemark(chat_model=my_model)
|
|
144
|
+
doc = client.convert("paper.pdf")
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
### CLI
|
|
148
|
+
|
|
149
|
+
```bash
|
|
150
|
+
pagemark convert paper.pdf -o paper.md \
|
|
151
|
+
--model qwen3-vl:8b \
|
|
152
|
+
--base-url http://localhost:11434/v1
|
|
153
|
+
|
|
154
|
+
pagemark convert paper.pdf -o paper.md --json paper.json \
|
|
155
|
+
--model qwen3-vl:8b \
|
|
156
|
+
--base-url http://localhost:11434/v1
|
|
157
|
+
|
|
158
|
+
pagemark doctor --base-url http://localhost:11434/v1 --model qwen3-vl:8b
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
`doctor` checks whether the endpoint is reachable, whether the model is present,
|
|
162
|
+
and where the API key came from.
|
|
163
|
+
|
|
164
|
+
## How it works
|
|
165
|
+
|
|
166
|
+
```
|
|
167
|
+
pdf -> render + anchor -> prompt (profile) -> ChatOpenAI -> guardrails -> parse -> assemble -> Document
|
|
168
|
+
pypdfium2 per-model langchain retry ladder md-it stitching md + json
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
1. Render each page to a PNG with pypdfium2, downscaled to the profile's pixel
|
|
172
|
+
budget.
|
|
173
|
+
2. Extract the PDF text layer as an anchor for the model. Long pages are
|
|
174
|
+
middle-elided to about 6k characters.
|
|
175
|
+
3. Send the image and anchor text to the VLM.
|
|
176
|
+
4. Check every response with five guardrails.
|
|
177
|
+
5. Retry with stricter strategies when a guardrail fails.
|
|
178
|
+
6. Assemble the pages into a `Document` with typed blocks, then stitch across
|
|
179
|
+
pages by stripping running headers and footers, joining split paragraphs, and
|
|
180
|
+
dehyphenating text.
|
|
181
|
+
|
|
182
|
+
### Guardrails
|
|
183
|
+
|
|
184
|
+
| Check | What it catches |
|
|
185
|
+
|---|---|
|
|
186
|
+
| Repetition | n-gram loops, collapsed gzip ratio |
|
|
187
|
+
| Truncation | `finish_reason == "length"` |
|
|
188
|
+
| Coverage | anchor tokens missing from output; catches silently dropped columns |
|
|
189
|
+
| Hallucination | high rate of output tokens absent from anchor (born-digital pages) |
|
|
190
|
+
| Refusal | "I'm sorry", "I cannot", near-empty output on a non-blank page |
|
|
191
|
+
|
|
192
|
+
### Escalation ladder
|
|
193
|
+
|
|
194
|
+
When any guardrail fails, pagemark retries with more aggressive strategies:
|
|
195
|
+
|
|
196
|
+
| Step | Strategy | Confidence ceiling |
|
|
197
|
+
|---|---|---|
|
|
198
|
+
| 1 | Normal render, standard prompt | 0.95 |
|
|
199
|
+
| 2 | Reseed: `temperature=0.3`, `seed=42` | 0.85 |
|
|
200
|
+
| 3 | High DPI: DPI x1.5 (capped at 400), pixels x1.5 | 0.75 |
|
|
201
|
+
| 4 | Fallback prompt: simpler instruction | 0.60 |
|
|
202
|
+
| 5 | Text layer fallback | 0.30 |
|
|
203
|
+
|
|
204
|
+
Each page includes `source`, `confidence`, and `warnings`, so downstream
|
|
205
|
+
consumers can filter by quality.
|
|
206
|
+
|
|
207
|
+
## Output
|
|
208
|
+
|
|
209
|
+
`convert()` returns a `Document` with:
|
|
210
|
+
|
|
211
|
+
- `markdown`: the full stitched Markdown output
|
|
212
|
+
- `pages`: list of `Page` objects, each with:
|
|
213
|
+
- `markdown`: per-page Markdown
|
|
214
|
+
- `blocks`: typed blocks (heading, paragraph, table, figure, code, list, formula, caption, footnote)
|
|
215
|
+
- `source`: `"vlm"` or `"text_layer"` (if the model couldn't handle the page)
|
|
216
|
+
- `confidence`: 0-1 score
|
|
217
|
+
- `warnings`: what went wrong, if anything
|
|
218
|
+
- `assets`: embedded images extracted from the PDF
|
|
219
|
+
- `model_dump_json()`: full structured output as JSON
|
|
220
|
+
- `save(path, assets_dir=...)`: writes the Markdown file and extracted images
|
|
221
|
+
|
|
222
|
+
## License
|
|
223
|
+
|
|
224
|
+
MIT. Runtime dependencies are permissively licensed - mostly MIT/Apache-2.0/BSD,
|
|
225
|
+
with a few under MPL-2.0 (certifi, orjson) and PSF-2.0 (typing-extensions). No
|
|
226
|
+
GPL/LGPL/AGPL dependencies in the tree.
|
pagemark-0.1.0/README.md
ADDED
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
# pagemark
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/pagemark/)
|
|
4
|
+
[](https://pypi.org/project/pagemark/)
|
|
5
|
+
[](https://github.com/uts58/pagemark/blob/main/LICENSE)
|
|
6
|
+
|
|
7
|
+
pagemark converts PDFs to Markdown with vision-language models.
|
|
8
|
+
|
|
9
|
+
Use it with any OpenAI-compatible endpoint. You provide the `base_url`, `model`,
|
|
10
|
+
and `api_key`. pagemark renders each page, grounds the model with the PDF text
|
|
11
|
+
layer, and checks the result for common failures: repetition loops, truncation,
|
|
12
|
+
dropped columns, and hallucinated tables. If a check fails, it retries with a
|
|
13
|
+
stricter strategy and falls back to the text layer if the model still cannot
|
|
14
|
+
recover.
|
|
15
|
+
|
|
16
|
+
If one page fails, the document still finishes. pagemark records warnings for
|
|
17
|
+
that page instead of aborting the whole conversion.
|
|
18
|
+
|
|
19
|
+
## Prerequisites
|
|
20
|
+
|
|
21
|
+
- Python 3.10+
|
|
22
|
+
- An OpenAI-compatible endpoint serving a vision-language model
|
|
23
|
+
(Ollama, vLLM, LM Studio, any hosted provider, etc.)
|
|
24
|
+
|
|
25
|
+
## Install
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
pip install pagemark # library only
|
|
29
|
+
pip install "pagemark[cli]" # with CLI
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
## Quick start
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
from pagemark import Pagemark
|
|
36
|
+
|
|
37
|
+
client = Pagemark(
|
|
38
|
+
base_url="http://localhost:11434/v1", # OpenAI-compatible API
|
|
39
|
+
api_key="your-key", # API key
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
doc = client.convert("paper.pdf", model="qwen3-vl:8b")
|
|
43
|
+
|
|
44
|
+
print(doc.markdown)
|
|
45
|
+
doc.save("out.md", assets_dir="out_assets/")
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
`api_key` is optional. pagemark resolves it from the explicit argument first,
|
|
49
|
+
then from `OPENAI_API_KEY`. If neither exists, it raises an error that names the
|
|
50
|
+
`base_url`.
|
|
51
|
+
|
|
52
|
+
### Async
|
|
53
|
+
|
|
54
|
+
`aconvert()` is the implementation behind `convert()`. `convert()` wraps it with
|
|
55
|
+
`asyncio.run()`. Pages run concurrently, bounded by a semaphore. The default
|
|
56
|
+
limit is 4.
|
|
57
|
+
|
|
58
|
+
```python
|
|
59
|
+
from pagemark import Pagemark
|
|
60
|
+
|
|
61
|
+
client = Pagemark(
|
|
62
|
+
base_url="http://localhost:11434/v1", # OpenAI-compatible API
|
|
63
|
+
api_key="your-key", # API key
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
doc = await client.aconvert(
|
|
67
|
+
"paper.pdf",
|
|
68
|
+
model="qwen3-vl:8b",
|
|
69
|
+
concurrency=8,
|
|
70
|
+
)
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
### Page selection
|
|
74
|
+
|
|
75
|
+
Convert selected pages instead of the whole document:
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
doc = client.convert("paper.pdf", model="qwen3-vl:8b", pages="1-5,9,12-")
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
`pages` accepts `"3"`, `"1-5"`, `"1-5,9,12-"`, or a `list[int]`. Out-of-range
|
|
82
|
+
indices raise an error instead of being silently clipped.
|
|
83
|
+
|
|
84
|
+
### Bring your own model
|
|
85
|
+
|
|
86
|
+
If you need settings the constructor does not expose, pass a pre-configured
|
|
87
|
+
`BaseChatModel`. That covers custom timeouts, proxies, non-OpenAI LangChain
|
|
88
|
+
backends like `ChatAnthropic` or `ChatOllama`, callbacks, and similar options.
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
from langchain_openai import ChatOpenAI
|
|
92
|
+
from pagemark import Pagemark
|
|
93
|
+
|
|
94
|
+
my_model = ChatOpenAI(
|
|
95
|
+
model="...",
|
|
96
|
+
base_url="...",
|
|
97
|
+
api_key="...",
|
|
98
|
+
timeout=120,
|
|
99
|
+
max_retries=5,
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
client = Pagemark(chat_model=my_model)
|
|
103
|
+
doc = client.convert("paper.pdf")
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
### CLI
|
|
107
|
+
|
|
108
|
+
```bash
|
|
109
|
+
pagemark convert paper.pdf -o paper.md \
|
|
110
|
+
--model qwen3-vl:8b \
|
|
111
|
+
--base-url http://localhost:11434/v1
|
|
112
|
+
|
|
113
|
+
pagemark convert paper.pdf -o paper.md --json paper.json \
|
|
114
|
+
--model qwen3-vl:8b \
|
|
115
|
+
--base-url http://localhost:11434/v1
|
|
116
|
+
|
|
117
|
+
pagemark doctor --base-url http://localhost:11434/v1 --model qwen3-vl:8b
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
`doctor` checks whether the endpoint is reachable, whether the model is present,
|
|
121
|
+
and where the API key came from.
|
|
122
|
+
|
|
123
|
+
## How it works
|
|
124
|
+
|
|
125
|
+
```
|
|
126
|
+
pdf -> render + anchor -> prompt (profile) -> ChatOpenAI -> guardrails -> parse -> assemble -> Document
|
|
127
|
+
pypdfium2 per-model langchain retry ladder md-it stitching md + json
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
1. Render each page to a PNG with pypdfium2, downscaled to the profile's pixel
|
|
131
|
+
budget.
|
|
132
|
+
2. Extract the PDF text layer as an anchor for the model. Long pages are
|
|
133
|
+
middle-elided to about 6k characters.
|
|
134
|
+
3. Send the image and anchor text to the VLM.
|
|
135
|
+
4. Check every response with five guardrails.
|
|
136
|
+
5. Retry with stricter strategies when a guardrail fails.
|
|
137
|
+
6. Assemble the pages into a `Document` with typed blocks, then stitch across
|
|
138
|
+
pages by stripping running headers and footers, joining split paragraphs, and
|
|
139
|
+
dehyphenating text.
|
|
140
|
+
|
|
141
|
+
### Guardrails
|
|
142
|
+
|
|
143
|
+
| Check | What it catches |
|
|
144
|
+
|---|---|
|
|
145
|
+
| Repetition | n-gram loops, collapsed gzip ratio |
|
|
146
|
+
| Truncation | `finish_reason == "length"` |
|
|
147
|
+
| Coverage | anchor tokens missing from output; catches silently dropped columns |
|
|
148
|
+
| Hallucination | high rate of output tokens absent from anchor (born-digital pages) |
|
|
149
|
+
| Refusal | "I'm sorry", "I cannot", near-empty output on a non-blank page |
|
|
150
|
+
|
|
151
|
+
### Escalation ladder
|
|
152
|
+
|
|
153
|
+
When any guardrail fails, pagemark retries with more aggressive strategies:
|
|
154
|
+
|
|
155
|
+
| Step | Strategy | Confidence ceiling |
|
|
156
|
+
|---|---|---|
|
|
157
|
+
| 1 | Normal render, standard prompt | 0.95 |
|
|
158
|
+
| 2 | Reseed: `temperature=0.3`, `seed=42` | 0.85 |
|
|
159
|
+
| 3 | High DPI: DPI x1.5 (capped at 400), pixels x1.5 | 0.75 |
|
|
160
|
+
| 4 | Fallback prompt: simpler instruction | 0.60 |
|
|
161
|
+
| 5 | Text layer fallback | 0.30 |
|
|
162
|
+
|
|
163
|
+
Each page includes `source`, `confidence`, and `warnings`, so downstream
|
|
164
|
+
consumers can filter by quality.
|
|
165
|
+
|
|
166
|
+
## Output
|
|
167
|
+
|
|
168
|
+
`convert()` returns a `Document` with:
|
|
169
|
+
|
|
170
|
+
- `markdown`: the full stitched Markdown output
|
|
171
|
+
- `pages`: list of `Page` objects, each with:
|
|
172
|
+
- `markdown`: per-page Markdown
|
|
173
|
+
- `blocks`: typed blocks (heading, paragraph, table, figure, code, list, formula, caption, footnote)
|
|
174
|
+
- `source`: `"vlm"` or `"text_layer"` (if the model couldn't handle the page)
|
|
175
|
+
- `confidence`: 0-1 score
|
|
176
|
+
- `warnings`: what went wrong, if anything
|
|
177
|
+
- `assets`: embedded images extracted from the PDF
|
|
178
|
+
- `model_dump_json()`: full structured output as JSON
|
|
179
|
+
- `save(path, assets_dir=...)`: writes the Markdown file and extracted images
|
|
180
|
+
|
|
181
|
+
## License
|
|
182
|
+
|
|
183
|
+
MIT. Runtime dependencies are permissively licensed - mostly MIT/Apache-2.0/BSD,
|
|
184
|
+
with a few under MPL-2.0 (certifi, orjson) and PSF-2.0 (typing-extensions). No
|
|
185
|
+
GPL/LGPL/AGPL dependencies in the tree.
|