parsosaur 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- parsosaur-0.1.0/.gitignore +12 -0
- parsosaur-0.1.0/.python-version +1 -0
- parsosaur-0.1.0/LICENSE +21 -0
- parsosaur-0.1.0/PKG-INFO +151 -0
- parsosaur-0.1.0/README.md +117 -0
- parsosaur-0.1.0/pyproject.toml +65 -0
- parsosaur-0.1.0/scripts/run_parsosaur.py +9 -0
- parsosaur-0.1.0/src/parsosaur/__init__.py +4 -0
- parsosaur-0.1.0/src/parsosaur/__main__.py +4 -0
- parsosaur-0.1.0/src/parsosaur/assets/bbox-viewer.html +185 -0
- parsosaur-0.1.0/src/parsosaur/assets/document.css +77 -0
- parsosaur-0.1.0/src/parsosaur/assets/document.js +145 -0
- parsosaur-0.1.0/src/parsosaur/parsosaur.py +5510 -0
- parsosaur-0.1.0/src/parsosaur/runtime.py +54 -0
- parsosaur-0.1.0/tests/test_packaging.py +54 -0
- parsosaur-0.1.0/tests/test_parser.py +776 -0
- parsosaur-0.1.0/tests/test_standalone.py +19 -0
- parsosaur-0.1.0/uv.lock +4545 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.12
|
parsosaur-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Anil Madhavapeddy
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
parsosaur-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: parsosaur
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Convert academic PDFs to JSON, TEI XML, and HTML.
|
|
5
|
+
Author-email: Anil Madhavapeddy <anil@recoil.org>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Keywords: academic,nemotron,ocr,pdf,tei
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Environment :: Console
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Topic :: Text Processing :: Markup :: XML
|
|
15
|
+
Requires-Python: <3.14,>=3.10
|
|
16
|
+
Requires-Dist: accelerate==1.12.0
|
|
17
|
+
Requires-Dist: beautifulsoup4==4.13.3
|
|
18
|
+
Requires-Dist: einops==0.8.1
|
|
19
|
+
Requires-Dist: huggingface-hub<2,>=1.24
|
|
20
|
+
Requires-Dist: lxml<7,>=5.3
|
|
21
|
+
Requires-Dist: markdown<4,>=3.7
|
|
22
|
+
Requires-Dist: numpy==2.2.6
|
|
23
|
+
Requires-Dist: openai<3,>=1.40
|
|
24
|
+
Requires-Dist: packaging>=24.2
|
|
25
|
+
Requires-Dist: pillow<13,>=12.2
|
|
26
|
+
Requires-Dist: pypdfium2<6,>=4.30
|
|
27
|
+
Requires-Dist: safetensors==0.8.0
|
|
28
|
+
Requires-Dist: timm==1.0.22
|
|
29
|
+
Requires-Dist: torch<2.14,>=2.11.0
|
|
30
|
+
Requires-Dist: torchvision<0.29,>=0.26.0
|
|
31
|
+
Requires-Dist: transformers<5.20,>=5.14.1
|
|
32
|
+
Requires-Dist: vllm==0.26.0
|
|
33
|
+
Description-Content-Type: text/markdown
|
|
34
|
+
|
|
35
|
+
# parsosaur
|
|
36
|
+
|
|
37
|
+
Convert academic PDFs to JSON, TEI XML, and HTML.
|
|
38
|
+
|
|
39
|
+
## Synopsis
|
|
40
|
+
|
|
41
|
+
```text
|
|
42
|
+
parsosaur [OPTIONS] DIR ...
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
## Description
|
|
46
|
+
|
|
47
|
+
Parsosaur extracts text, tables, figures, and bibliographic metadata from
|
|
48
|
+
academic PDFs. Each input directory must contain exactly one PDF and at most
|
|
49
|
+
one GROBID TEI file with a `.tei` or `.tei.xml` suffix. The TEI file supplies
|
|
50
|
+
context for image interpretation and metadata for bibliographic merging.
|
|
51
|
+
|
|
52
|
+
New parses require Linux x86-64, Python 3.10–3.13, a CUDA GPU, and an
|
|
53
|
+
OpenAI-compatible vision-model endpoint. Nemotron performs layout extraction
|
|
54
|
+
locally. PDF crops and prompts are sent to the selected endpoint.
|
|
55
|
+
|
|
56
|
+
The NVIDIA runtime and model are downloaded from a fixed Hugging Face
|
|
57
|
+
revision on first use. Allow several gigabytes of disk space for dependencies
|
|
58
|
+
and model files.
|
|
59
|
+
|
|
60
|
+
Outputs are written to `DIR/out/`. Existing compatible results are reused;
|
|
61
|
+
complete cached results require neither GPU inference nor endpoint requests.
|
|
62
|
+
Use `--force` to regenerate results after changing inputs or options.
|
|
63
|
+
|
|
64
|
+
## Installation
|
|
65
|
+
|
|
66
|
+
From a source checkout, install and run Parsosaur with uv:
|
|
67
|
+
|
|
68
|
+
```sh
|
|
69
|
+
uv sync --locked
|
|
70
|
+
uv run parsosaur --help
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
After publication to PyPI, run it without a checkout:
|
|
74
|
+
|
|
75
|
+
```sh
|
|
76
|
+
uvx parsosaur --help
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Use `uv tool install parsosaur` to install a persistent `parsosaur` command.
|
|
80
|
+
Alternatively, install it with `pip install parsosaur`.
|
|
81
|
+
|
|
82
|
+
After publication, `uv run URL [OPTIONS] DIR ...` can run the launcher at a
|
|
83
|
+
raw URL for `scripts/run_parsosaur.py`. The launcher installs the library
|
|
84
|
+
version specified in that file.
|
|
85
|
+
|
|
86
|
+
## Options
|
|
87
|
+
|
|
88
|
+
- `-h`, `--help` displays help and exits.
|
|
89
|
+
- `--base-url URL` selects the vision-model endpoint. It defaults to
|
|
90
|
+
`LLM_BASE_URL`, or `https://openrouter.ai/api/v1` if that variable is unset.
|
|
91
|
+
A URL without a path has `/v1` appended.
|
|
92
|
+
- `--model MODEL` selects the endpoint's model. It is required for OpenRouter.
|
|
93
|
+
Other endpoints default to the first model returned by their model list.
|
|
94
|
+
- `--no-context` omits TEI prose from image interpretation requests.
|
|
95
|
+
- `--no-merge` skips merging bibliographic metadata from the TEI file.
|
|
96
|
+
- `--force` deletes each input directory's `out/` subdirectory and regenerates
|
|
97
|
+
all results.
|
|
98
|
+
|
|
99
|
+
## Environment
|
|
100
|
+
|
|
101
|
+
- `LLM_BASE_URL` sets the default vision-model endpoint.
|
|
102
|
+
- `LLM_API_KEY` supplies the endpoint's API key. `OPENROUTER_API_KEY` is used
|
|
103
|
+
if `LLM_API_KEY` is unset. OpenRouter requires a key.
|
|
104
|
+
- `HF_HOME` sets the Hugging Face cache directory.
|
|
105
|
+
- `HF_HUB_OFFLINE=1` restricts model and runtime loading to cached files.
|
|
106
|
+
It does not disable requests to the vision-model endpoint.
|
|
107
|
+
|
|
108
|
+
## Files
|
|
109
|
+
|
|
110
|
+
All generated files reside in `DIR/out/`.
|
|
111
|
+
|
|
112
|
+
- `document.json`, `metadata.json`, and `figures.json` contain the structured
|
|
113
|
+
extraction results.
|
|
114
|
+
- `document.tei.xml` contains the document in TEI XML format.
|
|
115
|
+
- `index.html` provides a reading view with source-page highlights.
|
|
116
|
+
- `viewer.html` displays detected regions on the PDF.
|
|
117
|
+
- `pages/`, `crops/`, and `raw/` contain rendered pages, extracted images, and
|
|
118
|
+
intermediate model outputs.
|
|
119
|
+
- `merge_report.json` records bibliographic merge decisions when merging is
|
|
120
|
+
enabled and a TEI file is supplied.
|
|
121
|
+
|
|
122
|
+
## Examples
|
|
123
|
+
|
|
124
|
+
Parse one paper using a local vision-model endpoint:
|
|
125
|
+
|
|
126
|
+
```sh
|
|
127
|
+
uvx parsosaur paper/ --base-url http://localhost:8000
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
Parse several papers using the same endpoint:
|
|
131
|
+
|
|
132
|
+
```sh
|
|
133
|
+
export LLM_BASE_URL=http://localhost:8000
|
|
134
|
+
uvx parsosaur papers/*/
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
## Exit status
|
|
138
|
+
|
|
139
|
+
Parsosaur exits with status zero on success and a nonzero status on error.
|
|
140
|
+
During batch processing, individual paper failures are reported and
|
|
141
|
+
processing continues with the remaining papers.
|
|
142
|
+
|
|
143
|
+
## License
|
|
144
|
+
|
|
145
|
+
Parsosaur is distributed under the MIT license. NVIDIA's runtime and model
|
|
146
|
+
are subject to their upstream license and notices.
|
|
147
|
+
|
|
148
|
+
Please note that this is experimental research software, and is likely
|
|
149
|
+
to be rapidly iterated on with no backwards compatibility guarantees.
|
|
150
|
+
You should also carefully check the outputs to ensure that they are
|
|
151
|
+
suitable for your usecases.
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
# parsosaur
|
|
2
|
+
|
|
3
|
+
Convert academic PDFs to JSON, TEI XML, and HTML.
|
|
4
|
+
|
|
5
|
+
## Synopsis
|
|
6
|
+
|
|
7
|
+
```text
|
|
8
|
+
parsosaur [OPTIONS] DIR ...
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Description
|
|
12
|
+
|
|
13
|
+
Parsosaur extracts text, tables, figures, and bibliographic metadata from
|
|
14
|
+
academic PDFs. Each input directory must contain exactly one PDF and at most
|
|
15
|
+
one GROBID TEI file with a `.tei` or `.tei.xml` suffix. The TEI file supplies
|
|
16
|
+
context for image interpretation and metadata for bibliographic merging.
|
|
17
|
+
|
|
18
|
+
New parses require Linux x86-64, Python 3.10–3.13, a CUDA GPU, and an
|
|
19
|
+
OpenAI-compatible vision-model endpoint. Nemotron performs layout extraction
|
|
20
|
+
locally. PDF crops and prompts are sent to the selected endpoint.
|
|
21
|
+
|
|
22
|
+
The NVIDIA runtime and model are downloaded from a fixed Hugging Face
|
|
23
|
+
revision on first use. Allow several gigabytes of disk space for dependencies
|
|
24
|
+
and model files.
|
|
25
|
+
|
|
26
|
+
Outputs are written to `DIR/out/`. Existing compatible results are reused;
|
|
27
|
+
complete cached results require neither GPU inference nor endpoint requests.
|
|
28
|
+
Use `--force` to regenerate results after changing inputs or options.
|
|
29
|
+
|
|
30
|
+
## Installation
|
|
31
|
+
|
|
32
|
+
From a source checkout, install and run Parsosaur with uv:
|
|
33
|
+
|
|
34
|
+
```sh
|
|
35
|
+
uv sync --locked
|
|
36
|
+
uv run parsosaur --help
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
After publication to PyPI, run it without a checkout:
|
|
40
|
+
|
|
41
|
+
```sh
|
|
42
|
+
uvx parsosaur --help
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
Use `uv tool install parsosaur` to install a persistent `parsosaur` command.
|
|
46
|
+
Alternatively, install it with `pip install parsosaur`.
|
|
47
|
+
|
|
48
|
+
After publication, `uv run URL [OPTIONS] DIR ...` can run the launcher at a
|
|
49
|
+
raw URL for `scripts/run_parsosaur.py`. The launcher installs the library
|
|
50
|
+
version specified in that file.
|
|
51
|
+
|
|
52
|
+
## Options
|
|
53
|
+
|
|
54
|
+
- `-h`, `--help` displays help and exits.
|
|
55
|
+
- `--base-url URL` selects the vision-model endpoint. It defaults to
|
|
56
|
+
`LLM_BASE_URL`, or `https://openrouter.ai/api/v1` if that variable is unset.
|
|
57
|
+
A URL without a path has `/v1` appended.
|
|
58
|
+
- `--model MODEL` selects the endpoint's model. It is required for OpenRouter.
|
|
59
|
+
Other endpoints default to the first model returned by their model list.
|
|
60
|
+
- `--no-context` omits TEI prose from image interpretation requests.
|
|
61
|
+
- `--no-merge` skips merging bibliographic metadata from the TEI file.
|
|
62
|
+
- `--force` deletes each input directory's `out/` subdirectory and regenerates
|
|
63
|
+
all results.
|
|
64
|
+
|
|
65
|
+
## Environment
|
|
66
|
+
|
|
67
|
+
- `LLM_BASE_URL` sets the default vision-model endpoint.
|
|
68
|
+
- `LLM_API_KEY` supplies the endpoint's API key. `OPENROUTER_API_KEY` is used
|
|
69
|
+
if `LLM_API_KEY` is unset. OpenRouter requires a key.
|
|
70
|
+
- `HF_HOME` sets the Hugging Face cache directory.
|
|
71
|
+
- `HF_HUB_OFFLINE=1` restricts model and runtime loading to cached files.
|
|
72
|
+
It does not disable requests to the vision-model endpoint.
|
|
73
|
+
|
|
74
|
+
## Files
|
|
75
|
+
|
|
76
|
+
All generated files reside in `DIR/out/`.
|
|
77
|
+
|
|
78
|
+
- `document.json`, `metadata.json`, and `figures.json` contain the structured
|
|
79
|
+
extraction results.
|
|
80
|
+
- `document.tei.xml` contains the document in TEI XML format.
|
|
81
|
+
- `index.html` provides a reading view with source-page highlights.
|
|
82
|
+
- `viewer.html` displays detected regions on the PDF.
|
|
83
|
+
- `pages/`, `crops/`, and `raw/` contain rendered pages, extracted images, and
|
|
84
|
+
intermediate model outputs.
|
|
85
|
+
- `merge_report.json` records bibliographic merge decisions when merging is
|
|
86
|
+
enabled and a TEI file is supplied.
|
|
87
|
+
|
|
88
|
+
## Examples
|
|
89
|
+
|
|
90
|
+
Parse one paper using a local vision-model endpoint:
|
|
91
|
+
|
|
92
|
+
```sh
|
|
93
|
+
uvx parsosaur paper/ --base-url http://localhost:8000
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Parse several papers using the same endpoint:
|
|
97
|
+
|
|
98
|
+
```sh
|
|
99
|
+
export LLM_BASE_URL=http://localhost:8000
|
|
100
|
+
uvx parsosaur papers/*/
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
## Exit status
|
|
104
|
+
|
|
105
|
+
Parsosaur exits with status zero on success and a nonzero status on error.
|
|
106
|
+
During batch processing, individual paper failures are reported and
|
|
107
|
+
processing continues with the remaining papers.
|
|
108
|
+
|
|
109
|
+
## License
|
|
110
|
+
|
|
111
|
+
Parsosaur is distributed under the MIT license. NVIDIA's runtime and model
|
|
112
|
+
are subject to their upstream license and notices.
|
|
113
|
+
|
|
114
|
+
Please note that this is experimental research software, and is likely
|
|
115
|
+
to be rapidly iterated on with no backwards compatibility guarantees.
|
|
116
|
+
You should also carefully check the outputs to ensure that they are
|
|
117
|
+
suitable for your usecases.
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.27,<2"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "parsosaur"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Convert academic PDFs to JSON, TEI XML, and HTML."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
requires-python = ">=3.10,<3.14"
|
|
13
|
+
authors = [{ name = "Anil Madhavapeddy", email = "anil@recoil.org" }]
|
|
14
|
+
keywords = ["pdf", "ocr", "tei", "academic", "nemotron"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Environment :: Console",
|
|
18
|
+
"Intended Audience :: Science/Research",
|
|
19
|
+
"Operating System :: POSIX :: Linux",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Topic :: Text Processing :: Markup :: XML",
|
|
22
|
+
]
|
|
23
|
+
dependencies = [
|
|
24
|
+
"vllm==0.26.0",
|
|
25
|
+
# Requirements of NVIDIA's pinned runtime, downloaded on first use.
|
|
26
|
+
"torch>=2.11.0,<2.14",
|
|
27
|
+
"torchvision>=0.26.0,<0.29",
|
|
28
|
+
"transformers>=5.14.1,<5.20",
|
|
29
|
+
"packaging>=24.2",
|
|
30
|
+
"accelerate==1.12.0",
|
|
31
|
+
"timm==1.0.22",
|
|
32
|
+
"einops==0.8.1",
|
|
33
|
+
"numpy==2.2.6",
|
|
34
|
+
"safetensors==0.8.0",
|
|
35
|
+
"huggingface-hub>=1.24,<2",
|
|
36
|
+
"pypdfium2>=4.30,<6",
|
|
37
|
+
"pillow>=12.2,<13",
|
|
38
|
+
"beautifulsoup4==4.13.3",
|
|
39
|
+
"markdown>=3.7,<4",
|
|
40
|
+
"openai>=1.40,<3",
|
|
41
|
+
"lxml>=5.3,<7",
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
[project.scripts]
|
|
45
|
+
parsosaur = "parsosaur:main"
|
|
46
|
+
|
|
47
|
+
[project.entry-points."vllm.general_plugins"]
|
|
48
|
+
parsosaur_nemotron = "parsosaur.runtime:register"
|
|
49
|
+
|
|
50
|
+
[dependency-groups]
|
|
51
|
+
dev = ["pytest>=8,<10", "twine>=6,<7"]
|
|
52
|
+
|
|
53
|
+
[tool.uv]
|
|
54
|
+
environments = ["sys_platform == 'linux' and platform_machine == 'x86_64'"]
|
|
55
|
+
|
|
56
|
+
[tool.hatch.build.targets.wheel]
|
|
57
|
+
core-metadata-version = "2.4"
|
|
58
|
+
packages = ["src/parsosaur"]
|
|
59
|
+
|
|
60
|
+
[tool.hatch.build.targets.sdist]
|
|
61
|
+
core-metadata-version = "2.4"
|
|
62
|
+
include = ["/src", "/tests", "/scripts", "/README.md", "/pyproject.toml", "/uv.lock", "/.python-version", "/LICENSE"]
|
|
63
|
+
|
|
64
|
+
[tool.pytest.ini_options]
|
|
65
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="utf-8">
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
6
|
+
<title>Nemotron Parse bbox viewer</title>
|
|
7
|
+
<script src="https://cdnjs.cloudflare.com/ajax/libs/pdf.js/3.11.174/pdf.min.js"></script>
|
|
8
|
+
<style>
|
|
9
|
+
:root { color-scheme: light dark; }
|
|
10
|
+
body { font-family: system-ui, sans-serif; margin: 0; display: flex; height: 100vh; }
|
|
11
|
+
#sidebar { width: 300px; min-width: 220px; padding: 1rem; overflow-y: auto;
|
|
12
|
+
border-right: 1px solid #8884; flex-shrink: 0; }
|
|
13
|
+
#main { flex: 1; overflow-y: auto; padding: 1rem; background: #6663; }
|
|
14
|
+
.page-wrap { position: relative; margin: 0 auto 1.5rem; box-shadow: 0 1px 6px #0005;
|
|
15
|
+
width: fit-content; }
|
|
16
|
+
.page-wrap canvas { display: block; }
|
|
17
|
+
.box { position: absolute; border: 2px solid; border-radius: 2px; cursor: pointer;
|
|
18
|
+
opacity: 0.85; }
|
|
19
|
+
.box:hover { opacity: 1; background: #ffffff30; }
|
|
20
|
+
.box .tag { position: absolute; top: -1.15em; left: -2px; font-size: 10px;
|
|
21
|
+
padding: 0 3px; color: #fff; border-radius: 2px 2px 0 0;
|
|
22
|
+
white-space: nowrap; display: none; }
|
|
23
|
+
.box:hover .tag { display: block; }
|
|
24
|
+
#legend label { display: flex; align-items: center; gap: 0.4em; font-size: 0.85rem;
|
|
25
|
+
margin: 0.15rem 0; cursor: pointer; }
|
|
26
|
+
#legend .swatch { width: 0.9em; height: 0.9em; border-radius: 2px; display: inline-block; }
|
|
27
|
+
#detail { font-size: 0.8rem; white-space: pre-wrap; border-top: 1px solid #8884;
|
|
28
|
+
margin-top: 1rem; padding-top: 0.5rem; max-height: 40vh; overflow-y: auto; }
|
|
29
|
+
input[type=file] { width: 100%; margin-bottom: 0.5rem; font-size: 0.8rem; }
|
|
30
|
+
h1 { font-size: 1rem; } h2 { font-size: 0.85rem; margin: 1rem 0 0.3rem; }
|
|
31
|
+
#status { font-size: 0.8rem; color: #888; }
|
|
32
|
+
</style>
|
|
33
|
+
</head>
|
|
34
|
+
<body>
|
|
35
|
+
<div id="sidebar">
|
|
36
|
+
<h1>Nemotron Parse bbox viewer</h1>
|
|
37
|
+
<div id="fileControls">
|
|
38
|
+
<h2>1. PDF file</h2>
|
|
39
|
+
<input type="file" id="pdfFile" accept=".pdf">
|
|
40
|
+
<h2>2. document.json</h2>
|
|
41
|
+
<input type="file" id="jsonFile" accept=".json">
|
|
42
|
+
</div>
|
|
43
|
+
<p id="status">Select the source PDF and the document.json produced by the
|
|
44
|
+
parse pipeline. Everything runs locally in your browser.</p>
|
|
45
|
+
<h2>Element classes</h2>
|
|
46
|
+
<div id="legend"></div>
|
|
47
|
+
<h2>Element detail</h2>
|
|
48
|
+
<div id="detail">Click a box to see its class, bbox and extracted text.</div>
|
|
49
|
+
</div>
|
|
50
|
+
<div id="main"></div>
|
|
51
|
+
|
|
52
|
+
<!-- BUNDLE_DATA -->
|
|
53
|
+
<script>
|
|
54
|
+
pdfjsLib.GlobalWorkerOptions.workerSrc =
|
|
55
|
+
"https://cdnjs.cloudflare.com/ajax/libs/pdf.js/3.11.174/pdf.worker.min.js";
|
|
56
|
+
|
|
57
|
+
// Must match the parse pipeline's render target (processor input frame).
|
|
58
|
+
const TARGET_W = 1664, TARGET_H = 2048;
|
|
59
|
+
const RENDER_SCALE = 1.3; // on-screen pdf.js zoom
|
|
60
|
+
|
|
61
|
+
const PALETTE = [
|
|
62
|
+
"#e6194b", "#3cb44b", "#4363d8", "#f58231", "#911eb4", "#42d4f4",
|
|
63
|
+
"#f032e6", "#9a6324", "#800000", "#469990", "#808000", "#000075",
|
|
64
|
+
];
|
|
65
|
+
|
|
66
|
+
let pdfData = null, parseData = null;
|
|
67
|
+
const classColors = new Map(), hiddenClasses = new Set();
|
|
68
|
+
|
|
69
|
+
document.getElementById("pdfFile").addEventListener("change", async (e) => {
|
|
70
|
+
pdfData = new Uint8Array(await e.target.files[0].arrayBuffer());
|
|
71
|
+
render();
|
|
72
|
+
});
|
|
73
|
+
document.getElementById("jsonFile").addEventListener("change", async (e) => {
|
|
74
|
+
parseData = JSON.parse(await e.target.files[0].text());
|
|
75
|
+
render();
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
// Bundle mode: pdf_to_tei.py injects a <script id="bundle-data"> block with
|
|
79
|
+
// the PDF (base64) and document.json inline, so the page works from file://
|
|
80
|
+
// with no server and no file pickers.
|
|
81
|
+
const bundleEl = document.getElementById("bundle-data");
|
|
82
|
+
if (bundleEl) {
|
|
83
|
+
const bundle = JSON.parse(bundleEl.textContent);
|
|
84
|
+
parseData = bundle.doc;
|
|
85
|
+
pdfData = Uint8Array.from(atob(bundle.pdf_b64), c => c.charCodeAt(0));
|
|
86
|
+
document.getElementById("fileControls").style.display = "none";
|
|
87
|
+
window.addEventListener("DOMContentLoaded", render);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function colorFor(cls) {
|
|
91
|
+
if (!classColors.has(cls))
|
|
92
|
+
classColors.set(cls, PALETTE[classColors.size % PALETTE.length]);
|
|
93
|
+
return classColors.get(cls);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function buildLegend() {
|
|
97
|
+
const legend = document.getElementById("legend");
|
|
98
|
+
legend.innerHTML = "";
|
|
99
|
+
for (const [cls, color] of classColors) {
|
|
100
|
+
const label = document.createElement("label");
|
|
101
|
+
const cb = Object.assign(document.createElement("input"),
|
|
102
|
+
{ type: "checkbox", checked: !hiddenClasses.has(cls) });
|
|
103
|
+
cb.addEventListener("change", () => {
|
|
104
|
+
cb.checked ? hiddenClasses.delete(cls) : hiddenClasses.add(cls);
|
|
105
|
+
document.querySelectorAll(`[data-cls="${cls}"]`)
|
|
106
|
+
.forEach(b => b.style.display = cb.checked ? "" : "none");
|
|
107
|
+
});
|
|
108
|
+
const swatch = document.createElement("span");
|
|
109
|
+
swatch.className = "swatch";
|
|
110
|
+
swatch.style.background = color;
|
|
111
|
+
label.append(cb, swatch, document.createTextNode(cls));
|
|
112
|
+
legend.appendChild(label);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function showDetail(el) {
|
|
117
|
+
const d = document.getElementById("detail");
|
|
118
|
+
let text = el.text || "";
|
|
119
|
+
if (el.figure_text) text += "\n\n[figure text]\n" + el.figure_text;
|
|
120
|
+
if (el.figure_data_tables)
|
|
121
|
+
text += "\n\n[chart data]\n" + el.figure_data_tables
|
|
122
|
+
.map(t => t.map(r => r.join(" | ")).join("\n")).join("\n---\n");
|
|
123
|
+
d.textContent =
|
|
124
|
+
`page ${el.page} · class: ${el.class}` +
|
|
125
|
+
(el.reparsed_from_region ? " · (region re-parse)" : "") +
|
|
126
|
+
(el.truncated ? " · TRUNCATED" : "") +
|
|
127
|
+
`\nbbox_px: ${el.bbox_pixels ? el.bbox_pixels.join(", ") : "none"}` +
|
|
128
|
+
`\n\n${text.slice(0, 4000)}`;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
async function render() {
|
|
132
|
+
if (!pdfData || !parseData) return;
|
|
133
|
+
const main = document.getElementById("main");
|
|
134
|
+
main.innerHTML = "";
|
|
135
|
+
document.getElementById("status").textContent = "Rendering…";
|
|
136
|
+
|
|
137
|
+
const pdf = await pdfjsLib.getDocument({ data: pdfData }).promise;
|
|
138
|
+
for (let p = 1; p <= pdf.numPages; p++) {
|
|
139
|
+
const page = await pdf.getPage(p);
|
|
140
|
+
const vp1 = page.getViewport({ scale: 1 }); // PDF points
|
|
141
|
+
const vp = page.getViewport({ scale: RENDER_SCALE });
|
|
142
|
+
|
|
143
|
+
const wrap = document.createElement("div");
|
|
144
|
+
wrap.className = "page-wrap";
|
|
145
|
+
const canvas = document.createElement("canvas");
|
|
146
|
+
canvas.width = vp.width; canvas.height = vp.height;
|
|
147
|
+
wrap.appendChild(canvas);
|
|
148
|
+
main.appendChild(wrap);
|
|
149
|
+
await page.render({ canvasContext: canvas.getContext("2d"), viewport: vp }).promise;
|
|
150
|
+
|
|
151
|
+
// The pipeline rendered this page at scale = min(1664/w_pt, 2048/h_pt);
|
|
152
|
+
// bbox_pixels live in that image space. Map: px -> points -> screen.
|
|
153
|
+
const parseScale = Math.min(TARGET_W / vp1.width, TARGET_H / vp1.height);
|
|
154
|
+
const toScreen = v => v / parseScale * RENDER_SCALE;
|
|
155
|
+
|
|
156
|
+
const elements = parseData.pages[p - 1] || [];
|
|
157
|
+
for (const el of elements) {
|
|
158
|
+
if (!el.bbox_pixels) continue;
|
|
159
|
+
const [x1, y1, x2, y2] = el.bbox_pixels;
|
|
160
|
+
const box = document.createElement("div");
|
|
161
|
+
box.className = "box";
|
|
162
|
+
box.dataset.cls = el.class;
|
|
163
|
+
const color = colorFor(el.class);
|
|
164
|
+
box.style.borderColor = color;
|
|
165
|
+
box.style.left = toScreen(x1) + "px";
|
|
166
|
+
box.style.top = toScreen(y1) + "px";
|
|
167
|
+
box.style.width = Math.max(2, toScreen(x2 - x1)) + "px";
|
|
168
|
+
box.style.height = Math.max(2, toScreen(y2 - y1)) + "px";
|
|
169
|
+
if (hiddenClasses.has(el.class)) box.style.display = "none";
|
|
170
|
+
const tag = document.createElement("span");
|
|
171
|
+
tag.className = "tag";
|
|
172
|
+
tag.style.background = color;
|
|
173
|
+
tag.textContent = `${el.class} #${el.order}`;
|
|
174
|
+
box.appendChild(tag);
|
|
175
|
+
box.addEventListener("click", () => showDetail(el));
|
|
176
|
+
wrap.appendChild(box);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
buildLegend();
|
|
180
|
+
document.getElementById("status").textContent =
|
|
181
|
+
`${pdf.numPages} pages · ${parseData.pages.flat().length} elements`;
|
|
182
|
+
}
|
|
183
|
+
</script>
|
|
184
|
+
</body>
|
|
185
|
+
</html>
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
|
|
2
|
+
* { box-sizing: border-box; }
|
|
3
|
+
body { font-family: system-ui, sans-serif; margin: 0; height: 100vh;
|
|
4
|
+
display: flex; flex-direction: column; color: #1a1a1a; background: #fff; }
|
|
5
|
+
#topbar { display: flex; align-items: baseline; gap: 1rem; padding: 0.5rem 1rem;
|
|
6
|
+
border-bottom: 1px solid #ddd; background: #fafafa; flex-shrink: 0;
|
|
7
|
+
flex-wrap: wrap; }
|
|
8
|
+
#topbar b { font-size: 0.95rem; }
|
|
9
|
+
#topbar nav { margin-left: auto; display: flex; gap: 0.8rem; font-size: 0.8rem; }
|
|
10
|
+
#topbar nav a { color: #4363d8; text-decoration: none; }
|
|
11
|
+
#panes { display: flex; flex: 1; min-height: 0; }
|
|
12
|
+
#pdfpane { width: 52%; overflow-y: auto; background: #666; padding: 0.8rem;
|
|
13
|
+
resize: horizontal; scroll-behavior: smooth; }
|
|
14
|
+
#docpane { flex: 1; overflow-y: auto; padding: 1.5rem 2rem;
|
|
15
|
+
scroll-behavior: smooth; }
|
|
16
|
+
.page-wrap { position: relative; margin: 0 auto 1rem; width: fit-content;
|
|
17
|
+
box-shadow: 0 1px 6px #0006; }
|
|
18
|
+
.page-wrap canvas { display: block; }
|
|
19
|
+
.zone { position: absolute; border: 1.5px solid transparent; border-radius: 2px;
|
|
20
|
+
cursor: pointer; }
|
|
21
|
+
.zone:hover, .zone.active { border-color: #e6194b; background: #e6194b18; }
|
|
22
|
+
.zone.flash { border-color: #e6194b; background: #e6194b30; }
|
|
23
|
+
#docpane { font-family: Georgia, 'Times New Roman', serif; line-height: 1.55; }
|
|
24
|
+
#docpane > * { max-width: 46rem; margin-left: auto; margin-right: auto; }
|
|
25
|
+
.meta-card { background: #f6f7fb; border: 1px solid #dfe3f0; border-radius: 10px;
|
|
26
|
+
padding: 1.2rem 1.4rem; margin-bottom: 2rem;
|
|
27
|
+
font-family: system-ui, sans-serif; }
|
|
28
|
+
.meta-card h1 { font-size: 1.25rem; margin: 0 0 0.6rem; line-height: 1.35; }
|
|
29
|
+
.authors { font-size: 0.95rem; margin-bottom: 0.3rem; }
|
|
30
|
+
.authors sup { color: #4363d8; }
|
|
31
|
+
.affils { font-size: 0.78rem; color: #555; margin: 0.2rem 0 0.6rem; }
|
|
32
|
+
.publine { font-size: 0.85rem; color: #333; margin-bottom: 0.4rem; }
|
|
33
|
+
.publine a { color: #4363d8; }
|
|
34
|
+
.dates { font-size: 0.78rem; color: #777; }
|
|
35
|
+
.kw { display: inline-block; background: #e4e9fb; color: #2a3f9d;
|
|
36
|
+
border-radius: 99px; padding: 0.1rem 0.6rem; font-size: 0.75rem;
|
|
37
|
+
margin: 0.15rem 0.2rem 0 0; }
|
|
38
|
+
.abstract { font-size: 0.9rem; margin-top: 0.8rem; }
|
|
39
|
+
.abstract b { font-family: system-ui, sans-serif; }
|
|
40
|
+
.funding { font-size: 0.78rem; color: #555; margin-top: 0.6rem; }
|
|
41
|
+
h2.sec { font-size: 1.15rem; margin: 2rem auto 0.6rem; }
|
|
42
|
+
h3.sec { font-size: 1rem; margin: 1.5rem auto 0.5rem; }
|
|
43
|
+
h4.sec { font-size: 0.95rem; font-style: italic; margin: 1.2rem auto 0.4rem; }
|
|
44
|
+
[data-zone] { border-left: 3px solid transparent; padding-left: 0.5rem;
|
|
45
|
+
margin-left: -0.5rem; transition: border-color 0.15s; }
|
|
46
|
+
[data-zone]:hover { border-left-color: #e6194b66; }
|
|
47
|
+
[data-zone].flash { border-left-color: #e6194b; background: #e6194b0d; }
|
|
48
|
+
.meta-card span[data-zone] { border-left: none; padding-left: 0; margin-left: 0;
|
|
49
|
+
border-bottom: 1px dashed transparent; cursor: default; }
|
|
50
|
+
.meta-card span[data-zone]:hover { border-bottom-color: #e6194b;
|
|
51
|
+
background: #e6194b0d; }
|
|
52
|
+
a.cite { color: #2a3f9d; text-decoration: none; border-bottom: 1px dotted #2a3f9d55; }
|
|
53
|
+
a.objref { color: #0e7c50; text-decoration: none; border-bottom: 1px dotted #0e7c5055; }
|
|
54
|
+
figure { margin: 1.5rem auto; text-align: center; }
|
|
55
|
+
figure img { max-width: 100%; border: 1px solid #ddd; border-radius: 4px; }
|
|
56
|
+
figcaption { font-size: 0.83rem; color: #444; text-align: left; margin-top: 0.5rem; }
|
|
57
|
+
.figdata { font-size: 0.78rem; text-align: left; background: #f7f7f7;
|
|
58
|
+
padding: 0.6rem; border-radius: 6px; margin-top: 0.5rem; }
|
|
59
|
+
.figdata table { border-collapse: collapse; font-family: system-ui, sans-serif;
|
|
60
|
+
margin-top: 0.4rem; }
|
|
61
|
+
.figdata td, .figdata th { border: 1px solid #bbb; padding: 0.15rem 0.45rem; }
|
|
62
|
+
.figdata .warn { color: #a33; }
|
|
63
|
+
details.figdata-wrap summary { cursor: pointer; font-size: 0.8rem; color: #555;
|
|
64
|
+
font-family: system-ui, sans-serif; }
|
|
65
|
+
.footnote { font-size: 0.82rem; color: #555; }
|
|
66
|
+
.formula { text-align: center; margin: 1rem auto; }
|
|
67
|
+
#refs { font-family: system-ui, sans-serif; font-size: 0.85rem; }
|
|
68
|
+
#refs li { margin-bottom: 0.55rem; line-height: 1.4; }
|
|
69
|
+
#refs li.flash { background: #ffe9a820; border-left: 3px solid #e0a800;
|
|
70
|
+
padding-left: 0.4rem; }
|
|
71
|
+
#refs .ry { color: #666; } #refs .rj { font-style: italic; }
|
|
72
|
+
#refs a { color: #4363d8; }
|
|
73
|
+
#refs details { display: inline; } #refs summary { display: inline;
|
|
74
|
+
cursor: pointer; color: #999; font-size: 0.75rem; }
|
|
75
|
+
#tooltip { position: fixed; max-width: 26rem; background: #222; color: #eee;
|
|
76
|
+
font: 0.75rem/1.4 system-ui, sans-serif; padding: 0.5rem 0.7rem;
|
|
77
|
+
border-radius: 6px; pointer-events: none; display: none; z-index: 10; }
|