marker-pdf 0.2.2__tar.gz → 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/PKG-INFO +4 -3
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/README.md +2 -2
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/chunk_convert.py +4 -1
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/convert.py +1 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/cleaners/code.py +3 -3
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/output.py +2 -1
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/pdf/utils.py +3 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/postprocessors/markdown.py +7 -5
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/settings.py +1 -1
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/pyproject.toml +2 -1
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/LICENSE +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/chunk_convert.sh +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/convert_single.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/benchmark/scoring.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/cleaners/bullets.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/cleaners/fontstyle.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/cleaners/headers.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/cleaners/headings.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/cleaners/text.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/convert.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/debug/data.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/equations/equations.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/equations/inference.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/images/extract.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/images/save.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/layout/layout.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/layout/order.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/logger.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/models.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/ocr/detection.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/ocr/heuristics.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/ocr/lang.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/ocr/recognition.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/ocr/tesseract.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/ocr/utils.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/pdf/extract_text.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/pdf/images.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/postprocessors/editor.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/postprocessors/t5.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/schema/bbox.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/schema/block.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/schema/merged.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/schema/page.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/tables/cells.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/tables/table.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/tables/utils.py +0 -0
- {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: marker-pdf
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.4
|
|
4
4
|
Summary: Convert PDF to markdown with high speed and accuracy.
|
|
5
5
|
Home-page: https://github.com/VikParuchuri/marker
|
|
6
6
|
License: GPL-3.0-or-later
|
|
@@ -16,6 +16,7 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
16
16
|
Requires-Dist: Pillow (>=10.1.0,<11.0.0)
|
|
17
17
|
Requires-Dist: filetype (>=1.2.0,<2.0.0)
|
|
18
18
|
Requires-Dist: ftfy (>=6.1.1,<7.0.0)
|
|
19
|
+
Requires-Dist: grpcio (>=1.63.0,<2.0.0)
|
|
19
20
|
Requires-Dist: numpy (>=1.26.1,<2.0.0)
|
|
20
21
|
Requires-Dist: pdftext (>=0.3.7,<0.4.0)
|
|
21
22
|
Requires-Dist: pydantic (>=2.4.2,<3.0.0)
|
|
@@ -71,7 +72,7 @@ It only uses models where necessary, which improves speed and accuracy.
|
|
|
71
72
|
|
|
72
73
|

|
|
73
74
|
|
|
74
|
-
The above results are with marker and nougat setup so they each take ~
|
|
75
|
+
The above results are with marker and nougat setup so they each take ~4GB of VRAM on an A6000.
|
|
75
76
|
|
|
76
77
|
See [below](#benchmarks) for detailed speed and accuracy benchmarks, and instructions on how to run your own benchmarks.
|
|
77
78
|
|
|
@@ -214,7 +215,7 @@ Peak GPU memory usage during the benchmark is `4.2GB` for nougat, and `4.1GB` fo
|
|
|
214
215
|
|
|
215
216
|
**Throughput**
|
|
216
217
|
|
|
217
|
-
Marker takes about
|
|
218
|
+
Marker takes about 4.5GB of VRAM on average per task, so you can convert 10 documents in parallel on an A6000.
|
|
218
219
|
|
|
219
220
|

|
|
220
221
|
|
|
@@ -35,7 +35,7 @@ It only uses models where necessary, which improves speed and accuracy.
|
|
|
35
35
|
|
|
36
36
|

|
|
37
37
|
|
|
38
|
-
The above results are with marker and nougat setup so they each take ~
|
|
38
|
+
The above results are with marker and nougat setup so they each take ~4GB of VRAM on an A6000.
|
|
39
39
|
|
|
40
40
|
See [below](#benchmarks) for detailed speed and accuracy benchmarks, and instructions on how to run your own benchmarks.
|
|
41
41
|
|
|
@@ -178,7 +178,7 @@ Peak GPU memory usage during the benchmark is `4.2GB` for nougat, and `4.1GB` fo
|
|
|
178
178
|
|
|
179
179
|
**Throughput**
|
|
180
180
|
|
|
181
|
-
Marker takes about
|
|
181
|
+
Marker takes about 4.5GB of VRAM on average per task, so you can convert 10 documents in parallel on an A6000.
|
|
182
182
|
|
|
183
183
|

|
|
184
184
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import argparse
|
|
2
2
|
import subprocess
|
|
3
|
+
import pkg_resources
|
|
3
4
|
|
|
4
5
|
|
|
5
6
|
def main():
|
|
@@ -8,8 +9,10 @@ def main():
|
|
|
8
9
|
parser.add_argument("out_folder", help="Output folder")
|
|
9
10
|
args = parser.parse_args()
|
|
10
11
|
|
|
12
|
+
script_path = pkg_resources.resource_filename(__name__, 'chunk_convert.sh')
|
|
13
|
+
|
|
11
14
|
# Construct the command
|
|
12
|
-
cmd = f"
|
|
15
|
+
cmd = f"{script_path} {args.in_folder} {args.out_folder}"
|
|
13
16
|
|
|
14
17
|
# Execute the shell script
|
|
15
18
|
subprocess.run(cmd, shell=True, check=True)
|
|
@@ -24,6 +24,7 @@ def process_single_pdf(filepath: str, out_folder: str, model_refs, metadata: Opt
|
|
|
24
24
|
fname = os.path.basename(filepath)
|
|
25
25
|
if markdown_exists(out_folder, fname):
|
|
26
26
|
return
|
|
27
|
+
|
|
27
28
|
try:
|
|
28
29
|
# Skip trying to convert files that don't have a lot of embedded text
|
|
29
30
|
# This can indicate that they were scanned, and not OCRed properly
|
|
@@ -67,13 +67,13 @@ def identify_code_blocks(pages: List[Page]):
|
|
|
67
67
|
is_code = [
|
|
68
68
|
len(block.lines) > 3,
|
|
69
69
|
is_code_linelen(block.lines),
|
|
70
|
-
sum(is_indent) + comment_lines > len(block.lines) * .
|
|
70
|
+
sum(is_indent) + comment_lines > len(block.lines) * .7, # Indentation and comments are a majority
|
|
71
71
|
]
|
|
72
72
|
|
|
73
73
|
if avg_font_size is not None:
|
|
74
74
|
font_checks = [
|
|
75
|
-
mean(line_font_sizes) <= avg_font_size, # Lower than average font size and line height
|
|
76
|
-
mean(block_line_heights) < avg_line_height
|
|
75
|
+
mean(line_font_sizes) <= avg_font_size * .8, # Lower than average font size and line height
|
|
76
|
+
mean(block_line_heights) < avg_line_height * .8
|
|
77
77
|
]
|
|
78
78
|
is_code += font_checks
|
|
79
79
|
|
|
@@ -5,7 +5,6 @@ import json
|
|
|
5
5
|
def get_subfolder_path(out_folder, fname):
|
|
6
6
|
subfolder_name = fname.split(".")[0]
|
|
7
7
|
subfolder_path = os.path.join(out_folder, subfolder_name)
|
|
8
|
-
os.makedirs(subfolder_path, exist_ok=True)
|
|
9
8
|
return subfolder_path
|
|
10
9
|
|
|
11
10
|
|
|
@@ -23,6 +22,8 @@ def markdown_exists(out_folder, fname):
|
|
|
23
22
|
|
|
24
23
|
def save_markdown(out_folder, fname, full_text, images, out_metadata):
|
|
25
24
|
subfolder_path = get_subfolder_path(out_folder, fname)
|
|
25
|
+
os.makedirs(subfolder_path, exist_ok=True)
|
|
26
|
+
|
|
26
27
|
markdown_filepath = get_markdown_filepath(out_folder, fname)
|
|
27
28
|
out_meta_filepath = markdown_filepath.rsplit(".", 1)[0] + "_meta.json"
|
|
28
29
|
|
|
@@ -96,29 +96,31 @@ def block_surround(text, block_type):
|
|
|
96
96
|
def line_separator(line1, line2, block_type, is_continuation=False):
|
|
97
97
|
# Should cover latin-derived languages and russian
|
|
98
98
|
lowercase_letters = r'\p{Lo}|\p{Ll}|\d'
|
|
99
|
+
hyphens = r'-—¬'
|
|
99
100
|
# Remove hyphen in current line if next line and current line appear to be joined
|
|
100
|
-
hyphen_pattern = regex.compile(rf'.*[{lowercase_letters}][
|
|
101
|
+
hyphen_pattern = regex.compile(rf'.*[{lowercase_letters}][{hyphens}]\s?$', regex.DOTALL)
|
|
101
102
|
if line1 and hyphen_pattern.match(line1) and regex.match(rf"^\s?[{lowercase_letters}]", line2):
|
|
102
103
|
# Split on — or - from the right
|
|
103
|
-
line1 = regex.split(
|
|
104
|
+
line1 = regex.split(rf"[{hyphens}]\s?$", line1)[0]
|
|
104
105
|
return line1.rstrip() + line2.lstrip()
|
|
105
106
|
|
|
106
107
|
all_letters = r'\p{L}|\d'
|
|
107
|
-
sentence_continuations = r',;\(\—\"\''
|
|
108
|
+
sentence_continuations = r',;\(\—\"\'\*'
|
|
108
109
|
sentence_ends = r'。ๆ\.?!'
|
|
109
110
|
line_end_pattern = regex.compile(rf'.*[{lowercase_letters}][{sentence_continuations}]?\s?$', regex.DOTALL)
|
|
110
111
|
line_start_pattern = regex.compile(rf'^\s?[{all_letters}]', regex.DOTALL)
|
|
111
112
|
sentence_end_pattern = regex.compile(rf'.*[{sentence_ends}]\s?$', regex.DOTALL)
|
|
112
113
|
|
|
114
|
+
text_blocks = ["Text", "List-item", "Footnote", "Caption", "Figure"]
|
|
113
115
|
if block_type in ["Title", "Section-header"]:
|
|
114
116
|
return line1.rstrip() + " " + line2.lstrip()
|
|
115
117
|
elif block_type == "Formula":
|
|
116
118
|
return line1 + "\n" + line2
|
|
117
|
-
elif line_end_pattern.match(line1) and line_start_pattern.match(line2) and block_type
|
|
119
|
+
elif line_end_pattern.match(line1) and line_start_pattern.match(line2) and block_type in text_blocks:
|
|
118
120
|
return line1.rstrip() + " " + line2.lstrip()
|
|
119
121
|
elif is_continuation:
|
|
120
122
|
return line1.rstrip() + " " + line2.lstrip()
|
|
121
|
-
elif block_type
|
|
123
|
+
elif block_type in text_blocks and sentence_end_pattern.match(line1):
|
|
122
124
|
return line1 + "\n\n" + line2
|
|
123
125
|
elif block_type == "Table":
|
|
124
126
|
return line1 + "\n\n" + line2
|
|
@@ -27,7 +27,7 @@ class Settings(BaseSettings):
|
|
|
27
27
|
return "cpu"
|
|
28
28
|
|
|
29
29
|
INFERENCE_RAM: int = 40 # How much VRAM each GPU has (in GB).
|
|
30
|
-
VRAM_PER_TASK: float = 4 # How much VRAM to allocate per task (in GB). Peak marker VRAM usage is around 5GB, but avg across workers is lower.
|
|
30
|
+
VRAM_PER_TASK: float = 4.5 # How much VRAM to allocate per task (in GB). Peak marker VRAM usage is around 5GB, but avg across workers is lower.
|
|
31
31
|
DEFAULT_LANG: str = "English" # Default language we assume files to be in, should be one of the keys in TESSERACT_LANGUAGES
|
|
32
32
|
|
|
33
33
|
SUPPORTED_FILETYPES: Dict = {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "marker-pdf"
|
|
3
|
-
version = "0.2.
|
|
3
|
+
version = "0.2.4"
|
|
4
4
|
description = "Convert PDF to markdown with high speed and accuracy."
|
|
5
5
|
authors = ["Vik Paruchuri <github@vikas.sh>"]
|
|
6
6
|
readme = "README.md"
|
|
@@ -37,6 +37,7 @@ surya-ocr = "^0.4.3"
|
|
|
37
37
|
filetype = "^1.2.0"
|
|
38
38
|
regex = "^2024.4.28"
|
|
39
39
|
pdftext = "^0.3.7"
|
|
40
|
+
grpcio = "^1.63.0"
|
|
40
41
|
|
|
41
42
|
[tool.poetry.group.dev.dependencies]
|
|
42
43
|
jupyter = "^1.0.0"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|