marker-pdf 0.2.2__tar.gz → 0.2.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/PKG-INFO +4 -3
  2. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/README.md +2 -2
  3. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/chunk_convert.py +4 -1
  4. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/convert.py +1 -0
  5. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/cleaners/code.py +3 -3
  6. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/output.py +2 -1
  7. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/pdf/utils.py +3 -0
  8. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/postprocessors/markdown.py +7 -5
  9. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/settings.py +1 -1
  10. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/pyproject.toml +2 -1
  11. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/LICENSE +0 -0
  12. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/chunk_convert.sh +0 -0
  13. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/convert_single.py +0 -0
  14. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/benchmark/scoring.py +0 -0
  15. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/cleaners/bullets.py +0 -0
  16. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/cleaners/fontstyle.py +0 -0
  17. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/cleaners/headers.py +0 -0
  18. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/cleaners/headings.py +0 -0
  19. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/cleaners/text.py +0 -0
  20. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/convert.py +0 -0
  21. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/debug/data.py +0 -0
  22. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/equations/equations.py +0 -0
  23. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/equations/inference.py +0 -0
  24. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/images/extract.py +0 -0
  25. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/images/save.py +0 -0
  26. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/layout/layout.py +0 -0
  27. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/layout/order.py +0 -0
  28. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/logger.py +0 -0
  29. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/models.py +0 -0
  30. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/ocr/detection.py +0 -0
  31. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/ocr/heuristics.py +0 -0
  32. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/ocr/lang.py +0 -0
  33. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/ocr/recognition.py +0 -0
  34. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/ocr/tesseract.py +0 -0
  35. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/ocr/utils.py +0 -0
  36. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/pdf/extract_text.py +0 -0
  37. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/pdf/images.py +0 -0
  38. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/postprocessors/editor.py +0 -0
  39. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/postprocessors/t5.py +0 -0
  40. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/schema/bbox.py +0 -0
  41. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/schema/block.py +0 -0
  42. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/schema/merged.py +0 -0
  43. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/schema/page.py +0 -0
  44. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/tables/cells.py +0 -0
  45. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/tables/table.py +0 -0
  46. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/tables/utils.py +0 -0
  47. {marker_pdf-0.2.2 → marker_pdf-0.2.4}/marker/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: marker-pdf
3
- Version: 0.2.2
3
+ Version: 0.2.4
4
4
  Summary: Convert PDF to markdown with high speed and accuracy.
5
5
  Home-page: https://github.com/VikParuchuri/marker
6
6
  License: GPL-3.0-or-later
@@ -16,6 +16,7 @@ Classifier: Programming Language :: Python :: 3.11
16
16
  Requires-Dist: Pillow (>=10.1.0,<11.0.0)
17
17
  Requires-Dist: filetype (>=1.2.0,<2.0.0)
18
18
  Requires-Dist: ftfy (>=6.1.1,<7.0.0)
19
+ Requires-Dist: grpcio (>=1.63.0,<2.0.0)
19
20
  Requires-Dist: numpy (>=1.26.1,<2.0.0)
20
21
  Requires-Dist: pdftext (>=0.3.7,<0.4.0)
21
22
  Requires-Dist: pydantic (>=2.4.2,<3.0.0)
@@ -71,7 +72,7 @@ It only uses models where necessary, which improves speed and accuracy.
71
72
 
72
73
  ![Benchmark overall](data/images/overall.png)
73
74
 
74
- The above results are with marker and nougat setup so they each take ~3GB of VRAM on an A6000.
75
+ The above results are with marker and nougat setup so they each take ~4GB of VRAM on an A6000.
75
76
 
76
77
  See [below](#benchmarks) for detailed speed and accuracy benchmarks, and instructions on how to run your own benchmarks.
77
78
 
@@ -214,7 +215,7 @@ Peak GPU memory usage during the benchmark is `4.2GB` for nougat, and `4.1GB` fo
214
215
 
215
216
  **Throughput**
216
217
 
217
- Marker takes about 4GB of VRAM on average per task, so you can convert 12 documents in parallel on an A6000.
218
+ Marker takes about 4.5GB of VRAM on average per task, so you can convert 10 documents in parallel on an A6000.
218
219
 
219
220
  ![Benchmark results](data/images/per_doc.png)
220
221
 
@@ -35,7 +35,7 @@ It only uses models where necessary, which improves speed and accuracy.
35
35
 
36
36
  ![Benchmark overall](data/images/overall.png)
37
37
 
38
- The above results are with marker and nougat setup so they each take ~3GB of VRAM on an A6000.
38
+ The above results are with marker and nougat setup so they each take ~4GB of VRAM on an A6000.
39
39
 
40
40
  See [below](#benchmarks) for detailed speed and accuracy benchmarks, and instructions on how to run your own benchmarks.
41
41
 
@@ -178,7 +178,7 @@ Peak GPU memory usage during the benchmark is `4.2GB` for nougat, and `4.1GB` fo
178
178
 
179
179
  **Throughput**
180
180
 
181
- Marker takes about 4GB of VRAM on average per task, so you can convert 12 documents in parallel on an A6000.
181
+ Marker takes about 4.5GB of VRAM on average per task, so you can convert 10 documents in parallel on an A6000.
182
182
 
183
183
  ![Benchmark results](data/images/per_doc.png)
184
184
 
@@ -1,5 +1,6 @@
1
1
  import argparse
2
2
  import subprocess
3
+ import pkg_resources
3
4
 
4
5
 
5
6
  def main():
@@ -8,8 +9,10 @@ def main():
8
9
  parser.add_argument("out_folder", help="Output folder")
9
10
  args = parser.parse_args()
10
11
 
12
+ script_path = pkg_resources.resource_filename(__name__, 'chunk_convert.sh')
13
+
11
14
  # Construct the command
12
- cmd = f"./chunk_convert.sh {args.in_folder} {args.out_folder}"
15
+ cmd = f"{script_path} {args.in_folder} {args.out_folder}"
13
16
 
14
17
  # Execute the shell script
15
18
  subprocess.run(cmd, shell=True, check=True)
@@ -24,6 +24,7 @@ def process_single_pdf(filepath: str, out_folder: str, model_refs, metadata: Opt
24
24
  fname = os.path.basename(filepath)
25
25
  if markdown_exists(out_folder, fname):
26
26
  return
27
+
27
28
  try:
28
29
  # Skip trying to convert files that don't have a lot of embedded text
29
30
  # This can indicate that they were scanned, and not OCRed properly
@@ -67,13 +67,13 @@ def identify_code_blocks(pages: List[Page]):
67
67
  is_code = [
68
68
  len(block.lines) > 3,
69
69
  is_code_linelen(block.lines),
70
- sum(is_indent) + comment_lines > len(block.lines) * .3,
70
+ sum(is_indent) + comment_lines > len(block.lines) * .7, # Indentation and comments are a majority
71
71
  ]
72
72
 
73
73
  if avg_font_size is not None:
74
74
  font_checks = [
75
- mean(line_font_sizes) <= avg_font_size, # Lower than average font size and line height
76
- mean(block_line_heights) < avg_line_height
75
+ mean(line_font_sizes) <= avg_font_size * .8, # Lower than average font size and line height
76
+ mean(block_line_heights) < avg_line_height * .8
77
77
  ]
78
78
  is_code += font_checks
79
79
 
@@ -5,7 +5,6 @@ import json
5
5
  def get_subfolder_path(out_folder, fname):
6
6
  subfolder_name = fname.split(".")[0]
7
7
  subfolder_path = os.path.join(out_folder, subfolder_name)
8
- os.makedirs(subfolder_path, exist_ok=True)
9
8
  return subfolder_path
10
9
 
11
10
 
@@ -23,6 +22,8 @@ def markdown_exists(out_folder, fname):
23
22
 
24
23
  def save_markdown(out_folder, fname, full_text, images, out_metadata):
25
24
  subfolder_path = get_subfolder_path(out_folder, fname)
25
+ os.makedirs(subfolder_path, exist_ok=True)
26
+
26
27
  markdown_filepath = get_markdown_filepath(out_folder, fname)
27
28
  out_meta_filepath = markdown_filepath.rsplit(".", 1)[0] + "_meta.json"
28
29
 
@@ -7,6 +7,9 @@ from marker.settings import settings
7
7
 
8
8
  def find_filetype(fpath):
9
9
  kind = filetype.guess(fpath)
10
+ if kind is None:
11
+ print(f"Could not determine filetype for {fpath}")
12
+ return "other"
10
13
 
11
14
  mimetype = kind.mime
12
15
 
@@ -96,29 +96,31 @@ def block_surround(text, block_type):
96
96
  def line_separator(line1, line2, block_type, is_continuation=False):
97
97
  # Should cover latin-derived languages and russian
98
98
  lowercase_letters = r'\p{Lo}|\p{Ll}|\d'
99
+ hyphens = r'-—¬'
99
100
  # Remove hyphen in current line if next line and current line appear to be joined
100
- hyphen_pattern = regex.compile(rf'.*[{lowercase_letters}][-]\s?$', regex.DOTALL)
101
+ hyphen_pattern = regex.compile(rf'.*[{lowercase_letters}][{hyphens}]\s?$', regex.DOTALL)
101
102
  if line1 and hyphen_pattern.match(line1) and regex.match(rf"^\s?[{lowercase_letters}]", line2):
102
103
  # Split on — or - from the right
103
- line1 = regex.split(r"[-—]\s?$", line1)[0]
104
+ line1 = regex.split(rf"[{hyphens}]\s?$", line1)[0]
104
105
  return line1.rstrip() + line2.lstrip()
105
106
 
106
107
  all_letters = r'\p{L}|\d'
107
- sentence_continuations = r',;\(\—\"\''
108
+ sentence_continuations = r',;\(\—\"\'\*'
108
109
  sentence_ends = r'。ๆ\.?!'
109
110
  line_end_pattern = regex.compile(rf'.*[{lowercase_letters}][{sentence_continuations}]?\s?$', regex.DOTALL)
110
111
  line_start_pattern = regex.compile(rf'^\s?[{all_letters}]', regex.DOTALL)
111
112
  sentence_end_pattern = regex.compile(rf'.*[{sentence_ends}]\s?$', regex.DOTALL)
112
113
 
114
+ text_blocks = ["Text", "List-item", "Footnote", "Caption", "Figure"]
113
115
  if block_type in ["Title", "Section-header"]:
114
116
  return line1.rstrip() + " " + line2.lstrip()
115
117
  elif block_type == "Formula":
116
118
  return line1 + "\n" + line2
117
- elif line_end_pattern.match(line1) and line_start_pattern.match(line2) and block_type == "Text":
119
+ elif line_end_pattern.match(line1) and line_start_pattern.match(line2) and block_type in text_blocks:
118
120
  return line1.rstrip() + " " + line2.lstrip()
119
121
  elif is_continuation:
120
122
  return line1.rstrip() + " " + line2.lstrip()
121
- elif block_type == "Text" and sentence_end_pattern.match(line1):
123
+ elif block_type in text_blocks and sentence_end_pattern.match(line1):
122
124
  return line1 + "\n\n" + line2
123
125
  elif block_type == "Table":
124
126
  return line1 + "\n\n" + line2
@@ -27,7 +27,7 @@ class Settings(BaseSettings):
27
27
  return "cpu"
28
28
 
29
29
  INFERENCE_RAM: int = 40 # How much VRAM each GPU has (in GB).
30
- VRAM_PER_TASK: float = 4 # How much VRAM to allocate per task (in GB). Peak marker VRAM usage is around 5GB, but avg across workers is lower.
30
+ VRAM_PER_TASK: float = 4.5 # How much VRAM to allocate per task (in GB). Peak marker VRAM usage is around 5GB, but avg across workers is lower.
31
31
  DEFAULT_LANG: str = "English" # Default language we assume files to be in, should be one of the keys in TESSERACT_LANGUAGES
32
32
 
33
33
  SUPPORTED_FILETYPES: Dict = {
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "marker-pdf"
3
- version = "0.2.2"
3
+ version = "0.2.4"
4
4
  description = "Convert PDF to markdown with high speed and accuracy."
5
5
  authors = ["Vik Paruchuri <github@vikas.sh>"]
6
6
  readme = "README.md"
@@ -37,6 +37,7 @@ surya-ocr = "^0.4.3"
37
37
  filetype = "^1.2.0"
38
38
  regex = "^2024.4.28"
39
39
  pdftext = "^0.3.7"
40
+ grpcio = "^1.63.0"
40
41
 
41
42
  [tool.poetry.group.dev.dependencies]
42
43
  jupyter = "^1.0.0"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes