scanlayer 1.0.2__tar.gz → 1.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. {scanlayer-1.0.2/scanlayer.egg-info → scanlayer-1.1.0}/PKG-INFO +26 -28
  2. {scanlayer-1.0.2 → scanlayer-1.1.0}/README.md +24 -27
  3. {scanlayer-1.0.2 → scanlayer-1.1.0}/pyproject.toml +2 -2
  4. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/cli/parser.py +8 -0
  5. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/cli/run.py +8 -1
  6. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/config.py +7 -0
  7. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/main.py +101 -11
  8. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/ocr/engine.py +45 -7
  9. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/pdf/fonts.py +4 -1
  10. {scanlayer-1.0.2 → scanlayer-1.1.0/scanlayer.egg-info}/PKG-INFO +26 -28
  11. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer.egg-info/SOURCES.txt +1 -0
  12. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer.egg-info/requires.txt +1 -0
  13. scanlayer-1.1.0/tests/test_merge_workers.py +316 -0
  14. {scanlayer-1.0.2 → scanlayer-1.1.0}/LICENSE.md +0 -0
  15. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/__init__.py +0 -0
  16. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/__main__.py +0 -0
  17. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/cli/__init__.py +0 -0
  18. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/cli/dry_run.py +0 -0
  19. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/fonts/DejaVuSans.ttf +0 -0
  20. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/layout/__init__.py +0 -0
  21. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/layout/columns.py +0 -0
  22. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/ocr/__init__.py +0 -0
  23. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/ocr/export.py +0 -0
  24. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/pdf/__init__.py +0 -0
  25. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/pdf/builder.py +0 -0
  26. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/preprocessing/__init__.py +0 -0
  27. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/preprocessing/enhance.py +0 -0
  28. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/utils/__init__.py +0 -0
  29. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/utils/debug_image.py +0 -0
  30. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/utils/errors.py +0 -0
  31. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/utils/logger.py +0 -0
  32. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/utils/validators.py +0 -0
  33. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer.egg-info/dependency_links.txt +0 -0
  34. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer.egg-info/entry_points.txt +0 -0
  35. {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer.egg-info/top_level.txt +0 -0
  36. {scanlayer-1.0.2 → scanlayer-1.1.0}/setup.cfg +0 -0
  37. {scanlayer-1.0.2 → scanlayer-1.1.0}/tests/test_columns.py +0 -0
  38. {scanlayer-1.0.2 → scanlayer-1.1.0}/tests/test_config.py +0 -0
  39. {scanlayer-1.0.2 → scanlayer-1.1.0}/tests/test_dry_run.py +0 -0
  40. {scanlayer-1.0.2 → scanlayer-1.1.0}/tests/test_export.py +0 -0
  41. {scanlayer-1.0.2 → scanlayer-1.1.0}/tests/test_fonts.py +0 -0
  42. {scanlayer-1.0.2 → scanlayer-1.1.0}/tests/test_main_integration.py +0 -0
  43. {scanlayer-1.0.2 → scanlayer-1.1.0}/tests/test_pdf_builder.py +0 -0
  44. {scanlayer-1.0.2 → scanlayer-1.1.0}/tests/test_validators.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scanlayer
3
- Version: 1.0.2
3
+ Version: 1.1.0
4
4
  Summary: Image (scan/photo) to searchable PDF, or raw OCR export, via Tesseract.
5
5
  Author: Hyacinthe-primus
6
6
  Author-email: Hyacinthe-primus <hyacintheatho91@gmail.com>
@@ -19,27 +19,35 @@ Requires-Dist: numpy>=1.24.0
19
19
  Requires-Dist: pdf2image>=1.17.0
20
20
  Provides-Extra: test
21
21
  Requires-Dist: pytest>=7.0; extra == "test"
22
+ Requires-Dist: pytest-cov>=4.1; extra == "test"
22
23
  Provides-Extra: yaml
23
24
  Requires-Dist: PyYAML>=6.0; extra == "yaml"
24
25
  Dynamic: license-file
25
26
 
26
27
  <div align="center">
27
- <img src="scanlayer.png" alt="scanlayer" width="110"/>
28
+ <img src="https://raw.githubusercontent.com/Hyacinthe-primus/scanlayer/main/scanlayer.png" alt="scanlayer" width="110"/>
28
29
  <h1>scanlayer</h1>
29
- <p><em>Scanned image or photographed document &rarr; searchable PDF, or raw OCR text/JSON/TSV/hOCR.</em></p>
30
+ <p><em> Turn scanned image or photographed document into searchable PDF, or raw OCR text/JSON/TSV/hOCR.</em></p>
30
31
  </div>
31
32
 
32
33
  **Turn a scanned image or photographed document into a searchable PDF**, or
33
34
  export the raw OCR result as plain text, JSON, TSV, or hOCR. Use it as a
34
35
  command-line tool or as a Python library; both are the same engine underneath.
35
36
 
36
- [![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue)](#)
37
+ [![PyPI version](https://badge.fury.io/py/scanlayer.svg)](https://badge.fury.io/py/scanlayer)
38
+ [![codecov](https://codecov.io/gh/Hyacinthe-primus/scanlayer/branch/main/graph/badge.svg)](https://codecov.io/gh/Hyacinthe-primus/scanlayer)
39
+ [![Python](https://img.shields.io/badge/python-3.9%2B-3776AB)](#)
37
40
  ![Platforms](https://img.shields.io/badge/platform-Windows%20%7C%20macOS%20%7C%20Linux-lightgrey)
38
41
  [![](https://img.shields.io/badge/-documentation-green)](https://Hyacinthe-primus.github.io/scanlayer/)
42
+ [![GitHub last commit](https://img.shields.io/github/last-commit/Hyacinthe-primus/scanlayer)](https://github.com/Hyacinthe-primus/scanlayer)
39
43
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE.md)
40
44
 
41
45
  ---
42
46
 
47
+ ## Documentation
48
+
49
+ [Full documentation](https://Hyacinthe-primus.github.io/scanlayer/) – installation, CLI & API reference, configuration, output formats, examples, and troubleshooting.
50
+
43
51
  ## What it does
44
52
 
45
53
  A scanned invoice, a phone photo of a letter, a stack of photographed pages:
@@ -172,28 +180,6 @@ scanlayer.convert_merge(["page1.jpg", "page2.jpg", "page3.jpg"], "report.pdf")
172
180
  scanlayer.convert("invoice.jpg", "invoice.json", output_format="json")
173
181
  ```
174
182
 
175
- ## Documentation
176
-
177
- | | |
178
- |---|---|
179
- | **[Installation](https://Hyacinthe-primus.github.io/scanlayer/installation.html)** | Tesseract, poppler, and the two ways to install scanlayer itself |
180
- | **[Examples](https://Hyacinthe-primus.github.io/scanlayer/examples.html)** | Every feature, CLI and library side by side |
181
- | **[CLI Reference](https://Hyacinthe-primus.github.io/scanlayer/cli-reference.html)** | Every flag and exit code |
182
- | **[Library API](https://Hyacinthe-primus.github.io/scanlayer/library-api.html)** | `convert()`, `convert_batch()`, `convert_merge()`, exceptions, and the full low-level pipeline API |
183
- | **[Feature Catalog](https://Hyacinthe-primus.github.io/scanlayer/features.html)** | Everything scanlayer does, by pipeline stage |
184
- | **[Configuration](https://Hyacinthe-primus.github.io/scanlayer/configuration.html)** | `configure()`, config files, precedence, every tunable |
185
- | **[Output Formats](https://Hyacinthe-primus.github.io/scanlayer/output-formats.html)** | The `pdf`/`txt`/`json`/`tsv`/`hocr` schemas |
186
- | **[Multi-Page & Merge](https://Hyacinthe-primus.github.io/scanlayer/multi-page-merge.html)** | Batching, merging, native PDF input |
187
- | **[Debug Visualization](https://Hyacinthe-primus.github.io/scanlayer/debug-visualization.html)** | Reading the `--debug-image` confidence overlay |
188
- | **[Bundling Tesseract](https://Hyacinthe-primus.github.io/scanlayer/bundling-tesseract.html)** | Shipping your own Tesseract binary |
189
- | **[Troubleshooting](https://Hyacinthe-primus.github.io/scanlayer/troubleshooting.html)** | Common errors and fixes |
190
- | **[Roadmap & Limitations](https://Hyacinthe-primus.github.io/scanlayer/roadmap.html)** | What's missing and what's deliberately out of scope |
191
-
192
- The site is published from the `gh-pages` branch at
193
- https://Hyacinthe-primus.github.io/scanlayer/. To browse it locally, open
194
- `index.html` in a `gh-pages` worktree (e.g. `git worktree add <path> gh-pages`);
195
- it has no build step or server-side dependency.
196
-
197
183
  ## Repository layout
198
184
 
199
185
  ```
@@ -226,8 +212,20 @@ Bug reports, fixes, and feature discussions are welcome: see
226
212
  [CONTRIBUTING.md](CONTRIBUTING.md) for how to set up a development
227
213
  environment and what to include in a pull request. There is a `tests/`
228
214
  directory (now including `utils/validators.py` and the `--dry-run` flag, with
229
- cross-platform coverage for Tesseract discovery on Windows/macOS/Linux) and it runs with `pytest`, but no CI yet; see [Roadmap & Limitations](https://Hyacinthe-primus.github.io/scanlayer/roadmap.html) and the [Adding tests](CONTRIBUTING.md#adding-tests) section of the contributing guide for where coverage is thinnest.
215
+ cross-platform coverage for Tesseract discovery on Windows/macOS/Linux) and it runs with `pytest` automatically in CI on every push and pull request; see [Roadmap & Limitations](https://Hyacinthe-primus.github.io/scanlayer/roadmap.html) and the [Adding tests](CONTRIBUTING.md#adding-tests) section of the contributing guide for where coverage is thinnest.
216
+
217
+ ## Support the project
218
+
219
+ scanlayer is free and MIT-licensed, maintained in spare time. If it saves you
220
+ time, there is no fee – but a little support goes a long way:
221
+
222
+ - **Star the repository** – the simplest signal that the project is worth
223
+ maintaining.
224
+ - **Report bugs or request features** by opening a
225
+ [GitHub Issue](https://github.com/Hyacinthe-primus/scanlayer/issues).
226
+ - **Contribute** a fix, test, or example – see
227
+ [CONTRIBUTING.md](CONTRIBUTING.md).
230
228
 
231
229
  ## License
232
230
 
233
- [MIT](LICENSE.md).
231
+ This project is licensed under the [MIT License](LICENSE.md).
@@ -1,20 +1,27 @@
1
1
  <div align="center">
2
- <img src="scanlayer.png" alt="scanlayer" width="110"/>
2
+ <img src="https://raw.githubusercontent.com/Hyacinthe-primus/scanlayer/main/scanlayer.png" alt="scanlayer" width="110"/>
3
3
  <h1>scanlayer</h1>
4
- <p><em>Scanned image or photographed document &rarr; searchable PDF, or raw OCR text/JSON/TSV/hOCR.</em></p>
4
+ <p><em> Turn scanned image or photographed document into searchable PDF, or raw OCR text/JSON/TSV/hOCR.</em></p>
5
5
  </div>
6
6
 
7
7
  **Turn a scanned image or photographed document into a searchable PDF**, or
8
8
  export the raw OCR result as plain text, JSON, TSV, or hOCR. Use it as a
9
9
  command-line tool or as a Python library; both are the same engine underneath.
10
10
 
11
- [![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue)](#)
11
+ [![PyPI version](https://badge.fury.io/py/scanlayer.svg)](https://badge.fury.io/py/scanlayer)
12
+ [![codecov](https://codecov.io/gh/Hyacinthe-primus/scanlayer/branch/main/graph/badge.svg)](https://codecov.io/gh/Hyacinthe-primus/scanlayer)
13
+ [![Python](https://img.shields.io/badge/python-3.9%2B-3776AB)](#)
12
14
  ![Platforms](https://img.shields.io/badge/platform-Windows%20%7C%20macOS%20%7C%20Linux-lightgrey)
13
15
  [![](https://img.shields.io/badge/-documentation-green)](https://Hyacinthe-primus.github.io/scanlayer/)
16
+ [![GitHub last commit](https://img.shields.io/github/last-commit/Hyacinthe-primus/scanlayer)](https://github.com/Hyacinthe-primus/scanlayer)
14
17
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE.md)
15
18
 
16
19
  ---
17
20
 
21
+ ## Documentation
22
+
23
+ [Full documentation](https://Hyacinthe-primus.github.io/scanlayer/) – installation, CLI & API reference, configuration, output formats, examples, and troubleshooting.
24
+
18
25
  ## What it does
19
26
 
20
27
  A scanned invoice, a phone photo of a letter, a stack of photographed pages:
@@ -147,28 +154,6 @@ scanlayer.convert_merge(["page1.jpg", "page2.jpg", "page3.jpg"], "report.pdf")
147
154
  scanlayer.convert("invoice.jpg", "invoice.json", output_format="json")
148
155
  ```
149
156
 
150
- ## Documentation
151
-
152
- | | |
153
- |---|---|
154
- | **[Installation](https://Hyacinthe-primus.github.io/scanlayer/installation.html)** | Tesseract, poppler, and the two ways to install scanlayer itself |
155
- | **[Examples](https://Hyacinthe-primus.github.io/scanlayer/examples.html)** | Every feature, CLI and library side by side |
156
- | **[CLI Reference](https://Hyacinthe-primus.github.io/scanlayer/cli-reference.html)** | Every flag and exit code |
157
- | **[Library API](https://Hyacinthe-primus.github.io/scanlayer/library-api.html)** | `convert()`, `convert_batch()`, `convert_merge()`, exceptions, and the full low-level pipeline API |
158
- | **[Feature Catalog](https://Hyacinthe-primus.github.io/scanlayer/features.html)** | Everything scanlayer does, by pipeline stage |
159
- | **[Configuration](https://Hyacinthe-primus.github.io/scanlayer/configuration.html)** | `configure()`, config files, precedence, every tunable |
160
- | **[Output Formats](https://Hyacinthe-primus.github.io/scanlayer/output-formats.html)** | The `pdf`/`txt`/`json`/`tsv`/`hocr` schemas |
161
- | **[Multi-Page & Merge](https://Hyacinthe-primus.github.io/scanlayer/multi-page-merge.html)** | Batching, merging, native PDF input |
162
- | **[Debug Visualization](https://Hyacinthe-primus.github.io/scanlayer/debug-visualization.html)** | Reading the `--debug-image` confidence overlay |
163
- | **[Bundling Tesseract](https://Hyacinthe-primus.github.io/scanlayer/bundling-tesseract.html)** | Shipping your own Tesseract binary |
164
- | **[Troubleshooting](https://Hyacinthe-primus.github.io/scanlayer/troubleshooting.html)** | Common errors and fixes |
165
- | **[Roadmap & Limitations](https://Hyacinthe-primus.github.io/scanlayer/roadmap.html)** | What's missing and what's deliberately out of scope |
166
-
167
- The site is published from the `gh-pages` branch at
168
- https://Hyacinthe-primus.github.io/scanlayer/. To browse it locally, open
169
- `index.html` in a `gh-pages` worktree (e.g. `git worktree add <path> gh-pages`);
170
- it has no build step or server-side dependency.
171
-
172
157
  ## Repository layout
173
158
 
174
159
  ```
@@ -201,8 +186,20 @@ Bug reports, fixes, and feature discussions are welcome: see
201
186
  [CONTRIBUTING.md](CONTRIBUTING.md) for how to set up a development
202
187
  environment and what to include in a pull request. There is a `tests/`
203
188
  directory (now including `utils/validators.py` and the `--dry-run` flag, with
204
- cross-platform coverage for Tesseract discovery on Windows/macOS/Linux) and it runs with `pytest`, but no CI yet; see [Roadmap & Limitations](https://Hyacinthe-primus.github.io/scanlayer/roadmap.html) and the [Adding tests](CONTRIBUTING.md#adding-tests) section of the contributing guide for where coverage is thinnest.
189
+ cross-platform coverage for Tesseract discovery on Windows/macOS/Linux) and it runs with `pytest` automatically in CI on every push and pull request; see [Roadmap & Limitations](https://Hyacinthe-primus.github.io/scanlayer/roadmap.html) and the [Adding tests](CONTRIBUTING.md#adding-tests) section of the contributing guide for where coverage is thinnest.
190
+
191
+ ## Support the project
192
+
193
+ scanlayer is free and MIT-licensed, maintained in spare time. If it saves you
194
+ time, there is no fee – but a little support goes a long way:
195
+
196
+ - **Star the repository** – the simplest signal that the project is worth
197
+ maintaining.
198
+ - **Report bugs or request features** by opening a
199
+ [GitHub Issue](https://github.com/Hyacinthe-primus/scanlayer/issues).
200
+ - **Contribute** a fix, test, or example – see
201
+ [CONTRIBUTING.md](CONTRIBUTING.md).
205
202
 
206
203
  ## License
207
204
 
208
- [MIT](LICENSE.md).
205
+ This project is licensed under the [MIT License](LICENSE.md).
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "scanlayer"
7
- version = "1.0.2"
7
+ version = "1.1.0"
8
8
  description = "Image (scan/photo) to searchable PDF, or raw OCR export, via Tesseract."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.9"
@@ -23,7 +23,7 @@ dependencies = [
23
23
  ]
24
24
 
25
25
  [project.optional-dependencies]
26
- test = ["pytest>=7.0"]
26
+ test = ["pytest>=7.0", "pytest-cov>=4.1"]
27
27
  yaml = ["PyYAML>=6.0"]
28
28
 
29
29
  [project.scripts]
@@ -77,6 +77,14 @@ def _build_parser() -> argparse.ArgumentParser:
77
77
  "multi-page PDF input. Only with --format pdf. -o must "
78
78
  "be a file path, not a folder.",
79
79
  )
80
+ io_group.add_argument(
81
+ "--workers", type=int, default=None, metavar="N",
82
+ help="With --merge: process up to N pages at the same time "
83
+ f"(1-{config.MERGE_WORKERS_LIMIT}, default {config.MERGE_WORKERS}; "
84
+ "1 = one page at a time). Only changes speed: page order "
85
+ "and OCR output are the same. Ignored without --merge. "
86
+ "Equivalent to configure(merge_workers=N).",
87
+ )
80
88
  io_group.add_argument(
81
89
  "--config", default=None,
82
90
  help="Path to a .yaml/.yml/.json config profile applied via "
@@ -21,7 +21,7 @@ from scanlayer.cli.dry_run import (
21
21
  )
22
22
  from scanlayer.cli.parser import _build_parser
23
23
  from scanlayer.main import _expand_pdf_input, _resolve_output_path, convert, convert_merge
24
- from scanlayer.utils.logger import get_logger, log_error
24
+ from scanlayer.utils.logger import get_logger, log_error, log_warning
25
25
 
26
26
 
27
27
  def main(argv: list[str] = None) -> int:
@@ -112,6 +112,12 @@ def main(argv: list[str] = None) -> int:
112
112
  if args.merge and (args.output.endswith(("/", "\\")) or os.path.isdir(args.output)):
113
113
  log_error(log, "--merge requires -o/--output to be a file path, not a folder.")
114
114
  return EXIT_USER_ERROR
115
+ if args.workers is not None:
116
+ if args.workers < 1:
117
+ log_error(log, f"--workers must be 1 or more. Got {args.workers}.")
118
+ return EXIT_USER_ERROR
119
+ if not args.merge:
120
+ log_warning(log, "--workers only applies with --merge, ignoring it.")
115
121
 
116
122
  raw_inputs: list[str] = []
117
123
  for pat in args.input:
@@ -147,6 +153,7 @@ def main(argv: list[str] = None) -> int:
147
153
  pdf_metadata=pdf_metadata,
148
154
  force=args.force,
149
155
  orientation=orientation_value,
156
+ workers=args.workers,
150
157
  )
151
158
  return EXIT_OK
152
159
  except Exception as exc:
@@ -89,6 +89,12 @@ PSM_EARLY_EXIT_CONFIDENCE = 80.0
89
89
  PSM_PARALLEL = True
90
90
  PSM_MAX_WORKERS = None
91
91
 
92
+ # Pages processed concurrently by convert_merge()/--merge. 1 = one page at
93
+ # a time. Each page already races several Tesseract processes, so this is
94
+ # kept small; MERGE_WORKERS_LIMIT is a hard ceiling that is not configurable.
95
+ MERGE_WORKERS = 3
96
+ MERGE_WORKERS_LIMIT = 4
97
+
92
98
  OCR_TIMEOUT_SECONDS = 45
93
99
 
94
100
  # PSM candidates: engine.py runs one pass per PSM, keeps highest mean confidence.
@@ -161,6 +167,7 @@ _CONFIGURABLE_KEYS = {
161
167
  "psm_early_exit_confidence": "PSM_EARLY_EXIT_CONFIDENCE",
162
168
  "psm_parallel": "PSM_PARALLEL",
163
169
  "psm_max_workers": "PSM_MAX_WORKERS",
170
+ "merge_workers": "MERGE_WORKERS",
164
171
  "ocr_timeout_seconds": "OCR_TIMEOUT_SECONDS",
165
172
  "char_whitelist": "TESSERACT_CHAR_WHITELIST",
166
173
  "char_blacklist": "TESSERACT_CHAR_BLACKLIST",
@@ -9,7 +9,9 @@ from __future__ import annotations
9
9
  import glob
10
10
  import os
11
11
  import tempfile
12
+ import threading
12
13
  import time
14
+ from concurrent.futures import ThreadPoolExecutor
13
15
  from dataclasses import dataclass
14
16
  from typing import Optional
15
17
 
@@ -17,7 +19,7 @@ from PIL import Image
17
19
 
18
20
  from scanlayer import config
19
21
  from scanlayer.layout.columns import reorder_reading_order
20
- from scanlayer.ocr.engine import extract_words
22
+ from scanlayer.ocr.engine import extract_words, use_configured_tesseract
21
23
  from scanlayer.ocr.export import FORMATS, export_words
22
24
  from scanlayer.pdf.builder import build_searchable_pdf, build_searchable_pdf_multipage
23
25
  from scanlayer.preprocessing.enhance import preprocess
@@ -422,10 +424,12 @@ def _process_page_for_merge(
422
424
  char_blacklist: Optional[str],
423
425
  orientation: "str | float | None",
424
426
  force: bool,
427
+ ocr_slots: Optional[threading.Semaphore] = None,
425
428
  ):
426
429
  """Run preprocessing + OCR for one page of a --merge run.
427
430
 
428
431
  Returns (PageInput, stats_dict). Does not touch the output path.
432
+ Safe to call from several threads at once.
429
433
  """
430
434
  from scanlayer.pdf.builder import PageInput
431
435
 
@@ -467,6 +471,7 @@ def _process_page_for_merge(
467
471
  ocr_result = extract_words(
468
472
  result.ocr_image, result.ocr_scale, effective_dpi=result.effective_dpi,
469
473
  lang=lang, char_whitelist=char_whitelist, char_blacklist=char_blacklist,
474
+ ocr_slots=ocr_slots,
470
475
  )
471
476
  ocr_result.words, column_count = reorder_reading_order(
472
477
  ocr_result.words, page_width=result.background.width,
@@ -487,6 +492,86 @@ def _process_page_for_merge(
487
492
  return page, stats
488
493
 
489
494
 
495
+ def _resolve_merge_workers(workers: Optional[int], page_count: int) -> int:
496
+ """How many pages a merge run processes at once (always >= 1).
497
+
498
+ None means config.MERGE_WORKERS. The result never exceeds
499
+ config.MERGE_WORKERS_LIMIT, the page count, or the CPU count.
500
+ """
501
+ requested = config.MERGE_WORKERS if workers is None else workers
502
+ if isinstance(requested, bool) or not isinstance(requested, int) or requested < 1:
503
+ raise ValueError(f"workers must be an integer >= 1, got {requested!r}.")
504
+
505
+ limit = config.MERGE_WORKERS_LIMIT
506
+ if requested > limit:
507
+ log_warning(
508
+ get_logger("main"),
509
+ f"workers={requested} is above the maximum of {limit}, using {limit}.",
510
+ )
511
+ return max(1, min(requested, limit, page_count, os.cpu_count() or 1))
512
+
513
+
514
+ def _process_pages_for_merge(
515
+ input_paths: list[str],
516
+ workers: int,
517
+ lang: Optional[str],
518
+ dpi: Optional[int],
519
+ char_whitelist: Optional[str],
520
+ char_blacklist: Optional[str],
521
+ orientation: "str | float | None",
522
+ force: bool,
523
+ ) -> list:
524
+ """Run preprocessing + OCR for every page of a merge run.
525
+
526
+ Returns [(PageInput, stats), ...] in input order, whatever order the
527
+ pages finish in. If a page fails, the failure of the first bad page in
528
+ input order is raised and pages that have not started yet are dropped.
529
+ With workers == 1 pages are processed one at a time on the calling
530
+ thread, exactly as before.
531
+ """
532
+ log = get_logger("main")
533
+ page_args = (lang, dpi, char_whitelist, char_blacklist, orientation, force)
534
+ use_configured_tesseract()
535
+
536
+ def _done(input_path: str, stats: dict) -> None:
537
+ log.info(
538
+ f"'{input_path}': {stats['words']} words "
539
+ f"(mean confidence {stats['mean_conf']:.1f}%"
540
+ + (f", {stats['columns']} columns" if stats["columns"] > 1 else "")
541
+ + ")"
542
+ )
543
+
544
+ results: list = []
545
+ if workers <= 1:
546
+ for input_path in input_paths:
547
+ page, stats = _process_page_for_merge(input_path, *page_args)
548
+ _done(input_path, stats)
549
+ results.append((page, stats))
550
+ return results
551
+
552
+ # Every page already races several Tesseract processes. This shared
553
+ # semaphore keeps the total at the CPU count, so concurrent pages cannot
554
+ # starve each other into the per-call OCR timeout.
555
+ ocr_slots = threading.BoundedSemaphore(os.cpu_count() or 1)
556
+ log.info(f"Processing {len(input_paths)} page(s) with {workers} parallel workers.")
557
+
558
+ executor = ThreadPoolExecutor(max_workers=workers, thread_name_prefix="scanlayer-page")
559
+ try:
560
+ futures = [
561
+ executor.submit(
562
+ _process_page_for_merge, input_path, *page_args, ocr_slots=ocr_slots,
563
+ )
564
+ for input_path in input_paths
565
+ ]
566
+ for input_path, future in zip(input_paths, futures):
567
+ page, stats = future.result()
568
+ _done(input_path, stats)
569
+ results.append((page, stats))
570
+ finally:
571
+ executor.shutdown(wait=True, cancel_futures=True)
572
+ return results
573
+
574
+
490
575
  def convert_merge(
491
576
  input_paths: list[str],
492
577
  output_path: str,
@@ -498,15 +583,22 @@ def convert_merge(
498
583
  pdf_metadata: Optional[dict] = None,
499
584
  force: bool = False,
500
585
  orientation: "str | float | None" = None,
586
+ workers: Optional[int] = None,
501
587
  ) -> ConversionResult:
502
588
  """Combine several images into one multi-page searchable PDF.
503
589
 
504
590
  PDF-only. Raises BlankPageDetectedError if any page is blank and
505
591
  force=False.
592
+
593
+ workers: pages processed concurrently, 1 to config.MERGE_WORKERS_LIMIT
594
+ (4). None uses config.MERGE_WORKERS (default 3). 1 processes one
595
+ page at a time. It only changes speed: page order and OCR output are
596
+ the same for any value.
506
597
  """
507
598
  if not input_paths:
508
599
  raise ValueError("convert_merge() called with no input paths.")
509
600
 
601
+ worker_count = _resolve_merge_workers(workers, len(input_paths))
510
602
 
511
603
  log = get_logger("main")
512
604
  t0 = time.perf_counter()
@@ -518,22 +610,16 @@ def convert_merge(
518
610
  conf_weighted_sum = 0.0
519
611
  conf_weight_n = 0
520
612
  max_columns = 1
521
- for input_path in input_paths:
522
- page, stats = _process_page_for_merge(
523
- input_path, lang, dpi, char_whitelist, char_blacklist, orientation, force,
524
- )
613
+ for page, stats in _process_pages_for_merge(
614
+ input_paths, worker_count, lang, dpi, char_whitelist, char_blacklist,
615
+ orientation, force,
616
+ ):
525
617
  pages.append(page)
526
618
  total_words += stats["words"]
527
619
  if stats["words"]:
528
620
  conf_weighted_sum += stats["mean_conf"] * stats["words"]
529
621
  conf_weight_n += stats["words"]
530
622
  max_columns = max(max_columns, stats["columns"])
531
- log.info(
532
- f"'{input_path}': {stats['words']} words "
533
- f"(mean confidence {stats['mean_conf']:.1f}%"
534
- + (f", {stats['columns']} columns" if stats["columns"] > 1 else "")
535
- + ")"
536
- )
537
623
 
538
624
  output_size_bytes = build_searchable_pdf_multipage(
539
625
  pages, output_abs, jpeg_quality=jpeg_quality, metadata=pdf_metadata,
@@ -577,6 +663,7 @@ def convert_batch(
577
663
  orientation: "str | float | None" = None,
578
664
  output_format: str = "pdf",
579
665
  debug_image: bool = False,
666
+ workers: Optional[int] = None,
580
667
  verbose: bool = False,
581
668
  quiet: bool = False,
582
669
  ) -> BatchResult:
@@ -588,6 +675,8 @@ def convert_batch(
588
675
  (same directory/stem, extension from output_format). Not valid
589
676
  with merge=True, which needs one explicit shared output path.
590
677
  merge: combine all inputs into one multi-page PDF.
678
+ workers: with merge=True, pages processed concurrently (see
679
+ convert_merge()). Ignored without merge.
591
680
  verbose/quiet: set log level for the call.
592
681
 
593
682
  Never raises for per-file failures. Check BatchResult.failures.
@@ -636,6 +725,7 @@ def convert_batch(
636
725
  lang=lang, dpi=dpi, jpeg_quality=jpeg_quality,
637
726
  char_whitelist=char_whitelist, char_blacklist=char_blacklist,
638
727
  pdf_metadata=pdf_metadata, force=force, orientation=orientation,
728
+ workers=workers,
639
729
  ))
640
730
  except Exception as exc:
641
731
  failures.append(("<merge>", str(exc)))
@@ -8,6 +8,7 @@ confidence. Supports early-exit and parallel execution via threads.
8
8
  from __future__ import annotations
9
9
 
10
10
  import os
11
+ import threading
11
12
  import time
12
13
  import unicodedata
13
14
  from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
@@ -168,8 +169,29 @@ def _run_ocr(
168
169
  )
169
170
 
170
171
 
171
- def _safe_run_ocr(*args, psm: int, **kwargs) -> tuple[PsmAttempt | None, Exception | None]:
172
- """Run one PSM candidate, converting exceptions into return values."""
172
+ def use_configured_tesseract() -> None:
173
+ """Point pytesseract at config.TESSERACT_CMD.
174
+
175
+ pytesseract keeps the binary path in a module global, and
176
+ preprocessing's OSD call runs before the first extract_words(), so
177
+ callers that start concurrent work set it once up front.
178
+ """
179
+ pytesseract.pytesseract.tesseract_cmd = config.TESSERACT_CMD
180
+
181
+
182
+ def _safe_run_ocr(
183
+ *args,
184
+ psm: int,
185
+ ocr_slots: Optional[threading.Semaphore] = None,
186
+ **kwargs,
187
+ ) -> tuple[PsmAttempt | None, Exception | None]:
188
+ """Run one PSM candidate, converting exceptions into return values.
189
+
190
+ With ocr_slots, waits for a free slot first, so the Tesseract timeout
191
+ only starts counting once the process actually runs.
192
+ """
193
+ if ocr_slots is not None:
194
+ ocr_slots.acquire()
173
195
  try:
174
196
  return _run_ocr(*args, **kwargs), None
175
197
  except pytesseract.TesseractNotFoundError as exc:
@@ -183,6 +205,9 @@ def _safe_run_ocr(*args, psm: int, **kwargs) -> tuple[PsmAttempt | None, Excepti
183
205
  except Exception as exc: # genuinely unexpected, still surfaced, not swallowed
184
206
  log_warning(log, f"PSM {psm}, unexpected error ({type(exc).__name__}: {exc})")
185
207
  return None, exc
208
+ finally:
209
+ if ocr_slots is not None:
210
+ ocr_slots.release()
186
211
 
187
212
 
188
213
  def extract_words(
@@ -192,16 +217,21 @@ def extract_words(
192
217
  lang: Optional[str] = None,
193
218
  char_whitelist: Optional[str] = None,
194
219
  char_blacklist: Optional[str] = None,
220
+ ocr_slots: Optional[threading.Semaphore] = None,
195
221
  ) -> OcrResult:
196
222
  """Run OCR with automatic PSM selection.
197
223
 
224
+ ocr_slots, if given, is shared by concurrent callers (convert_merge()
225
+ processing several pages at once) and bounds how many Tesseract passes
226
+ run at the same time. It never changes which result is selected.
227
+
198
228
  Raises OCRProcessingError if every PSM candidate fails.
199
229
  """
200
230
  lang = lang or config.DEFAULT_OCR_LANG
201
231
  wl = char_whitelist if char_whitelist is not None else config.TESSERACT_CHAR_WHITELIST
202
232
  bl = char_blacklist if char_blacklist is not None else config.TESSERACT_CHAR_BLACKLIST
203
233
 
204
- pytesseract.pytesseract.tesseract_cmd = config.TESSERACT_CMD
234
+ use_configured_tesseract()
205
235
  tess_base = f'--tessdata-dir "{config.TESSDATA_DIR}"' if config.TESSDATA_DIR else ""
206
236
 
207
237
  psm_candidates = list(config.TESSERACT_PSM_CANDIDATES)
@@ -224,7 +254,7 @@ def extract_words(
224
254
  if len(psm_candidates) == 1:
225
255
  attempt, error = _safe_run_ocr(
226
256
  ocr_image, lang, psm_candidates[0], tess_base,
227
- effective_dpi, wl, bl, psm=psm_candidates[0],
257
+ effective_dpi, wl, bl, psm=psm_candidates[0], ocr_slots=ocr_slots,
228
258
  )
229
259
  if attempt is not None:
230
260
  attempts.append(attempt)
@@ -236,12 +266,13 @@ def extract_words(
236
266
  max_workers = config.PSM_MAX_WORKERS or min(32, (os.cpu_count() or 1) + 4)
237
267
  max_workers = min(max_workers, len(psm_candidates))
238
268
  can_skip_unstarted = max_workers < len(psm_candidates)
269
+ order = {psm: i for i, psm in enumerate(psm_candidates)}
239
270
 
240
271
  executor = ThreadPoolExecutor(max_workers=max_workers)
241
272
  futures = {
242
273
  executor.submit(
243
274
  _safe_run_ocr, ocr_image, lang, psm, tess_base,
244
- effective_dpi, wl, bl, psm=psm,
275
+ effective_dpi, wl, bl, psm=psm, ocr_slots=ocr_slots,
245
276
  ): psm
246
277
  for psm in psm_candidates
247
278
  }
@@ -253,7 +284,14 @@ def extract_words(
253
284
  attempt, error = future.result()
254
285
  if attempt is not None:
255
286
  attempts.append(attempt)
256
- if best is None or attempt.mean_confidence > best.mean_confidence:
287
+ if (
288
+ best is None
289
+ or attempt.mean_confidence > best.mean_confidence
290
+ or (
291
+ attempt.mean_confidence == best.mean_confidence
292
+ and order[attempt.psm] < order[best.psm]
293
+ )
294
+ ):
257
295
  best = attempt
258
296
  else:
259
297
  errors.append(error)
@@ -277,7 +315,7 @@ def extract_words(
277
315
  for psm in psm_candidates:
278
316
  attempt, error = _safe_run_ocr(
279
317
  ocr_image, lang, psm, tess_base,
280
- effective_dpi, wl, bl, psm=psm,
318
+ effective_dpi, wl, bl, psm=psm, ocr_slots=ocr_slots,
281
319
  )
282
320
  if attempt is not None:
283
321
  attempts.append(attempt)
@@ -12,6 +12,7 @@ Set config.FONT_PATH to force a specific TTF and skip auto-selection.
12
12
  from __future__ import annotations
13
13
 
14
14
  import os
15
+ import sys
15
16
 
16
17
  from reportlab.pdfbase import pdfmetrics
17
18
  from reportlab.pdfbase.cidfonts import UnicodeCIDFont
@@ -25,7 +26,9 @@ log = get_logger(__name__)
25
26
  FALLBACK_FONT = "Helvetica"
26
27
 
27
28
  _UNICODE_TTF_NAME = "ScanlayerUnicode"
28
- _UNICODE_TTF_PATH = os.path.join(os.path.dirname(__file__), "..", "fonts", "DejaVuSans.ttf")
29
+ _scanlayer_root = os.path.dirname(os.path.dirname(__file__))
30
+ _bundle_root = getattr(sys, "_MEIPASS", os.path.dirname(_scanlayer_root))
31
+ _UNICODE_TTF_PATH = os.path.join(_bundle_root, "scanlayer", "fonts", "DejaVuSans.ttf")
29
32
 
30
33
  _CUSTOM_TTF_NAME = "ScanlayerCustom"
31
34
  _custom_registered_path: str | None = None
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scanlayer
3
- Version: 1.0.2
3
+ Version: 1.1.0
4
4
  Summary: Image (scan/photo) to searchable PDF, or raw OCR export, via Tesseract.
5
5
  Author: Hyacinthe-primus
6
6
  Author-email: Hyacinthe-primus <hyacintheatho91@gmail.com>
@@ -19,27 +19,35 @@ Requires-Dist: numpy>=1.24.0
19
19
  Requires-Dist: pdf2image>=1.17.0
20
20
  Provides-Extra: test
21
21
  Requires-Dist: pytest>=7.0; extra == "test"
22
+ Requires-Dist: pytest-cov>=4.1; extra == "test"
22
23
  Provides-Extra: yaml
23
24
  Requires-Dist: PyYAML>=6.0; extra == "yaml"
24
25
  Dynamic: license-file
25
26
 
26
27
  <div align="center">
27
- <img src="scanlayer.png" alt="scanlayer" width="110"/>
28
+ <img src="https://raw.githubusercontent.com/Hyacinthe-primus/scanlayer/main/scanlayer.png" alt="scanlayer" width="110"/>
28
29
  <h1>scanlayer</h1>
29
- <p><em>Scanned image or photographed document &rarr; searchable PDF, or raw OCR text/JSON/TSV/hOCR.</em></p>
30
+ <p><em> Turn scanned image or photographed document into searchable PDF, or raw OCR text/JSON/TSV/hOCR.</em></p>
30
31
  </div>
31
32
 
32
33
  **Turn a scanned image or photographed document into a searchable PDF**, or
33
34
  export the raw OCR result as plain text, JSON, TSV, or hOCR. Use it as a
34
35
  command-line tool or as a Python library; both are the same engine underneath.
35
36
 
36
- [![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue)](#)
37
+ [![PyPI version](https://badge.fury.io/py/scanlayer.svg)](https://badge.fury.io/py/scanlayer)
38
+ [![codecov](https://codecov.io/gh/Hyacinthe-primus/scanlayer/branch/main/graph/badge.svg)](https://codecov.io/gh/Hyacinthe-primus/scanlayer)
39
+ [![Python](https://img.shields.io/badge/python-3.9%2B-3776AB)](#)
37
40
  ![Platforms](https://img.shields.io/badge/platform-Windows%20%7C%20macOS%20%7C%20Linux-lightgrey)
38
41
  [![](https://img.shields.io/badge/-documentation-green)](https://Hyacinthe-primus.github.io/scanlayer/)
42
+ [![GitHub last commit](https://img.shields.io/github/last-commit/Hyacinthe-primus/scanlayer)](https://github.com/Hyacinthe-primus/scanlayer)
39
43
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE.md)
40
44
 
41
45
  ---
42
46
 
47
+ ## Documentation
48
+
49
+ [Full documentation](https://Hyacinthe-primus.github.io/scanlayer/) – installation, CLI & API reference, configuration, output formats, examples, and troubleshooting.
50
+
43
51
  ## What it does
44
52
 
45
53
  A scanned invoice, a phone photo of a letter, a stack of photographed pages:
@@ -172,28 +180,6 @@ scanlayer.convert_merge(["page1.jpg", "page2.jpg", "page3.jpg"], "report.pdf")
172
180
  scanlayer.convert("invoice.jpg", "invoice.json", output_format="json")
173
181
  ```
174
182
 
175
- ## Documentation
176
-
177
- | | |
178
- |---|---|
179
- | **[Installation](https://Hyacinthe-primus.github.io/scanlayer/installation.html)** | Tesseract, poppler, and the two ways to install scanlayer itself |
180
- | **[Examples](https://Hyacinthe-primus.github.io/scanlayer/examples.html)** | Every feature, CLI and library side by side |
181
- | **[CLI Reference](https://Hyacinthe-primus.github.io/scanlayer/cli-reference.html)** | Every flag and exit code |
182
- | **[Library API](https://Hyacinthe-primus.github.io/scanlayer/library-api.html)** | `convert()`, `convert_batch()`, `convert_merge()`, exceptions, and the full low-level pipeline API |
183
- | **[Feature Catalog](https://Hyacinthe-primus.github.io/scanlayer/features.html)** | Everything scanlayer does, by pipeline stage |
184
- | **[Configuration](https://Hyacinthe-primus.github.io/scanlayer/configuration.html)** | `configure()`, config files, precedence, every tunable |
185
- | **[Output Formats](https://Hyacinthe-primus.github.io/scanlayer/output-formats.html)** | The `pdf`/`txt`/`json`/`tsv`/`hocr` schemas |
186
- | **[Multi-Page & Merge](https://Hyacinthe-primus.github.io/scanlayer/multi-page-merge.html)** | Batching, merging, native PDF input |
187
- | **[Debug Visualization](https://Hyacinthe-primus.github.io/scanlayer/debug-visualization.html)** | Reading the `--debug-image` confidence overlay |
188
- | **[Bundling Tesseract](https://Hyacinthe-primus.github.io/scanlayer/bundling-tesseract.html)** | Shipping your own Tesseract binary |
189
- | **[Troubleshooting](https://Hyacinthe-primus.github.io/scanlayer/troubleshooting.html)** | Common errors and fixes |
190
- | **[Roadmap & Limitations](https://Hyacinthe-primus.github.io/scanlayer/roadmap.html)** | What's missing and what's deliberately out of scope |
191
-
192
- The site is published from the `gh-pages` branch at
193
- https://Hyacinthe-primus.github.io/scanlayer/. To browse it locally, open
194
- `index.html` in a `gh-pages` worktree (e.g. `git worktree add <path> gh-pages`);
195
- it has no build step or server-side dependency.
196
-
197
183
  ## Repository layout
198
184
 
199
185
  ```
@@ -226,8 +212,20 @@ Bug reports, fixes, and feature discussions are welcome: see
226
212
  [CONTRIBUTING.md](CONTRIBUTING.md) for how to set up a development
227
213
  environment and what to include in a pull request. There is a `tests/`
228
214
  directory (now including `utils/validators.py` and the `--dry-run` flag, with
229
- cross-platform coverage for Tesseract discovery on Windows/macOS/Linux) and it runs with `pytest`, but no CI yet; see [Roadmap & Limitations](https://Hyacinthe-primus.github.io/scanlayer/roadmap.html) and the [Adding tests](CONTRIBUTING.md#adding-tests) section of the contributing guide for where coverage is thinnest.
215
+ cross-platform coverage for Tesseract discovery on Windows/macOS/Linux) and it runs with `pytest` automatically in CI on every push and pull request; see [Roadmap & Limitations](https://Hyacinthe-primus.github.io/scanlayer/roadmap.html) and the [Adding tests](CONTRIBUTING.md#adding-tests) section of the contributing guide for where coverage is thinnest.
216
+
217
+ ## Support the project
218
+
219
+ scanlayer is free and MIT-licensed, maintained in spare time. If it saves you
220
+ time, there is no fee – but a little support goes a long way:
221
+
222
+ - **Star the repository** – the simplest signal that the project is worth
223
+ maintaining.
224
+ - **Report bugs or request features** by opening a
225
+ [GitHub Issue](https://github.com/Hyacinthe-primus/scanlayer/issues).
226
+ - **Contribute** a fix, test, or example – see
227
+ [CONTRIBUTING.md](CONTRIBUTING.md).
230
228
 
231
229
  ## License
232
230
 
233
- [MIT](LICENSE.md).
231
+ This project is licensed under the [MIT License](LICENSE.md).
@@ -37,5 +37,6 @@ tests/test_dry_run.py
37
37
  tests/test_export.py
38
38
  tests/test_fonts.py
39
39
  tests/test_main_integration.py
40
+ tests/test_merge_workers.py
40
41
  tests/test_pdf_builder.py
41
42
  tests/test_validators.py
@@ -7,6 +7,7 @@ pdf2image>=1.17.0
7
7
 
8
8
  [test]
9
9
  pytest>=7.0
10
+ pytest-cov>=4.1
10
11
 
11
12
  [yaml]
12
13
  PyYAML>=6.0
@@ -0,0 +1,316 @@
1
+ """Tests for processing several pages of a --merge run at once.
2
+
3
+ Everything here fakes the per-page work, so no Tesseract is needed: the
4
+ point is the scheduling (order, real concurrency, the worker cap, error
5
+ reporting, the shared OCR slot limit), not the OCR itself.
6
+ """
7
+
8
+ import os
9
+ import threading
10
+ import time
11
+ from concurrent.futures import ThreadPoolExecutor
12
+
13
+ import numpy as np
14
+ import pytest
15
+
16
+ from scanlayer import config
17
+ from scanlayer import main as main_module
18
+ from scanlayer.cli import main
19
+ from scanlayer.cli.dry_run import EXIT_OK, EXIT_USER_ERROR
20
+ from scanlayer.ocr import engine
21
+ from scanlayer.utils.errors import BlankPageDetectedError, OCRProcessingError
22
+
23
+ STATS = {"words": 1, "mean_conf": 90.0, "best_psm": 3, "columns": 1}
24
+
25
+
26
+ class FakePages:
27
+ """Stands in for _process_page_for_merge and records how it was called."""
28
+
29
+ def __init__(self, sleep=0.0, delays=None, errors=None):
30
+ self.sleep = sleep
31
+ self.delays = delays or {}
32
+ self.errors = errors or {}
33
+ self.lock = threading.Lock()
34
+ self.running = 0
35
+ self.max_running = 0
36
+ self.started = []
37
+ self.thread_ids = set()
38
+ self.slots = []
39
+
40
+ def __call__(
41
+ self, input_path, lang, dpi, whitelist, blacklist, orientation, force,
42
+ ocr_slots=None,
43
+ ):
44
+ with self.lock:
45
+ self.started.append(input_path)
46
+ self.running += 1
47
+ self.max_running = max(self.max_running, self.running)
48
+ self.thread_ids.add(threading.get_ident())
49
+ self.slots.append(ocr_slots)
50
+ try:
51
+ time.sleep(self.delays.get(input_path, self.sleep))
52
+ if input_path in self.errors:
53
+ raise self.errors[input_path]
54
+ return f"page:{input_path}", dict(STATS)
55
+ finally:
56
+ with self.lock:
57
+ self.running -= 1
58
+
59
+
60
+ @pytest.fixture
61
+ def merge_env(monkeypatch, tmp_path):
62
+ """Fake page processing + PDF writer, 8 CPUs, returns (run, built)."""
63
+ built = []
64
+
65
+ def fake_build(pages, output_path, jpeg_quality=None, metadata=None):
66
+ built.append(list(pages))
67
+ return 123
68
+
69
+ monkeypatch.setattr(main_module, "build_searchable_pdf_multipage", fake_build)
70
+ monkeypatch.setattr(os, "cpu_count", lambda: 8)
71
+
72
+ def run(fake, count, **kwargs):
73
+ monkeypatch.setattr(main_module, "_process_page_for_merge", fake)
74
+ paths = [f"p{i}" for i in range(count)]
75
+ return main_module.convert_merge(paths, str(tmp_path / "out.pdf"), **kwargs)
76
+
77
+ return run, built
78
+
79
+
80
+ def test_pages_keep_input_order_when_they_finish_out_of_order(merge_env):
81
+ run, built = merge_env
82
+ fake = FakePages(delays={"p0": 0.15, "p1": 0.1, "p2": 0.05})
83
+ run(fake, 6, workers=3)
84
+ assert built == [[f"page:p{i}" for i in range(6)]]
85
+
86
+
87
+ def test_pages_really_run_concurrently(merge_env):
88
+ run, _ = merge_env
89
+ fake = FakePages(sleep=0.1)
90
+ run(fake, 6, workers=3)
91
+ assert fake.max_running == 3
92
+
93
+
94
+ def test_workers_above_the_limit_are_capped(merge_env):
95
+ run, _ = merge_env
96
+ fake = FakePages(sleep=0.05)
97
+ run(fake, 10, workers=10)
98
+ assert fake.max_running == config.MERGE_WORKERS_LIMIT == 4
99
+
100
+
101
+ def test_one_worker_runs_on_the_calling_thread_without_slots(merge_env):
102
+ run, built = merge_env
103
+ fake = FakePages(sleep=0.01)
104
+ run(fake, 4, workers=1)
105
+ assert fake.max_running == 1
106
+ assert fake.thread_ids == {threading.get_ident()}
107
+ assert fake.slots == [None] * 4
108
+ assert built == [[f"page:p{i}" for i in range(4)]]
109
+
110
+
111
+ def test_default_comes_from_configure(merge_env):
112
+ run, _ = merge_env
113
+ config.configure(merge_workers=2)
114
+ fake = FakePages(sleep=0.1)
115
+ run(fake, 6)
116
+ assert fake.max_running == 2
117
+
118
+
119
+ def test_default_is_three(merge_env):
120
+ run, _ = merge_env
121
+ fake = FakePages(sleep=0.1)
122
+ run(fake, 6)
123
+ assert fake.max_running == 3
124
+
125
+
126
+ def test_workers_never_exceed_the_cpu_count(merge_env, monkeypatch):
127
+ run, _ = merge_env
128
+ monkeypatch.setattr(os, "cpu_count", lambda: 2)
129
+ fake = FakePages(sleep=0.1)
130
+ run(fake, 6, workers=4)
131
+ assert fake.max_running == 2
132
+
133
+
134
+ def test_workers_never_exceed_the_page_count(merge_env):
135
+ run, _ = merge_env
136
+ fake = FakePages(sleep=0.05)
137
+ run(fake, 2, workers=4)
138
+ assert fake.max_running == 2
139
+
140
+
141
+ @pytest.mark.parametrize("bad", [0, -1, 1.5, True, "3"])
142
+ def test_invalid_workers_rejected_before_any_work(merge_env, bad):
143
+ run, built = merge_env
144
+ fake = FakePages()
145
+ with pytest.raises(ValueError, match="workers"):
146
+ run(fake, 3, workers=bad)
147
+ assert fake.started == []
148
+ assert built == []
149
+
150
+
151
+ def test_first_bad_page_in_input_order_is_the_one_reported(merge_env):
152
+ run, built = merge_env
153
+ fake = FakePages(
154
+ delays={"p0": 0.2, "p1": 0.1},
155
+ errors={
156
+ "p1": BlankPageDetectedError("p1 is blank"),
157
+ "p2": BlankPageDetectedError("p2 is blank"),
158
+ },
159
+ )
160
+ with pytest.raises(BlankPageDetectedError, match="p1 is blank"):
161
+ run(fake, 4, workers=3)
162
+ assert built == []
163
+
164
+
165
+ def test_a_failure_drops_pages_that_have_not_started(merge_env):
166
+ run, built = merge_env
167
+ fake = FakePages(
168
+ sleep=0.05, delays={"p0": 0.0}, errors={"p0": OCRProcessingError("boom")},
169
+ )
170
+ with pytest.raises(OCRProcessingError, match="boom"):
171
+ run(fake, 20, workers=2)
172
+ assert len(fake.started) < 20
173
+ assert built == []
174
+
175
+
176
+ def test_concurrent_pages_share_one_slot_pool_sized_to_the_cpu_count(merge_env):
177
+ run, _ = merge_env
178
+ fake = FakePages(sleep=0.02)
179
+ run(fake, 5, workers=3)
180
+ slots = fake.slots[0]
181
+ assert slots is not None and all(s is slots for s in fake.slots)
182
+ free = sum(1 for _ in range(20) if slots.acquire(blocking=False))
183
+ assert free == 8
184
+
185
+
186
+ def test_convert_batch_passes_workers_to_merge(monkeypatch, tmp_path, text_image):
187
+ seen = {}
188
+
189
+ def fake_merge(**kwargs):
190
+ seen.update(kwargs)
191
+ return "result"
192
+
193
+ monkeypatch.setattr(main_module, "convert_merge", fake_merge)
194
+ batch = main_module.convert_batch(
195
+ [text_image, text_image], str(tmp_path / "out.pdf"), merge=True, workers=2,
196
+ )
197
+ assert seen["workers"] == 2
198
+ assert batch.results == ["result"]
199
+
200
+
201
+ def _fake_psm_runner(calls):
202
+ def fake(*args, psm, ocr_slots=None, **kwargs):
203
+ calls.append((psm, ocr_slots))
204
+ return engine.PsmAttempt(psm=psm, words=[], mean_confidence=0.0, elapsed_ms=0.0), None
205
+
206
+ return fake
207
+
208
+
209
+ @pytest.mark.parametrize(
210
+ "settings",
211
+ [
212
+ {"psm_candidates": [3]},
213
+ {"psm_candidates": [3, 4, 6, 11], "psm_parallel": True},
214
+ {"psm_candidates": [3, 4, 6, 11], "psm_parallel": False},
215
+ ],
216
+ )
217
+ def test_every_psm_pass_gets_the_shared_slots(monkeypatch, settings):
218
+ calls = []
219
+ monkeypatch.setattr(engine, "_safe_run_ocr", _fake_psm_runner(calls))
220
+ config.configure(**settings)
221
+ slots = threading.BoundedSemaphore(2)
222
+ engine.extract_words(
223
+ np.zeros((10, 10), dtype=np.uint8), 1.0, 300, ocr_slots=slots,
224
+ )
225
+ assert len(calls) == len(settings["psm_candidates"])
226
+ assert all(s is slots for _, s in calls)
227
+
228
+
229
+ def _racing_psm_runner(confidence, delays):
230
+ def fake(*args, psm, ocr_slots=None, **kwargs):
231
+ time.sleep(delays.get(psm, 0.0))
232
+ attempt = engine.PsmAttempt(
233
+ psm=psm, words=[], mean_confidence=confidence[psm], elapsed_ms=0.0,
234
+ )
235
+ return attempt, None
236
+
237
+ return fake
238
+
239
+
240
+ def test_psm_tie_goes_to_the_first_candidate_even_if_it_finishes_last(monkeypatch):
241
+ monkeypatch.setattr(
242
+ engine,
243
+ "_safe_run_ocr",
244
+ _racing_psm_runner({3: 50.0, 4: 50.0, 6: 50.0, 11: 50.0}, {3: 0.15}),
245
+ )
246
+ config.configure(psm_candidates=[3, 4, 6, 11], psm_parallel=True)
247
+ result = engine.extract_words(np.zeros((10, 10), dtype=np.uint8), 1.0, 300)
248
+ assert result.best_psm == 3
249
+
250
+
251
+ def test_psm_higher_confidence_still_beats_an_earlier_candidate(monkeypatch):
252
+ monkeypatch.setattr(
253
+ engine,
254
+ "_safe_run_ocr",
255
+ _racing_psm_runner({3: 40.0, 4: 40.0, 6: 60.0, 11: 40.0}, {6: 0.15}),
256
+ )
257
+ config.configure(psm_candidates=[3, 4, 6, 11], psm_parallel=True)
258
+ result = engine.extract_words(np.zeros((10, 10), dtype=np.uint8), 1.0, 300)
259
+ assert result.best_psm == 6
260
+
261
+
262
+ def test_slots_bound_concurrent_tesseract_passes_and_are_released(monkeypatch):
263
+ lock = threading.Lock()
264
+ state = {"running": 0, "max": 0}
265
+
266
+ def fake_run_ocr(*args, **kwargs):
267
+ with lock:
268
+ state["running"] += 1
269
+ state["max"] = max(state["max"], state["running"])
270
+ time.sleep(0.03)
271
+ with lock:
272
+ state["running"] -= 1
273
+ if kwargs.get("fail"):
274
+ raise OCRProcessingError("timeout")
275
+ return engine.PsmAttempt(psm=3, words=[], mean_confidence=0.0, elapsed_ms=0.0)
276
+
277
+ monkeypatch.setattr(engine, "_run_ocr", fake_run_ocr)
278
+ slots = threading.BoundedSemaphore(2)
279
+
280
+ def one(_):
281
+ return engine._safe_run_ocr(None, "eng", 3, "", 300, None, None, psm=3, ocr_slots=slots)
282
+
283
+ with ThreadPoolExecutor(max_workers=6) as pool:
284
+ list(pool.map(one, range(6)))
285
+ assert state["max"] == 2
286
+
287
+ failing = engine._safe_run_ocr(
288
+ None, "eng", 3, "", 300, None, None, psm=3, ocr_slots=slots, fail=True,
289
+ )
290
+ assert failing[0] is None
291
+ assert sum(1 for _ in range(5) if slots.acquire(blocking=False)) == 2
292
+
293
+
294
+ def test_workers_flag_parses():
295
+ from scanlayer.cli.parser import _build_parser
296
+
297
+ args = _build_parser().parse_args(["a.jpg", "-o", "x.pdf", "--merge", "--workers", "3"])
298
+ assert args.workers == 3
299
+ assert _build_parser().parse_args(["a.jpg"]).workers is None
300
+
301
+
302
+ def test_cli_rejects_zero_workers(text_image, tmp_path):
303
+ out = tmp_path / "merged.pdf"
304
+ code = main([text_image, "-o", str(out), "--merge", "--workers", "0"])
305
+ assert code == EXIT_USER_ERROR
306
+
307
+
308
+ def test_cli_passes_workers_to_convert_merge(monkeypatch, text_image, tmp_path):
309
+ import scanlayer.cli.run as run_module
310
+
311
+ seen = {}
312
+ monkeypatch.setattr(run_module, "convert_merge", lambda **kw: seen.update(kw))
313
+ out = tmp_path / "merged.pdf"
314
+ code = main([text_image, text_image, "-o", str(out), "--merge", "--workers", "2"])
315
+ assert code == EXIT_OK
316
+ assert seen["workers"] == 2
File without changes
File without changes
File without changes