scanlayer 1.0.2__tar.gz → 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scanlayer-1.0.2/scanlayer.egg-info → scanlayer-1.1.0}/PKG-INFO +26 -28
- {scanlayer-1.0.2 → scanlayer-1.1.0}/README.md +24 -27
- {scanlayer-1.0.2 → scanlayer-1.1.0}/pyproject.toml +2 -2
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/cli/parser.py +8 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/cli/run.py +8 -1
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/config.py +7 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/main.py +101 -11
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/ocr/engine.py +45 -7
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/pdf/fonts.py +4 -1
- {scanlayer-1.0.2 → scanlayer-1.1.0/scanlayer.egg-info}/PKG-INFO +26 -28
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer.egg-info/SOURCES.txt +1 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer.egg-info/requires.txt +1 -0
- scanlayer-1.1.0/tests/test_merge_workers.py +316 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/LICENSE.md +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/__init__.py +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/__main__.py +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/cli/__init__.py +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/cli/dry_run.py +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/fonts/DejaVuSans.ttf +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/layout/__init__.py +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/layout/columns.py +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/ocr/__init__.py +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/ocr/export.py +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/pdf/__init__.py +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/pdf/builder.py +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/preprocessing/__init__.py +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/preprocessing/enhance.py +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/utils/__init__.py +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/utils/debug_image.py +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/utils/errors.py +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/utils/logger.py +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer/utils/validators.py +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer.egg-info/dependency_links.txt +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer.egg-info/entry_points.txt +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/scanlayer.egg-info/top_level.txt +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/setup.cfg +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/tests/test_columns.py +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/tests/test_config.py +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/tests/test_dry_run.py +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/tests/test_export.py +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/tests/test_fonts.py +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/tests/test_main_integration.py +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/tests/test_pdf_builder.py +0 -0
- {scanlayer-1.0.2 → scanlayer-1.1.0}/tests/test_validators.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scanlayer
|
|
3
|
-
Version: 1.0
|
|
3
|
+
Version: 1.1.0
|
|
4
4
|
Summary: Image (scan/photo) to searchable PDF, or raw OCR export, via Tesseract.
|
|
5
5
|
Author: Hyacinthe-primus
|
|
6
6
|
Author-email: Hyacinthe-primus <hyacintheatho91@gmail.com>
|
|
@@ -19,27 +19,35 @@ Requires-Dist: numpy>=1.24.0
|
|
|
19
19
|
Requires-Dist: pdf2image>=1.17.0
|
|
20
20
|
Provides-Extra: test
|
|
21
21
|
Requires-Dist: pytest>=7.0; extra == "test"
|
|
22
|
+
Requires-Dist: pytest-cov>=4.1; extra == "test"
|
|
22
23
|
Provides-Extra: yaml
|
|
23
24
|
Requires-Dist: PyYAML>=6.0; extra == "yaml"
|
|
24
25
|
Dynamic: license-file
|
|
25
26
|
|
|
26
27
|
<div align="center">
|
|
27
|
-
<img src="scanlayer.png" alt="scanlayer" width="110"/>
|
|
28
|
+
<img src="https://raw.githubusercontent.com/Hyacinthe-primus/scanlayer/main/scanlayer.png" alt="scanlayer" width="110"/>
|
|
28
29
|
<h1>scanlayer</h1>
|
|
29
|
-
<p><em>
|
|
30
|
+
<p><em> Turn scanned image or photographed document into searchable PDF, or raw OCR text/JSON/TSV/hOCR.</em></p>
|
|
30
31
|
</div>
|
|
31
32
|
|
|
32
33
|
**Turn a scanned image or photographed document into a searchable PDF**, or
|
|
33
34
|
export the raw OCR result as plain text, JSON, TSV, or hOCR. Use it as a
|
|
34
35
|
command-line tool or as a Python library; both are the same engine underneath.
|
|
35
36
|
|
|
36
|
-
[](https://badge.fury.io/py/scanlayer)
|
|
38
|
+
[](https://codecov.io/gh/Hyacinthe-primus/scanlayer)
|
|
39
|
+
[](#)
|
|
37
40
|

|
|
38
41
|
[](https://Hyacinthe-primus.github.io/scanlayer/)
|
|
42
|
+
[](https://github.com/Hyacinthe-primus/scanlayer)
|
|
39
43
|
[](LICENSE.md)
|
|
40
44
|
|
|
41
45
|
---
|
|
42
46
|
|
|
47
|
+
## Documentation
|
|
48
|
+
|
|
49
|
+
[Full documentation](https://Hyacinthe-primus.github.io/scanlayer/) – installation, CLI & API reference, configuration, output formats, examples, and troubleshooting.
|
|
50
|
+
|
|
43
51
|
## What it does
|
|
44
52
|
|
|
45
53
|
A scanned invoice, a phone photo of a letter, a stack of photographed pages:
|
|
@@ -172,28 +180,6 @@ scanlayer.convert_merge(["page1.jpg", "page2.jpg", "page3.jpg"], "report.pdf")
|
|
|
172
180
|
scanlayer.convert("invoice.jpg", "invoice.json", output_format="json")
|
|
173
181
|
```
|
|
174
182
|
|
|
175
|
-
## Documentation
|
|
176
|
-
|
|
177
|
-
| | |
|
|
178
|
-
|---|---|
|
|
179
|
-
| **[Installation](https://Hyacinthe-primus.github.io/scanlayer/installation.html)** | Tesseract, poppler, and the two ways to install scanlayer itself |
|
|
180
|
-
| **[Examples](https://Hyacinthe-primus.github.io/scanlayer/examples.html)** | Every feature, CLI and library side by side |
|
|
181
|
-
| **[CLI Reference](https://Hyacinthe-primus.github.io/scanlayer/cli-reference.html)** | Every flag and exit code |
|
|
182
|
-
| **[Library API](https://Hyacinthe-primus.github.io/scanlayer/library-api.html)** | `convert()`, `convert_batch()`, `convert_merge()`, exceptions, and the full low-level pipeline API |
|
|
183
|
-
| **[Feature Catalog](https://Hyacinthe-primus.github.io/scanlayer/features.html)** | Everything scanlayer does, by pipeline stage |
|
|
184
|
-
| **[Configuration](https://Hyacinthe-primus.github.io/scanlayer/configuration.html)** | `configure()`, config files, precedence, every tunable |
|
|
185
|
-
| **[Output Formats](https://Hyacinthe-primus.github.io/scanlayer/output-formats.html)** | The `pdf`/`txt`/`json`/`tsv`/`hocr` schemas |
|
|
186
|
-
| **[Multi-Page & Merge](https://Hyacinthe-primus.github.io/scanlayer/multi-page-merge.html)** | Batching, merging, native PDF input |
|
|
187
|
-
| **[Debug Visualization](https://Hyacinthe-primus.github.io/scanlayer/debug-visualization.html)** | Reading the `--debug-image` confidence overlay |
|
|
188
|
-
| **[Bundling Tesseract](https://Hyacinthe-primus.github.io/scanlayer/bundling-tesseract.html)** | Shipping your own Tesseract binary |
|
|
189
|
-
| **[Troubleshooting](https://Hyacinthe-primus.github.io/scanlayer/troubleshooting.html)** | Common errors and fixes |
|
|
190
|
-
| **[Roadmap & Limitations](https://Hyacinthe-primus.github.io/scanlayer/roadmap.html)** | What's missing and what's deliberately out of scope |
|
|
191
|
-
|
|
192
|
-
The site is published from the `gh-pages` branch at
|
|
193
|
-
https://Hyacinthe-primus.github.io/scanlayer/. To browse it locally, open
|
|
194
|
-
`index.html` in a `gh-pages` worktree (e.g. `git worktree add <path> gh-pages`);
|
|
195
|
-
it has no build step or server-side dependency.
|
|
196
|
-
|
|
197
183
|
## Repository layout
|
|
198
184
|
|
|
199
185
|
```
|
|
@@ -226,8 +212,20 @@ Bug reports, fixes, and feature discussions are welcome: see
|
|
|
226
212
|
[CONTRIBUTING.md](CONTRIBUTING.md) for how to set up a development
|
|
227
213
|
environment and what to include in a pull request. There is a `tests/`
|
|
228
214
|
directory (now including `utils/validators.py` and the `--dry-run` flag, with
|
|
229
|
-
cross-platform coverage for Tesseract discovery on Windows/macOS/Linux) and it runs with `pytest
|
|
215
|
+
cross-platform coverage for Tesseract discovery on Windows/macOS/Linux) and it runs with `pytest` automatically in CI on every push and pull request; see [Roadmap & Limitations](https://Hyacinthe-primus.github.io/scanlayer/roadmap.html) and the [Adding tests](CONTRIBUTING.md#adding-tests) section of the contributing guide for where coverage is thinnest.
|
|
216
|
+
|
|
217
|
+
## Support the project
|
|
218
|
+
|
|
219
|
+
scanlayer is free and MIT-licensed, maintained in spare time. If it saves you
|
|
220
|
+
time, there is no fee – but a little support goes a long way:
|
|
221
|
+
|
|
222
|
+
- **Star the repository** – the simplest signal that the project is worth
|
|
223
|
+
maintaining.
|
|
224
|
+
- **Report bugs or request features** by opening a
|
|
225
|
+
[GitHub Issue](https://github.com/Hyacinthe-primus/scanlayer/issues).
|
|
226
|
+
- **Contribute** a fix, test, or example – see
|
|
227
|
+
[CONTRIBUTING.md](CONTRIBUTING.md).
|
|
230
228
|
|
|
231
229
|
## License
|
|
232
230
|
|
|
233
|
-
[MIT](LICENSE.md).
|
|
231
|
+
This project is licensed under the [MIT License](LICENSE.md).
|
|
@@ -1,20 +1,27 @@
|
|
|
1
1
|
<div align="center">
|
|
2
|
-
<img src="scanlayer.png" alt="scanlayer" width="110"/>
|
|
2
|
+
<img src="https://raw.githubusercontent.com/Hyacinthe-primus/scanlayer/main/scanlayer.png" alt="scanlayer" width="110"/>
|
|
3
3
|
<h1>scanlayer</h1>
|
|
4
|
-
<p><em>
|
|
4
|
+
<p><em> Turn scanned image or photographed document into searchable PDF, or raw OCR text/JSON/TSV/hOCR.</em></p>
|
|
5
5
|
</div>
|
|
6
6
|
|
|
7
7
|
**Turn a scanned image or photographed document into a searchable PDF**, or
|
|
8
8
|
export the raw OCR result as plain text, JSON, TSV, or hOCR. Use it as a
|
|
9
9
|
command-line tool or as a Python library; both are the same engine underneath.
|
|
10
10
|
|
|
11
|
-
[](https://badge.fury.io/py/scanlayer)
|
|
12
|
+
[](https://codecov.io/gh/Hyacinthe-primus/scanlayer)
|
|
13
|
+
[](#)
|
|
12
14
|

|
|
13
15
|
[](https://Hyacinthe-primus.github.io/scanlayer/)
|
|
16
|
+
[](https://github.com/Hyacinthe-primus/scanlayer)
|
|
14
17
|
[](LICENSE.md)
|
|
15
18
|
|
|
16
19
|
---
|
|
17
20
|
|
|
21
|
+
## Documentation
|
|
22
|
+
|
|
23
|
+
[Full documentation](https://Hyacinthe-primus.github.io/scanlayer/) – installation, CLI & API reference, configuration, output formats, examples, and troubleshooting.
|
|
24
|
+
|
|
18
25
|
## What it does
|
|
19
26
|
|
|
20
27
|
A scanned invoice, a phone photo of a letter, a stack of photographed pages:
|
|
@@ -147,28 +154,6 @@ scanlayer.convert_merge(["page1.jpg", "page2.jpg", "page3.jpg"], "report.pdf")
|
|
|
147
154
|
scanlayer.convert("invoice.jpg", "invoice.json", output_format="json")
|
|
148
155
|
```
|
|
149
156
|
|
|
150
|
-
## Documentation
|
|
151
|
-
|
|
152
|
-
| | |
|
|
153
|
-
|---|---|
|
|
154
|
-
| **[Installation](https://Hyacinthe-primus.github.io/scanlayer/installation.html)** | Tesseract, poppler, and the two ways to install scanlayer itself |
|
|
155
|
-
| **[Examples](https://Hyacinthe-primus.github.io/scanlayer/examples.html)** | Every feature, CLI and library side by side |
|
|
156
|
-
| **[CLI Reference](https://Hyacinthe-primus.github.io/scanlayer/cli-reference.html)** | Every flag and exit code |
|
|
157
|
-
| **[Library API](https://Hyacinthe-primus.github.io/scanlayer/library-api.html)** | `convert()`, `convert_batch()`, `convert_merge()`, exceptions, and the full low-level pipeline API |
|
|
158
|
-
| **[Feature Catalog](https://Hyacinthe-primus.github.io/scanlayer/features.html)** | Everything scanlayer does, by pipeline stage |
|
|
159
|
-
| **[Configuration](https://Hyacinthe-primus.github.io/scanlayer/configuration.html)** | `configure()`, config files, precedence, every tunable |
|
|
160
|
-
| **[Output Formats](https://Hyacinthe-primus.github.io/scanlayer/output-formats.html)** | The `pdf`/`txt`/`json`/`tsv`/`hocr` schemas |
|
|
161
|
-
| **[Multi-Page & Merge](https://Hyacinthe-primus.github.io/scanlayer/multi-page-merge.html)** | Batching, merging, native PDF input |
|
|
162
|
-
| **[Debug Visualization](https://Hyacinthe-primus.github.io/scanlayer/debug-visualization.html)** | Reading the `--debug-image` confidence overlay |
|
|
163
|
-
| **[Bundling Tesseract](https://Hyacinthe-primus.github.io/scanlayer/bundling-tesseract.html)** | Shipping your own Tesseract binary |
|
|
164
|
-
| **[Troubleshooting](https://Hyacinthe-primus.github.io/scanlayer/troubleshooting.html)** | Common errors and fixes |
|
|
165
|
-
| **[Roadmap & Limitations](https://Hyacinthe-primus.github.io/scanlayer/roadmap.html)** | What's missing and what's deliberately out of scope |
|
|
166
|
-
|
|
167
|
-
The site is published from the `gh-pages` branch at
|
|
168
|
-
https://Hyacinthe-primus.github.io/scanlayer/. To browse it locally, open
|
|
169
|
-
`index.html` in a `gh-pages` worktree (e.g. `git worktree add <path> gh-pages`);
|
|
170
|
-
it has no build step or server-side dependency.
|
|
171
|
-
|
|
172
157
|
## Repository layout
|
|
173
158
|
|
|
174
159
|
```
|
|
@@ -201,8 +186,20 @@ Bug reports, fixes, and feature discussions are welcome: see
|
|
|
201
186
|
[CONTRIBUTING.md](CONTRIBUTING.md) for how to set up a development
|
|
202
187
|
environment and what to include in a pull request. There is a `tests/`
|
|
203
188
|
directory (now including `utils/validators.py` and the `--dry-run` flag, with
|
|
204
|
-
cross-platform coverage for Tesseract discovery on Windows/macOS/Linux) and it runs with `pytest
|
|
189
|
+
cross-platform coverage for Tesseract discovery on Windows/macOS/Linux) and it runs with `pytest` automatically in CI on every push and pull request; see [Roadmap & Limitations](https://Hyacinthe-primus.github.io/scanlayer/roadmap.html) and the [Adding tests](CONTRIBUTING.md#adding-tests) section of the contributing guide for where coverage is thinnest.
|
|
190
|
+
|
|
191
|
+
## Support the project
|
|
192
|
+
|
|
193
|
+
scanlayer is free and MIT-licensed, maintained in spare time. If it saves you
|
|
194
|
+
time, there is no fee – but a little support goes a long way:
|
|
195
|
+
|
|
196
|
+
- **Star the repository** – the simplest signal that the project is worth
|
|
197
|
+
maintaining.
|
|
198
|
+
- **Report bugs or request features** by opening a
|
|
199
|
+
[GitHub Issue](https://github.com/Hyacinthe-primus/scanlayer/issues).
|
|
200
|
+
- **Contribute** a fix, test, or example – see
|
|
201
|
+
[CONTRIBUTING.md](CONTRIBUTING.md).
|
|
205
202
|
|
|
206
203
|
## License
|
|
207
204
|
|
|
208
|
-
[MIT](LICENSE.md).
|
|
205
|
+
This project is licensed under the [MIT License](LICENSE.md).
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "scanlayer"
|
|
7
|
-
version = "1.0
|
|
7
|
+
version = "1.1.0"
|
|
8
8
|
description = "Image (scan/photo) to searchable PDF, or raw OCR export, via Tesseract."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.9"
|
|
@@ -23,7 +23,7 @@ dependencies = [
|
|
|
23
23
|
]
|
|
24
24
|
|
|
25
25
|
[project.optional-dependencies]
|
|
26
|
-
test = ["pytest>=7.0"]
|
|
26
|
+
test = ["pytest>=7.0", "pytest-cov>=4.1"]
|
|
27
27
|
yaml = ["PyYAML>=6.0"]
|
|
28
28
|
|
|
29
29
|
[project.scripts]
|
|
@@ -77,6 +77,14 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
77
77
|
"multi-page PDF input. Only with --format pdf. -o must "
|
|
78
78
|
"be a file path, not a folder.",
|
|
79
79
|
)
|
|
80
|
+
io_group.add_argument(
|
|
81
|
+
"--workers", type=int, default=None, metavar="N",
|
|
82
|
+
help="With --merge: process up to N pages at the same time "
|
|
83
|
+
f"(1-{config.MERGE_WORKERS_LIMIT}, default {config.MERGE_WORKERS}; "
|
|
84
|
+
"1 = one page at a time). Only changes speed: page order "
|
|
85
|
+
"and OCR output are the same. Ignored without --merge. "
|
|
86
|
+
"Equivalent to configure(merge_workers=N).",
|
|
87
|
+
)
|
|
80
88
|
io_group.add_argument(
|
|
81
89
|
"--config", default=None,
|
|
82
90
|
help="Path to a .yaml/.yml/.json config profile applied via "
|
|
@@ -21,7 +21,7 @@ from scanlayer.cli.dry_run import (
|
|
|
21
21
|
)
|
|
22
22
|
from scanlayer.cli.parser import _build_parser
|
|
23
23
|
from scanlayer.main import _expand_pdf_input, _resolve_output_path, convert, convert_merge
|
|
24
|
-
from scanlayer.utils.logger import get_logger, log_error
|
|
24
|
+
from scanlayer.utils.logger import get_logger, log_error, log_warning
|
|
25
25
|
|
|
26
26
|
|
|
27
27
|
def main(argv: list[str] = None) -> int:
|
|
@@ -112,6 +112,12 @@ def main(argv: list[str] = None) -> int:
|
|
|
112
112
|
if args.merge and (args.output.endswith(("/", "\\")) or os.path.isdir(args.output)):
|
|
113
113
|
log_error(log, "--merge requires -o/--output to be a file path, not a folder.")
|
|
114
114
|
return EXIT_USER_ERROR
|
|
115
|
+
if args.workers is not None:
|
|
116
|
+
if args.workers < 1:
|
|
117
|
+
log_error(log, f"--workers must be 1 or more. Got {args.workers}.")
|
|
118
|
+
return EXIT_USER_ERROR
|
|
119
|
+
if not args.merge:
|
|
120
|
+
log_warning(log, "--workers only applies with --merge, ignoring it.")
|
|
115
121
|
|
|
116
122
|
raw_inputs: list[str] = []
|
|
117
123
|
for pat in args.input:
|
|
@@ -147,6 +153,7 @@ def main(argv: list[str] = None) -> int:
|
|
|
147
153
|
pdf_metadata=pdf_metadata,
|
|
148
154
|
force=args.force,
|
|
149
155
|
orientation=orientation_value,
|
|
156
|
+
workers=args.workers,
|
|
150
157
|
)
|
|
151
158
|
return EXIT_OK
|
|
152
159
|
except Exception as exc:
|
|
@@ -89,6 +89,12 @@ PSM_EARLY_EXIT_CONFIDENCE = 80.0
|
|
|
89
89
|
PSM_PARALLEL = True
|
|
90
90
|
PSM_MAX_WORKERS = None
|
|
91
91
|
|
|
92
|
+
# Pages processed concurrently by convert_merge()/--merge. 1 = one page at
|
|
93
|
+
# a time. Each page already races several Tesseract processes, so this is
|
|
94
|
+
# kept small; MERGE_WORKERS_LIMIT is a hard ceiling that is not configurable.
|
|
95
|
+
MERGE_WORKERS = 3
|
|
96
|
+
MERGE_WORKERS_LIMIT = 4
|
|
97
|
+
|
|
92
98
|
OCR_TIMEOUT_SECONDS = 45
|
|
93
99
|
|
|
94
100
|
# PSM candidates: engine.py runs one pass per PSM, keeps highest mean confidence.
|
|
@@ -161,6 +167,7 @@ _CONFIGURABLE_KEYS = {
|
|
|
161
167
|
"psm_early_exit_confidence": "PSM_EARLY_EXIT_CONFIDENCE",
|
|
162
168
|
"psm_parallel": "PSM_PARALLEL",
|
|
163
169
|
"psm_max_workers": "PSM_MAX_WORKERS",
|
|
170
|
+
"merge_workers": "MERGE_WORKERS",
|
|
164
171
|
"ocr_timeout_seconds": "OCR_TIMEOUT_SECONDS",
|
|
165
172
|
"char_whitelist": "TESSERACT_CHAR_WHITELIST",
|
|
166
173
|
"char_blacklist": "TESSERACT_CHAR_BLACKLIST",
|
|
@@ -9,7 +9,9 @@ from __future__ import annotations
|
|
|
9
9
|
import glob
|
|
10
10
|
import os
|
|
11
11
|
import tempfile
|
|
12
|
+
import threading
|
|
12
13
|
import time
|
|
14
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
13
15
|
from dataclasses import dataclass
|
|
14
16
|
from typing import Optional
|
|
15
17
|
|
|
@@ -17,7 +19,7 @@ from PIL import Image
|
|
|
17
19
|
|
|
18
20
|
from scanlayer import config
|
|
19
21
|
from scanlayer.layout.columns import reorder_reading_order
|
|
20
|
-
from scanlayer.ocr.engine import extract_words
|
|
22
|
+
from scanlayer.ocr.engine import extract_words, use_configured_tesseract
|
|
21
23
|
from scanlayer.ocr.export import FORMATS, export_words
|
|
22
24
|
from scanlayer.pdf.builder import build_searchable_pdf, build_searchable_pdf_multipage
|
|
23
25
|
from scanlayer.preprocessing.enhance import preprocess
|
|
@@ -422,10 +424,12 @@ def _process_page_for_merge(
|
|
|
422
424
|
char_blacklist: Optional[str],
|
|
423
425
|
orientation: "str | float | None",
|
|
424
426
|
force: bool,
|
|
427
|
+
ocr_slots: Optional[threading.Semaphore] = None,
|
|
425
428
|
):
|
|
426
429
|
"""Run preprocessing + OCR for one page of a --merge run.
|
|
427
430
|
|
|
428
431
|
Returns (PageInput, stats_dict). Does not touch the output path.
|
|
432
|
+
Safe to call from several threads at once.
|
|
429
433
|
"""
|
|
430
434
|
from scanlayer.pdf.builder import PageInput
|
|
431
435
|
|
|
@@ -467,6 +471,7 @@ def _process_page_for_merge(
|
|
|
467
471
|
ocr_result = extract_words(
|
|
468
472
|
result.ocr_image, result.ocr_scale, effective_dpi=result.effective_dpi,
|
|
469
473
|
lang=lang, char_whitelist=char_whitelist, char_blacklist=char_blacklist,
|
|
474
|
+
ocr_slots=ocr_slots,
|
|
470
475
|
)
|
|
471
476
|
ocr_result.words, column_count = reorder_reading_order(
|
|
472
477
|
ocr_result.words, page_width=result.background.width,
|
|
@@ -487,6 +492,86 @@ def _process_page_for_merge(
|
|
|
487
492
|
return page, stats
|
|
488
493
|
|
|
489
494
|
|
|
495
|
+
def _resolve_merge_workers(workers: Optional[int], page_count: int) -> int:
|
|
496
|
+
"""How many pages a merge run processes at once (always >= 1).
|
|
497
|
+
|
|
498
|
+
None means config.MERGE_WORKERS. The result never exceeds
|
|
499
|
+
config.MERGE_WORKERS_LIMIT, the page count, or the CPU count.
|
|
500
|
+
"""
|
|
501
|
+
requested = config.MERGE_WORKERS if workers is None else workers
|
|
502
|
+
if isinstance(requested, bool) or not isinstance(requested, int) or requested < 1:
|
|
503
|
+
raise ValueError(f"workers must be an integer >= 1, got {requested!r}.")
|
|
504
|
+
|
|
505
|
+
limit = config.MERGE_WORKERS_LIMIT
|
|
506
|
+
if requested > limit:
|
|
507
|
+
log_warning(
|
|
508
|
+
get_logger("main"),
|
|
509
|
+
f"workers={requested} is above the maximum of {limit}, using {limit}.",
|
|
510
|
+
)
|
|
511
|
+
return max(1, min(requested, limit, page_count, os.cpu_count() or 1))
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
def _process_pages_for_merge(
|
|
515
|
+
input_paths: list[str],
|
|
516
|
+
workers: int,
|
|
517
|
+
lang: Optional[str],
|
|
518
|
+
dpi: Optional[int],
|
|
519
|
+
char_whitelist: Optional[str],
|
|
520
|
+
char_blacklist: Optional[str],
|
|
521
|
+
orientation: "str | float | None",
|
|
522
|
+
force: bool,
|
|
523
|
+
) -> list:
|
|
524
|
+
"""Run preprocessing + OCR for every page of a merge run.
|
|
525
|
+
|
|
526
|
+
Returns [(PageInput, stats), ...] in input order, whatever order the
|
|
527
|
+
pages finish in. If a page fails, the failure of the first bad page in
|
|
528
|
+
input order is raised and pages that have not started yet are dropped.
|
|
529
|
+
With workers == 1 pages are processed one at a time on the calling
|
|
530
|
+
thread, exactly as before.
|
|
531
|
+
"""
|
|
532
|
+
log = get_logger("main")
|
|
533
|
+
page_args = (lang, dpi, char_whitelist, char_blacklist, orientation, force)
|
|
534
|
+
use_configured_tesseract()
|
|
535
|
+
|
|
536
|
+
def _done(input_path: str, stats: dict) -> None:
|
|
537
|
+
log.info(
|
|
538
|
+
f"'{input_path}': {stats['words']} words "
|
|
539
|
+
f"(mean confidence {stats['mean_conf']:.1f}%"
|
|
540
|
+
+ (f", {stats['columns']} columns" if stats["columns"] > 1 else "")
|
|
541
|
+
+ ")"
|
|
542
|
+
)
|
|
543
|
+
|
|
544
|
+
results: list = []
|
|
545
|
+
if workers <= 1:
|
|
546
|
+
for input_path in input_paths:
|
|
547
|
+
page, stats = _process_page_for_merge(input_path, *page_args)
|
|
548
|
+
_done(input_path, stats)
|
|
549
|
+
results.append((page, stats))
|
|
550
|
+
return results
|
|
551
|
+
|
|
552
|
+
# Every page already races several Tesseract processes. This shared
|
|
553
|
+
# semaphore keeps the total at the CPU count, so concurrent pages cannot
|
|
554
|
+
# starve each other into the per-call OCR timeout.
|
|
555
|
+
ocr_slots = threading.BoundedSemaphore(os.cpu_count() or 1)
|
|
556
|
+
log.info(f"Processing {len(input_paths)} page(s) with {workers} parallel workers.")
|
|
557
|
+
|
|
558
|
+
executor = ThreadPoolExecutor(max_workers=workers, thread_name_prefix="scanlayer-page")
|
|
559
|
+
try:
|
|
560
|
+
futures = [
|
|
561
|
+
executor.submit(
|
|
562
|
+
_process_page_for_merge, input_path, *page_args, ocr_slots=ocr_slots,
|
|
563
|
+
)
|
|
564
|
+
for input_path in input_paths
|
|
565
|
+
]
|
|
566
|
+
for input_path, future in zip(input_paths, futures):
|
|
567
|
+
page, stats = future.result()
|
|
568
|
+
_done(input_path, stats)
|
|
569
|
+
results.append((page, stats))
|
|
570
|
+
finally:
|
|
571
|
+
executor.shutdown(wait=True, cancel_futures=True)
|
|
572
|
+
return results
|
|
573
|
+
|
|
574
|
+
|
|
490
575
|
def convert_merge(
|
|
491
576
|
input_paths: list[str],
|
|
492
577
|
output_path: str,
|
|
@@ -498,15 +583,22 @@ def convert_merge(
|
|
|
498
583
|
pdf_metadata: Optional[dict] = None,
|
|
499
584
|
force: bool = False,
|
|
500
585
|
orientation: "str | float | None" = None,
|
|
586
|
+
workers: Optional[int] = None,
|
|
501
587
|
) -> ConversionResult:
|
|
502
588
|
"""Combine several images into one multi-page searchable PDF.
|
|
503
589
|
|
|
504
590
|
PDF-only. Raises BlankPageDetectedError if any page is blank and
|
|
505
591
|
force=False.
|
|
592
|
+
|
|
593
|
+
workers: pages processed concurrently, 1 to config.MERGE_WORKERS_LIMIT
|
|
594
|
+
(4). None uses config.MERGE_WORKERS (default 3). 1 processes one
|
|
595
|
+
page at a time. It only changes speed: page order and OCR output are
|
|
596
|
+
the same for any value.
|
|
506
597
|
"""
|
|
507
598
|
if not input_paths:
|
|
508
599
|
raise ValueError("convert_merge() called with no input paths.")
|
|
509
600
|
|
|
601
|
+
worker_count = _resolve_merge_workers(workers, len(input_paths))
|
|
510
602
|
|
|
511
603
|
log = get_logger("main")
|
|
512
604
|
t0 = time.perf_counter()
|
|
@@ -518,22 +610,16 @@ def convert_merge(
|
|
|
518
610
|
conf_weighted_sum = 0.0
|
|
519
611
|
conf_weight_n = 0
|
|
520
612
|
max_columns = 1
|
|
521
|
-
for
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
613
|
+
for page, stats in _process_pages_for_merge(
|
|
614
|
+
input_paths, worker_count, lang, dpi, char_whitelist, char_blacklist,
|
|
615
|
+
orientation, force,
|
|
616
|
+
):
|
|
525
617
|
pages.append(page)
|
|
526
618
|
total_words += stats["words"]
|
|
527
619
|
if stats["words"]:
|
|
528
620
|
conf_weighted_sum += stats["mean_conf"] * stats["words"]
|
|
529
621
|
conf_weight_n += stats["words"]
|
|
530
622
|
max_columns = max(max_columns, stats["columns"])
|
|
531
|
-
log.info(
|
|
532
|
-
f"'{input_path}': {stats['words']} words "
|
|
533
|
-
f"(mean confidence {stats['mean_conf']:.1f}%"
|
|
534
|
-
+ (f", {stats['columns']} columns" if stats["columns"] > 1 else "")
|
|
535
|
-
+ ")"
|
|
536
|
-
)
|
|
537
623
|
|
|
538
624
|
output_size_bytes = build_searchable_pdf_multipage(
|
|
539
625
|
pages, output_abs, jpeg_quality=jpeg_quality, metadata=pdf_metadata,
|
|
@@ -577,6 +663,7 @@ def convert_batch(
|
|
|
577
663
|
orientation: "str | float | None" = None,
|
|
578
664
|
output_format: str = "pdf",
|
|
579
665
|
debug_image: bool = False,
|
|
666
|
+
workers: Optional[int] = None,
|
|
580
667
|
verbose: bool = False,
|
|
581
668
|
quiet: bool = False,
|
|
582
669
|
) -> BatchResult:
|
|
@@ -588,6 +675,8 @@ def convert_batch(
|
|
|
588
675
|
(same directory/stem, extension from output_format). Not valid
|
|
589
676
|
with merge=True, which needs one explicit shared output path.
|
|
590
677
|
merge: combine all inputs into one multi-page PDF.
|
|
678
|
+
workers: with merge=True, pages processed concurrently (see
|
|
679
|
+
convert_merge()). Ignored without merge.
|
|
591
680
|
verbose/quiet: set log level for the call.
|
|
592
681
|
|
|
593
682
|
Never raises for per-file failures. Check BatchResult.failures.
|
|
@@ -636,6 +725,7 @@ def convert_batch(
|
|
|
636
725
|
lang=lang, dpi=dpi, jpeg_quality=jpeg_quality,
|
|
637
726
|
char_whitelist=char_whitelist, char_blacklist=char_blacklist,
|
|
638
727
|
pdf_metadata=pdf_metadata, force=force, orientation=orientation,
|
|
728
|
+
workers=workers,
|
|
639
729
|
))
|
|
640
730
|
except Exception as exc:
|
|
641
731
|
failures.append(("<merge>", str(exc)))
|
|
@@ -8,6 +8,7 @@ confidence. Supports early-exit and parallel execution via threads.
|
|
|
8
8
|
from __future__ import annotations
|
|
9
9
|
|
|
10
10
|
import os
|
|
11
|
+
import threading
|
|
11
12
|
import time
|
|
12
13
|
import unicodedata
|
|
13
14
|
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
|
|
@@ -168,8 +169,29 @@ def _run_ocr(
|
|
|
168
169
|
)
|
|
169
170
|
|
|
170
171
|
|
|
171
|
-
def
|
|
172
|
-
"""
|
|
172
|
+
def use_configured_tesseract() -> None:
|
|
173
|
+
"""Point pytesseract at config.TESSERACT_CMD.
|
|
174
|
+
|
|
175
|
+
pytesseract keeps the binary path in a module global, and
|
|
176
|
+
preprocessing's OSD call runs before the first extract_words(), so
|
|
177
|
+
callers that start concurrent work set it once up front.
|
|
178
|
+
"""
|
|
179
|
+
pytesseract.pytesseract.tesseract_cmd = config.TESSERACT_CMD
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _safe_run_ocr(
|
|
183
|
+
*args,
|
|
184
|
+
psm: int,
|
|
185
|
+
ocr_slots: Optional[threading.Semaphore] = None,
|
|
186
|
+
**kwargs,
|
|
187
|
+
) -> tuple[PsmAttempt | None, Exception | None]:
|
|
188
|
+
"""Run one PSM candidate, converting exceptions into return values.
|
|
189
|
+
|
|
190
|
+
With ocr_slots, waits for a free slot first, so the Tesseract timeout
|
|
191
|
+
only starts counting once the process actually runs.
|
|
192
|
+
"""
|
|
193
|
+
if ocr_slots is not None:
|
|
194
|
+
ocr_slots.acquire()
|
|
173
195
|
try:
|
|
174
196
|
return _run_ocr(*args, **kwargs), None
|
|
175
197
|
except pytesseract.TesseractNotFoundError as exc:
|
|
@@ -183,6 +205,9 @@ def _safe_run_ocr(*args, psm: int, **kwargs) -> tuple[PsmAttempt | None, Excepti
|
|
|
183
205
|
except Exception as exc: # genuinely unexpected, still surfaced, not swallowed
|
|
184
206
|
log_warning(log, f"PSM {psm}, unexpected error ({type(exc).__name__}: {exc})")
|
|
185
207
|
return None, exc
|
|
208
|
+
finally:
|
|
209
|
+
if ocr_slots is not None:
|
|
210
|
+
ocr_slots.release()
|
|
186
211
|
|
|
187
212
|
|
|
188
213
|
def extract_words(
|
|
@@ -192,16 +217,21 @@ def extract_words(
|
|
|
192
217
|
lang: Optional[str] = None,
|
|
193
218
|
char_whitelist: Optional[str] = None,
|
|
194
219
|
char_blacklist: Optional[str] = None,
|
|
220
|
+
ocr_slots: Optional[threading.Semaphore] = None,
|
|
195
221
|
) -> OcrResult:
|
|
196
222
|
"""Run OCR with automatic PSM selection.
|
|
197
223
|
|
|
224
|
+
ocr_slots, if given, is shared by concurrent callers (convert_merge()
|
|
225
|
+
processing several pages at once) and bounds how many Tesseract passes
|
|
226
|
+
run at the same time. It never changes which result is selected.
|
|
227
|
+
|
|
198
228
|
Raises OCRProcessingError if every PSM candidate fails.
|
|
199
229
|
"""
|
|
200
230
|
lang = lang or config.DEFAULT_OCR_LANG
|
|
201
231
|
wl = char_whitelist if char_whitelist is not None else config.TESSERACT_CHAR_WHITELIST
|
|
202
232
|
bl = char_blacklist if char_blacklist is not None else config.TESSERACT_CHAR_BLACKLIST
|
|
203
233
|
|
|
204
|
-
|
|
234
|
+
use_configured_tesseract()
|
|
205
235
|
tess_base = f'--tessdata-dir "{config.TESSDATA_DIR}"' if config.TESSDATA_DIR else ""
|
|
206
236
|
|
|
207
237
|
psm_candidates = list(config.TESSERACT_PSM_CANDIDATES)
|
|
@@ -224,7 +254,7 @@ def extract_words(
|
|
|
224
254
|
if len(psm_candidates) == 1:
|
|
225
255
|
attempt, error = _safe_run_ocr(
|
|
226
256
|
ocr_image, lang, psm_candidates[0], tess_base,
|
|
227
|
-
effective_dpi, wl, bl, psm=psm_candidates[0],
|
|
257
|
+
effective_dpi, wl, bl, psm=psm_candidates[0], ocr_slots=ocr_slots,
|
|
228
258
|
)
|
|
229
259
|
if attempt is not None:
|
|
230
260
|
attempts.append(attempt)
|
|
@@ -236,12 +266,13 @@ def extract_words(
|
|
|
236
266
|
max_workers = config.PSM_MAX_WORKERS or min(32, (os.cpu_count() or 1) + 4)
|
|
237
267
|
max_workers = min(max_workers, len(psm_candidates))
|
|
238
268
|
can_skip_unstarted = max_workers < len(psm_candidates)
|
|
269
|
+
order = {psm: i for i, psm in enumerate(psm_candidates)}
|
|
239
270
|
|
|
240
271
|
executor = ThreadPoolExecutor(max_workers=max_workers)
|
|
241
272
|
futures = {
|
|
242
273
|
executor.submit(
|
|
243
274
|
_safe_run_ocr, ocr_image, lang, psm, tess_base,
|
|
244
|
-
effective_dpi, wl, bl, psm=psm,
|
|
275
|
+
effective_dpi, wl, bl, psm=psm, ocr_slots=ocr_slots,
|
|
245
276
|
): psm
|
|
246
277
|
for psm in psm_candidates
|
|
247
278
|
}
|
|
@@ -253,7 +284,14 @@ def extract_words(
|
|
|
253
284
|
attempt, error = future.result()
|
|
254
285
|
if attempt is not None:
|
|
255
286
|
attempts.append(attempt)
|
|
256
|
-
if
|
|
287
|
+
if (
|
|
288
|
+
best is None
|
|
289
|
+
or attempt.mean_confidence > best.mean_confidence
|
|
290
|
+
or (
|
|
291
|
+
attempt.mean_confidence == best.mean_confidence
|
|
292
|
+
and order[attempt.psm] < order[best.psm]
|
|
293
|
+
)
|
|
294
|
+
):
|
|
257
295
|
best = attempt
|
|
258
296
|
else:
|
|
259
297
|
errors.append(error)
|
|
@@ -277,7 +315,7 @@ def extract_words(
|
|
|
277
315
|
for psm in psm_candidates:
|
|
278
316
|
attempt, error = _safe_run_ocr(
|
|
279
317
|
ocr_image, lang, psm, tess_base,
|
|
280
|
-
effective_dpi, wl, bl, psm=psm,
|
|
318
|
+
effective_dpi, wl, bl, psm=psm, ocr_slots=ocr_slots,
|
|
281
319
|
)
|
|
282
320
|
if attempt is not None:
|
|
283
321
|
attempts.append(attempt)
|
|
@@ -12,6 +12,7 @@ Set config.FONT_PATH to force a specific TTF and skip auto-selection.
|
|
|
12
12
|
from __future__ import annotations
|
|
13
13
|
|
|
14
14
|
import os
|
|
15
|
+
import sys
|
|
15
16
|
|
|
16
17
|
from reportlab.pdfbase import pdfmetrics
|
|
17
18
|
from reportlab.pdfbase.cidfonts import UnicodeCIDFont
|
|
@@ -25,7 +26,9 @@ log = get_logger(__name__)
|
|
|
25
26
|
FALLBACK_FONT = "Helvetica"
|
|
26
27
|
|
|
27
28
|
_UNICODE_TTF_NAME = "ScanlayerUnicode"
|
|
28
|
-
|
|
29
|
+
_scanlayer_root = os.path.dirname(os.path.dirname(__file__))
|
|
30
|
+
_bundle_root = getattr(sys, "_MEIPASS", os.path.dirname(_scanlayer_root))
|
|
31
|
+
_UNICODE_TTF_PATH = os.path.join(_bundle_root, "scanlayer", "fonts", "DejaVuSans.ttf")
|
|
29
32
|
|
|
30
33
|
_CUSTOM_TTF_NAME = "ScanlayerCustom"
|
|
31
34
|
_custom_registered_path: str | None = None
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scanlayer
|
|
3
|
-
Version: 1.0
|
|
3
|
+
Version: 1.1.0
|
|
4
4
|
Summary: Image (scan/photo) to searchable PDF, or raw OCR export, via Tesseract.
|
|
5
5
|
Author: Hyacinthe-primus
|
|
6
6
|
Author-email: Hyacinthe-primus <hyacintheatho91@gmail.com>
|
|
@@ -19,27 +19,35 @@ Requires-Dist: numpy>=1.24.0
|
|
|
19
19
|
Requires-Dist: pdf2image>=1.17.0
|
|
20
20
|
Provides-Extra: test
|
|
21
21
|
Requires-Dist: pytest>=7.0; extra == "test"
|
|
22
|
+
Requires-Dist: pytest-cov>=4.1; extra == "test"
|
|
22
23
|
Provides-Extra: yaml
|
|
23
24
|
Requires-Dist: PyYAML>=6.0; extra == "yaml"
|
|
24
25
|
Dynamic: license-file
|
|
25
26
|
|
|
26
27
|
<div align="center">
|
|
27
|
-
<img src="scanlayer.png" alt="scanlayer" width="110"/>
|
|
28
|
+
<img src="https://raw.githubusercontent.com/Hyacinthe-primus/scanlayer/main/scanlayer.png" alt="scanlayer" width="110"/>
|
|
28
29
|
<h1>scanlayer</h1>
|
|
29
|
-
<p><em>
|
|
30
|
+
<p><em> Turn scanned image or photographed document into searchable PDF, or raw OCR text/JSON/TSV/hOCR.</em></p>
|
|
30
31
|
</div>
|
|
31
32
|
|
|
32
33
|
**Turn a scanned image or photographed document into a searchable PDF**, or
|
|
33
34
|
export the raw OCR result as plain text, JSON, TSV, or hOCR. Use it as a
|
|
34
35
|
command-line tool or as a Python library; both are the same engine underneath.
|
|
35
36
|
|
|
36
|
-
[](https://badge.fury.io/py/scanlayer)
|
|
38
|
+
[](https://codecov.io/gh/Hyacinthe-primus/scanlayer)
|
|
39
|
+
[](#)
|
|
37
40
|

|
|
38
41
|
[](https://Hyacinthe-primus.github.io/scanlayer/)
|
|
42
|
+
[](https://github.com/Hyacinthe-primus/scanlayer)
|
|
39
43
|
[](LICENSE.md)
|
|
40
44
|
|
|
41
45
|
---
|
|
42
46
|
|
|
47
|
+
## Documentation
|
|
48
|
+
|
|
49
|
+
[Full documentation](https://Hyacinthe-primus.github.io/scanlayer/) – installation, CLI & API reference, configuration, output formats, examples, and troubleshooting.
|
|
50
|
+
|
|
43
51
|
## What it does
|
|
44
52
|
|
|
45
53
|
A scanned invoice, a phone photo of a letter, a stack of photographed pages:
|
|
@@ -172,28 +180,6 @@ scanlayer.convert_merge(["page1.jpg", "page2.jpg", "page3.jpg"], "report.pdf")
|
|
|
172
180
|
scanlayer.convert("invoice.jpg", "invoice.json", output_format="json")
|
|
173
181
|
```
|
|
174
182
|
|
|
175
|
-
## Documentation
|
|
176
|
-
|
|
177
|
-
| | |
|
|
178
|
-
|---|---|
|
|
179
|
-
| **[Installation](https://Hyacinthe-primus.github.io/scanlayer/installation.html)** | Tesseract, poppler, and the two ways to install scanlayer itself |
|
|
180
|
-
| **[Examples](https://Hyacinthe-primus.github.io/scanlayer/examples.html)** | Every feature, CLI and library side by side |
|
|
181
|
-
| **[CLI Reference](https://Hyacinthe-primus.github.io/scanlayer/cli-reference.html)** | Every flag and exit code |
|
|
182
|
-
| **[Library API](https://Hyacinthe-primus.github.io/scanlayer/library-api.html)** | `convert()`, `convert_batch()`, `convert_merge()`, exceptions, and the full low-level pipeline API |
|
|
183
|
-
| **[Feature Catalog](https://Hyacinthe-primus.github.io/scanlayer/features.html)** | Everything scanlayer does, by pipeline stage |
|
|
184
|
-
| **[Configuration](https://Hyacinthe-primus.github.io/scanlayer/configuration.html)** | `configure()`, config files, precedence, every tunable |
|
|
185
|
-
| **[Output Formats](https://Hyacinthe-primus.github.io/scanlayer/output-formats.html)** | The `pdf`/`txt`/`json`/`tsv`/`hocr` schemas |
|
|
186
|
-
| **[Multi-Page & Merge](https://Hyacinthe-primus.github.io/scanlayer/multi-page-merge.html)** | Batching, merging, native PDF input |
|
|
187
|
-
| **[Debug Visualization](https://Hyacinthe-primus.github.io/scanlayer/debug-visualization.html)** | Reading the `--debug-image` confidence overlay |
|
|
188
|
-
| **[Bundling Tesseract](https://Hyacinthe-primus.github.io/scanlayer/bundling-tesseract.html)** | Shipping your own Tesseract binary |
|
|
189
|
-
| **[Troubleshooting](https://Hyacinthe-primus.github.io/scanlayer/troubleshooting.html)** | Common errors and fixes |
|
|
190
|
-
| **[Roadmap & Limitations](https://Hyacinthe-primus.github.io/scanlayer/roadmap.html)** | What's missing and what's deliberately out of scope |
|
|
191
|
-
|
|
192
|
-
The site is published from the `gh-pages` branch at
|
|
193
|
-
https://Hyacinthe-primus.github.io/scanlayer/. To browse it locally, open
|
|
194
|
-
`index.html` in a `gh-pages` worktree (e.g. `git worktree add <path> gh-pages`);
|
|
195
|
-
it has no build step or server-side dependency.
|
|
196
|
-
|
|
197
183
|
## Repository layout
|
|
198
184
|
|
|
199
185
|
```
|
|
@@ -226,8 +212,20 @@ Bug reports, fixes, and feature discussions are welcome: see
|
|
|
226
212
|
[CONTRIBUTING.md](CONTRIBUTING.md) for how to set up a development
|
|
227
213
|
environment and what to include in a pull request. There is a `tests/`
|
|
228
214
|
directory (now including `utils/validators.py` and the `--dry-run` flag, with
|
|
229
|
-
cross-platform coverage for Tesseract discovery on Windows/macOS/Linux) and it runs with `pytest
|
|
215
|
+
cross-platform coverage for Tesseract discovery on Windows/macOS/Linux) and it runs with `pytest` automatically in CI on every push and pull request; see [Roadmap & Limitations](https://Hyacinthe-primus.github.io/scanlayer/roadmap.html) and the [Adding tests](CONTRIBUTING.md#adding-tests) section of the contributing guide for where coverage is thinnest.
|
|
216
|
+
|
|
217
|
+
## Support the project
|
|
218
|
+
|
|
219
|
+
scanlayer is free and MIT-licensed, maintained in spare time. If it saves you
|
|
220
|
+
time, there is no fee – but a little support goes a long way:
|
|
221
|
+
|
|
222
|
+
- **Star the repository** – the simplest signal that the project is worth
|
|
223
|
+
maintaining.
|
|
224
|
+
- **Report bugs or request features** by opening a
|
|
225
|
+
[GitHub Issue](https://github.com/Hyacinthe-primus/scanlayer/issues).
|
|
226
|
+
- **Contribute** a fix, test, or example – see
|
|
227
|
+
[CONTRIBUTING.md](CONTRIBUTING.md).
|
|
230
228
|
|
|
231
229
|
## License
|
|
232
230
|
|
|
233
|
-
[MIT](LICENSE.md).
|
|
231
|
+
This project is licensed under the [MIT License](LICENSE.md).
|
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
"""Tests for processing several pages of a --merge run at once.
|
|
2
|
+
|
|
3
|
+
Everything here fakes the per-page work, so no Tesseract is needed: the
|
|
4
|
+
point is the scheduling (order, real concurrency, the worker cap, error
|
|
5
|
+
reporting, the shared OCR slot limit), not the OCR itself.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import os
|
|
9
|
+
import threading
|
|
10
|
+
import time
|
|
11
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
12
|
+
|
|
13
|
+
import numpy as np
|
|
14
|
+
import pytest
|
|
15
|
+
|
|
16
|
+
from scanlayer import config
|
|
17
|
+
from scanlayer import main as main_module
|
|
18
|
+
from scanlayer.cli import main
|
|
19
|
+
from scanlayer.cli.dry_run import EXIT_OK, EXIT_USER_ERROR
|
|
20
|
+
from scanlayer.ocr import engine
|
|
21
|
+
from scanlayer.utils.errors import BlankPageDetectedError, OCRProcessingError
|
|
22
|
+
|
|
23
|
+
STATS = {"words": 1, "mean_conf": 90.0, "best_psm": 3, "columns": 1}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class FakePages:
|
|
27
|
+
"""Stands in for _process_page_for_merge and records how it was called."""
|
|
28
|
+
|
|
29
|
+
def __init__(self, sleep=0.0, delays=None, errors=None):
|
|
30
|
+
self.sleep = sleep
|
|
31
|
+
self.delays = delays or {}
|
|
32
|
+
self.errors = errors or {}
|
|
33
|
+
self.lock = threading.Lock()
|
|
34
|
+
self.running = 0
|
|
35
|
+
self.max_running = 0
|
|
36
|
+
self.started = []
|
|
37
|
+
self.thread_ids = set()
|
|
38
|
+
self.slots = []
|
|
39
|
+
|
|
40
|
+
def __call__(
|
|
41
|
+
self, input_path, lang, dpi, whitelist, blacklist, orientation, force,
|
|
42
|
+
ocr_slots=None,
|
|
43
|
+
):
|
|
44
|
+
with self.lock:
|
|
45
|
+
self.started.append(input_path)
|
|
46
|
+
self.running += 1
|
|
47
|
+
self.max_running = max(self.max_running, self.running)
|
|
48
|
+
self.thread_ids.add(threading.get_ident())
|
|
49
|
+
self.slots.append(ocr_slots)
|
|
50
|
+
try:
|
|
51
|
+
time.sleep(self.delays.get(input_path, self.sleep))
|
|
52
|
+
if input_path in self.errors:
|
|
53
|
+
raise self.errors[input_path]
|
|
54
|
+
return f"page:{input_path}", dict(STATS)
|
|
55
|
+
finally:
|
|
56
|
+
with self.lock:
|
|
57
|
+
self.running -= 1
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@pytest.fixture
|
|
61
|
+
def merge_env(monkeypatch, tmp_path):
|
|
62
|
+
"""Fake page processing + PDF writer, 8 CPUs, returns (run, built)."""
|
|
63
|
+
built = []
|
|
64
|
+
|
|
65
|
+
def fake_build(pages, output_path, jpeg_quality=None, metadata=None):
|
|
66
|
+
built.append(list(pages))
|
|
67
|
+
return 123
|
|
68
|
+
|
|
69
|
+
monkeypatch.setattr(main_module, "build_searchable_pdf_multipage", fake_build)
|
|
70
|
+
monkeypatch.setattr(os, "cpu_count", lambda: 8)
|
|
71
|
+
|
|
72
|
+
def run(fake, count, **kwargs):
|
|
73
|
+
monkeypatch.setattr(main_module, "_process_page_for_merge", fake)
|
|
74
|
+
paths = [f"p{i}" for i in range(count)]
|
|
75
|
+
return main_module.convert_merge(paths, str(tmp_path / "out.pdf"), **kwargs)
|
|
76
|
+
|
|
77
|
+
return run, built
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def test_pages_keep_input_order_when_they_finish_out_of_order(merge_env):
|
|
81
|
+
run, built = merge_env
|
|
82
|
+
fake = FakePages(delays={"p0": 0.15, "p1": 0.1, "p2": 0.05})
|
|
83
|
+
run(fake, 6, workers=3)
|
|
84
|
+
assert built == [[f"page:p{i}" for i in range(6)]]
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def test_pages_really_run_concurrently(merge_env):
|
|
88
|
+
run, _ = merge_env
|
|
89
|
+
fake = FakePages(sleep=0.1)
|
|
90
|
+
run(fake, 6, workers=3)
|
|
91
|
+
assert fake.max_running == 3
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def test_workers_above_the_limit_are_capped(merge_env):
|
|
95
|
+
run, _ = merge_env
|
|
96
|
+
fake = FakePages(sleep=0.05)
|
|
97
|
+
run(fake, 10, workers=10)
|
|
98
|
+
assert fake.max_running == config.MERGE_WORKERS_LIMIT == 4
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def test_one_worker_runs_on_the_calling_thread_without_slots(merge_env):
|
|
102
|
+
run, built = merge_env
|
|
103
|
+
fake = FakePages(sleep=0.01)
|
|
104
|
+
run(fake, 4, workers=1)
|
|
105
|
+
assert fake.max_running == 1
|
|
106
|
+
assert fake.thread_ids == {threading.get_ident()}
|
|
107
|
+
assert fake.slots == [None] * 4
|
|
108
|
+
assert built == [[f"page:p{i}" for i in range(4)]]
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def test_default_comes_from_configure(merge_env):
|
|
112
|
+
run, _ = merge_env
|
|
113
|
+
config.configure(merge_workers=2)
|
|
114
|
+
fake = FakePages(sleep=0.1)
|
|
115
|
+
run(fake, 6)
|
|
116
|
+
assert fake.max_running == 2
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def test_default_is_three(merge_env):
|
|
120
|
+
run, _ = merge_env
|
|
121
|
+
fake = FakePages(sleep=0.1)
|
|
122
|
+
run(fake, 6)
|
|
123
|
+
assert fake.max_running == 3
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def test_workers_never_exceed_the_cpu_count(merge_env, monkeypatch):
|
|
127
|
+
run, _ = merge_env
|
|
128
|
+
monkeypatch.setattr(os, "cpu_count", lambda: 2)
|
|
129
|
+
fake = FakePages(sleep=0.1)
|
|
130
|
+
run(fake, 6, workers=4)
|
|
131
|
+
assert fake.max_running == 2
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def test_workers_never_exceed_the_page_count(merge_env):
|
|
135
|
+
run, _ = merge_env
|
|
136
|
+
fake = FakePages(sleep=0.05)
|
|
137
|
+
run(fake, 2, workers=4)
|
|
138
|
+
assert fake.max_running == 2
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
@pytest.mark.parametrize("bad", [0, -1, 1.5, True, "3"])
|
|
142
|
+
def test_invalid_workers_rejected_before_any_work(merge_env, bad):
|
|
143
|
+
run, built = merge_env
|
|
144
|
+
fake = FakePages()
|
|
145
|
+
with pytest.raises(ValueError, match="workers"):
|
|
146
|
+
run(fake, 3, workers=bad)
|
|
147
|
+
assert fake.started == []
|
|
148
|
+
assert built == []
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def test_first_bad_page_in_input_order_is_the_one_reported(merge_env):
|
|
152
|
+
run, built = merge_env
|
|
153
|
+
fake = FakePages(
|
|
154
|
+
delays={"p0": 0.2, "p1": 0.1},
|
|
155
|
+
errors={
|
|
156
|
+
"p1": BlankPageDetectedError("p1 is blank"),
|
|
157
|
+
"p2": BlankPageDetectedError("p2 is blank"),
|
|
158
|
+
},
|
|
159
|
+
)
|
|
160
|
+
with pytest.raises(BlankPageDetectedError, match="p1 is blank"):
|
|
161
|
+
run(fake, 4, workers=3)
|
|
162
|
+
assert built == []
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def test_a_failure_drops_pages_that_have_not_started(merge_env):
|
|
166
|
+
run, built = merge_env
|
|
167
|
+
fake = FakePages(
|
|
168
|
+
sleep=0.05, delays={"p0": 0.0}, errors={"p0": OCRProcessingError("boom")},
|
|
169
|
+
)
|
|
170
|
+
with pytest.raises(OCRProcessingError, match="boom"):
|
|
171
|
+
run(fake, 20, workers=2)
|
|
172
|
+
assert len(fake.started) < 20
|
|
173
|
+
assert built == []
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def test_concurrent_pages_share_one_slot_pool_sized_to_the_cpu_count(merge_env):
|
|
177
|
+
run, _ = merge_env
|
|
178
|
+
fake = FakePages(sleep=0.02)
|
|
179
|
+
run(fake, 5, workers=3)
|
|
180
|
+
slots = fake.slots[0]
|
|
181
|
+
assert slots is not None and all(s is slots for s in fake.slots)
|
|
182
|
+
free = sum(1 for _ in range(20) if slots.acquire(blocking=False))
|
|
183
|
+
assert free == 8
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def test_convert_batch_passes_workers_to_merge(monkeypatch, tmp_path, text_image):
|
|
187
|
+
seen = {}
|
|
188
|
+
|
|
189
|
+
def fake_merge(**kwargs):
|
|
190
|
+
seen.update(kwargs)
|
|
191
|
+
return "result"
|
|
192
|
+
|
|
193
|
+
monkeypatch.setattr(main_module, "convert_merge", fake_merge)
|
|
194
|
+
batch = main_module.convert_batch(
|
|
195
|
+
[text_image, text_image], str(tmp_path / "out.pdf"), merge=True, workers=2,
|
|
196
|
+
)
|
|
197
|
+
assert seen["workers"] == 2
|
|
198
|
+
assert batch.results == ["result"]
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _fake_psm_runner(calls):
|
|
202
|
+
def fake(*args, psm, ocr_slots=None, **kwargs):
|
|
203
|
+
calls.append((psm, ocr_slots))
|
|
204
|
+
return engine.PsmAttempt(psm=psm, words=[], mean_confidence=0.0, elapsed_ms=0.0), None
|
|
205
|
+
|
|
206
|
+
return fake
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
@pytest.mark.parametrize(
|
|
210
|
+
"settings",
|
|
211
|
+
[
|
|
212
|
+
{"psm_candidates": [3]},
|
|
213
|
+
{"psm_candidates": [3, 4, 6, 11], "psm_parallel": True},
|
|
214
|
+
{"psm_candidates": [3, 4, 6, 11], "psm_parallel": False},
|
|
215
|
+
],
|
|
216
|
+
)
|
|
217
|
+
def test_every_psm_pass_gets_the_shared_slots(monkeypatch, settings):
|
|
218
|
+
calls = []
|
|
219
|
+
monkeypatch.setattr(engine, "_safe_run_ocr", _fake_psm_runner(calls))
|
|
220
|
+
config.configure(**settings)
|
|
221
|
+
slots = threading.BoundedSemaphore(2)
|
|
222
|
+
engine.extract_words(
|
|
223
|
+
np.zeros((10, 10), dtype=np.uint8), 1.0, 300, ocr_slots=slots,
|
|
224
|
+
)
|
|
225
|
+
assert len(calls) == len(settings["psm_candidates"])
|
|
226
|
+
assert all(s is slots for _, s in calls)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _racing_psm_runner(confidence, delays):
|
|
230
|
+
def fake(*args, psm, ocr_slots=None, **kwargs):
|
|
231
|
+
time.sleep(delays.get(psm, 0.0))
|
|
232
|
+
attempt = engine.PsmAttempt(
|
|
233
|
+
psm=psm, words=[], mean_confidence=confidence[psm], elapsed_ms=0.0,
|
|
234
|
+
)
|
|
235
|
+
return attempt, None
|
|
236
|
+
|
|
237
|
+
return fake
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def test_psm_tie_goes_to_the_first_candidate_even_if_it_finishes_last(monkeypatch):
|
|
241
|
+
monkeypatch.setattr(
|
|
242
|
+
engine,
|
|
243
|
+
"_safe_run_ocr",
|
|
244
|
+
_racing_psm_runner({3: 50.0, 4: 50.0, 6: 50.0, 11: 50.0}, {3: 0.15}),
|
|
245
|
+
)
|
|
246
|
+
config.configure(psm_candidates=[3, 4, 6, 11], psm_parallel=True)
|
|
247
|
+
result = engine.extract_words(np.zeros((10, 10), dtype=np.uint8), 1.0, 300)
|
|
248
|
+
assert result.best_psm == 3
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def test_psm_higher_confidence_still_beats_an_earlier_candidate(monkeypatch):
|
|
252
|
+
monkeypatch.setattr(
|
|
253
|
+
engine,
|
|
254
|
+
"_safe_run_ocr",
|
|
255
|
+
_racing_psm_runner({3: 40.0, 4: 40.0, 6: 60.0, 11: 40.0}, {6: 0.15}),
|
|
256
|
+
)
|
|
257
|
+
config.configure(psm_candidates=[3, 4, 6, 11], psm_parallel=True)
|
|
258
|
+
result = engine.extract_words(np.zeros((10, 10), dtype=np.uint8), 1.0, 300)
|
|
259
|
+
assert result.best_psm == 6
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def test_slots_bound_concurrent_tesseract_passes_and_are_released(monkeypatch):
|
|
263
|
+
lock = threading.Lock()
|
|
264
|
+
state = {"running": 0, "max": 0}
|
|
265
|
+
|
|
266
|
+
def fake_run_ocr(*args, **kwargs):
|
|
267
|
+
with lock:
|
|
268
|
+
state["running"] += 1
|
|
269
|
+
state["max"] = max(state["max"], state["running"])
|
|
270
|
+
time.sleep(0.03)
|
|
271
|
+
with lock:
|
|
272
|
+
state["running"] -= 1
|
|
273
|
+
if kwargs.get("fail"):
|
|
274
|
+
raise OCRProcessingError("timeout")
|
|
275
|
+
return engine.PsmAttempt(psm=3, words=[], mean_confidence=0.0, elapsed_ms=0.0)
|
|
276
|
+
|
|
277
|
+
monkeypatch.setattr(engine, "_run_ocr", fake_run_ocr)
|
|
278
|
+
slots = threading.BoundedSemaphore(2)
|
|
279
|
+
|
|
280
|
+
def one(_):
|
|
281
|
+
return engine._safe_run_ocr(None, "eng", 3, "", 300, None, None, psm=3, ocr_slots=slots)
|
|
282
|
+
|
|
283
|
+
with ThreadPoolExecutor(max_workers=6) as pool:
|
|
284
|
+
list(pool.map(one, range(6)))
|
|
285
|
+
assert state["max"] == 2
|
|
286
|
+
|
|
287
|
+
failing = engine._safe_run_ocr(
|
|
288
|
+
None, "eng", 3, "", 300, None, None, psm=3, ocr_slots=slots, fail=True,
|
|
289
|
+
)
|
|
290
|
+
assert failing[0] is None
|
|
291
|
+
assert sum(1 for _ in range(5) if slots.acquire(blocking=False)) == 2
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def test_workers_flag_parses():
|
|
295
|
+
from scanlayer.cli.parser import _build_parser
|
|
296
|
+
|
|
297
|
+
args = _build_parser().parse_args(["a.jpg", "-o", "x.pdf", "--merge", "--workers", "3"])
|
|
298
|
+
assert args.workers == 3
|
|
299
|
+
assert _build_parser().parse_args(["a.jpg"]).workers is None
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def test_cli_rejects_zero_workers(text_image, tmp_path):
|
|
303
|
+
out = tmp_path / "merged.pdf"
|
|
304
|
+
code = main([text_image, "-o", str(out), "--merge", "--workers", "0"])
|
|
305
|
+
assert code == EXIT_USER_ERROR
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def test_cli_passes_workers_to_convert_merge(monkeypatch, text_image, tmp_path):
|
|
309
|
+
import scanlayer.cli.run as run_module
|
|
310
|
+
|
|
311
|
+
seen = {}
|
|
312
|
+
monkeypatch.setattr(run_module, "convert_merge", lambda **kw: seen.update(kw))
|
|
313
|
+
out = tmp_path / "merged.pdf"
|
|
314
|
+
code = main([text_image, text_image, "-o", str(out), "--merge", "--workers", "2"])
|
|
315
|
+
assert code == EXIT_OK
|
|
316
|
+
assert seen["workers"] == 2
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|