heocr 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. heocr-0.1.0/LICENSE +202 -0
  2. heocr-0.1.0/NOTICE +8 -0
  3. heocr-0.1.0/PKG-INFO +172 -0
  4. heocr-0.1.0/README.md +138 -0
  5. heocr-0.1.0/THIRD_PARTY_LICENSES.md +88 -0
  6. heocr-0.1.0/pyproject.toml +55 -0
  7. heocr-0.1.0/setup.cfg +4 -0
  8. heocr-0.1.0/src/heocr/__init__.py +8 -0
  9. heocr-0.1.0/src/heocr/__main__.py +3 -0
  10. heocr-0.1.0/src/heocr/api.py +145 -0
  11. heocr-0.1.0/src/heocr/cli.py +120 -0
  12. heocr-0.1.0/src/heocr/contract.py +497 -0
  13. heocr-0.1.0/src/heocr/convert.py +207 -0
  14. heocr-0.1.0/src/heocr/docx/__init__.py +0 -0
  15. heocr-0.1.0/src/heocr/docx/numbering.py +250 -0
  16. heocr-0.1.0/src/heocr/docx/numformat.py +102 -0
  17. heocr-0.1.0/src/heocr/docx/reader.py +596 -0
  18. heocr-0.1.0/src/heocr/docx/styles.py +163 -0
  19. heocr-0.1.0/src/heocr/htmlread.py +908 -0
  20. heocr-0.1.0/src/heocr/identify.py +175 -0
  21. heocr-0.1.0/src/heocr/images.py +103 -0
  22. heocr-0.1.0/src/heocr/models.py +136 -0
  23. heocr-0.1.0/src/heocr/ocr/__init__.py +79 -0
  24. heocr-0.1.0/src/heocr/ocr/regions.py +198 -0
  25. heocr-0.1.0/src/heocr/ocr/tesseract.py +264 -0
  26. heocr-0.1.0/src/heocr/pdf/__init__.py +0 -0
  27. heocr-0.1.0/src/heocr/pdf/classify.py +241 -0
  28. heocr-0.1.0/src/heocr/pdf/columns.py +483 -0
  29. heocr-0.1.0/src/heocr/pdf/fontmap.py +221 -0
  30. heocr-0.1.0/src/heocr/pdf/frame.py +121 -0
  31. heocr-0.1.0/src/heocr/pdf/furniture.py +160 -0
  32. heocr-0.1.0/src/heocr/pdf/lines.py +516 -0
  33. heocr-0.1.0/src/heocr/pdf/numbers.py +108 -0
  34. heocr-0.1.0/src/heocr/pdf/ocrtables.py +163 -0
  35. heocr-0.1.0/src/heocr/pdf/reader.py +2518 -0
  36. heocr-0.1.0/src/heocr/pdf/tables.py +409 -0
  37. heocr-0.1.0/src/heocr/pdf/webprint.py +234 -0
  38. heocr-0.1.0/src/heocr/pptx.py +606 -0
  39. heocr-0.1.0/src/heocr/render.py +155 -0
  40. heocr-0.1.0/src/heocr/repairs.py +352 -0
  41. heocr-0.1.0/src/heocr/schemas/heocr_document.v1.schema.json +165 -0
  42. heocr-0.1.0/src/heocr/textflags.py +60 -0
  43. heocr-0.1.0/src/heocr/xlsx.py +685 -0
  44. heocr-0.1.0/src/heocr.egg-info/PKG-INFO +172 -0
  45. heocr-0.1.0/src/heocr.egg-info/SOURCES.txt +84 -0
  46. heocr-0.1.0/src/heocr.egg-info/dependency_links.txt +1 -0
  47. heocr-0.1.0/src/heocr.egg-info/entry_points.txt +2 -0
  48. heocr-0.1.0/src/heocr.egg-info/requires.txt +6 -0
  49. heocr-0.1.0/src/heocr.egg-info/top_level.txt +1 -0
  50. heocr-0.1.0/tests/test_cli.py +58 -0
  51. heocr-0.1.0/tests/test_contract.py +104 -0
  52. heocr-0.1.0/tests/test_convert.py +182 -0
  53. heocr-0.1.0/tests/test_cp1255_shifted.py +61 -0
  54. heocr-0.1.0/tests/test_docx_images.py +100 -0
  55. heocr-0.1.0/tests/test_docx_note_refs.py +41 -0
  56. heocr-0.1.0/tests/test_docx_numbering.py +139 -0
  57. heocr-0.1.0/tests/test_docx_reader.py +276 -0
  58. heocr-0.1.0/tests/test_external_producer.py +253 -0
  59. heocr-0.1.0/tests/test_fontmap.py +157 -0
  60. heocr-0.1.0/tests/test_hostile.py +179 -0
  61. heocr-0.1.0/tests/test_html.py +389 -0
  62. heocr-0.1.0/tests/test_identify.py +83 -0
  63. heocr-0.1.0/tests/test_images.py +197 -0
  64. heocr-0.1.0/tests/test_models.py +134 -0
  65. heocr-0.1.0/tests/test_number_checks.py +209 -0
  66. heocr-0.1.0/tests/test_ocr.py +545 -0
  67. heocr-0.1.0/tests/test_ocr_numbers.py +475 -0
  68. heocr-0.1.0/tests/test_ocr_tables.py +548 -0
  69. heocr-0.1.0/tests/test_pdf_classes.py +163 -0
  70. heocr-0.1.0/tests/test_pdf_columns.py +257 -0
  71. heocr-0.1.0/tests/test_pdf_frame.py +165 -0
  72. heocr-0.1.0/tests/test_pdf_furniture.py +83 -0
  73. heocr-0.1.0/tests/test_pdf_lines.py +137 -0
  74. heocr-0.1.0/tests/test_pdf_spreads.py +73 -0
  75. heocr-0.1.0/tests/test_pdf_t1.py +178 -0
  76. heocr-0.1.0/tests/test_pdf_tables.py +235 -0
  77. heocr-0.1.0/tests/test_pdf_webprint.py +185 -0
  78. heocr-0.1.0/tests/test_pptx.py +241 -0
  79. heocr-0.1.0/tests/test_render.py +56 -0
  80. heocr-0.1.0/tests/test_repairs.py +86 -0
  81. heocr-0.1.0/tests/test_repairs_d2c.py +185 -0
  82. heocr-0.1.0/tests/test_repo_rules.py +93 -0
  83. heocr-0.1.0/tests/test_schema.py +79 -0
  84. heocr-0.1.0/tests/test_time_limit.py +146 -0
  85. heocr-0.1.0/tests/test_version.py +12 -0
  86. heocr-0.1.0/tests/test_xlsx.py +248 -0
heocr-0.1.0/LICENSE ADDED
@@ -0,0 +1,202 @@
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
+
7
+ 1. Definitions.
8
+
9
+ "License" shall mean the terms and conditions for use, reproduction,
10
+ and distribution as defined by Sections 1 through 9 of this document.
11
+
12
+ "Licensor" shall mean the copyright owner or entity authorized by
13
+ the copyright owner that is granting the License.
14
+
15
+ "Legal Entity" shall mean the union of the acting entity and all
16
+ other entities that control, are controlled by, or are under common
17
+ control with that entity. For the purposes of this definition,
18
+ "control" means (i) the power, direct or indirect, to cause the
19
+ direction or management of such entity, whether by contract or
20
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21
+ outstanding shares, or (iii) beneficial ownership of such entity.
22
+
23
+ "You" (or "Your") shall mean an individual or Legal Entity
24
+ exercising permissions granted by this License.
25
+
26
+ "Source" form shall mean the preferred form for making modifications,
27
+ including but not limited to software source code, documentation
28
+ source, and configuration files.
29
+
30
+ "Object" form shall mean any form resulting from mechanical
31
+ transformation or translation of a Source form, including but
32
+ not limited to compiled object code, generated documentation,
33
+ and conversions to other media types.
34
+
35
+ "Work" shall mean the work of authorship, whether in Source or
36
+ Object form, made available under the License, as indicated by a
37
+ copyright notice that is included in or attached to the work
38
+ (an example is provided in the Appendix below).
39
+
40
+ "Derivative Works" shall mean any work, whether in Source or Object
41
+ form, that is based on (or derived from) the Work and for which the
42
+ editorial revisions, annotations, elaborations, or other modifications
43
+ represent, as a whole, an original work of authorship. For the purposes
44
+ of this License, Derivative Works shall not include works that remain
45
+ separable from, or merely link (or bind by name) to the interfaces of,
46
+ the Work and Derivative Works thereof.
47
+
48
+ "Contribution" shall mean any work of authorship, including
49
+ the original version of the Work and any modifications or additions
50
+ to that Work or Derivative Works thereof, that is intentionally
51
+ submitted to Licensor for inclusion in the Work by the copyright owner
52
+ or by an individual or Legal Entity authorized to submit on behalf of
53
+ the copyright owner. For the purposes of this definition, "submitted"
54
+ means any form of electronic, verbal, or written communication sent
55
+ to the Licensor or its representatives, including but not limited to
56
+ communication on electronic mailing lists, source code control systems,
57
+ and issue tracking systems that are managed by, or on behalf of, the
58
+ Licensor for the purpose of discussing and improving the Work, but
59
+ excluding communication that is conspicuously marked or otherwise
60
+ designated in writing by the copyright owner as "Not a Contribution."
61
+
62
+ "Contributor" shall mean Licensor and any individual or Legal Entity
63
+ on behalf of whom a Contribution has been received by Licensor and
64
+ subsequently incorporated within the Work.
65
+
66
+ 2. Grant of Copyright License. Subject to the terms and conditions of
67
+ this License, each Contributor hereby grants to You a perpetual,
68
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69
+ copyright license to reproduce, prepare Derivative Works of,
70
+ publicly display, publicly perform, sublicense, and distribute the
71
+ Work and such Derivative Works in Source or Object form.
72
+
73
+ 3. Grant of Patent License. Subject to the terms and conditions of
74
+ this License, each Contributor hereby grants to You a perpetual,
75
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76
+ (except as stated in this section) patent license to make, have made,
77
+ use, offer to sell, sell, import, and otherwise transfer the Work,
78
+ where such license applies only to those patent claims licensable
79
+ by such Contributor that are necessarily infringed by their
80
+ Contribution(s) alone or by combination of their Contribution(s)
81
+ with the Work to which such Contribution(s) was submitted. If You
82
+ institute patent litigation against any entity (including a
83
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
84
+ or a Contribution incorporated within the Work constitutes direct
85
+ or contributory patent infringement, then any patent licenses
86
+ granted to You under this License for that Work shall terminate
87
+ as of the date such litigation is filed.
88
+
89
+ 4. Redistribution. You may reproduce and distribute copies of the
90
+ Work or Derivative Works thereof in any medium, with or without
91
+ modifications, and in Source or Object form, provided that You
92
+ meet the following conditions:
93
+
94
+ (a) You must give any other recipients of the Work or
95
+ Derivative Works a copy of this License; and
96
+
97
+ (b) You must cause any modified files to carry prominent notices
98
+ stating that You changed the files; and
99
+
100
+ (c) You must retain, in the Source form of any Derivative Works
101
+ that You distribute, all copyright, patent, trademark, and
102
+ attribution notices from the Source form of the Work,
103
+ excluding those notices that do not pertain to any part of
104
+ the Derivative Works; and
105
+
106
+ (d) If the Work includes a "NOTICE" text file as part of its
107
+ distribution, then any Derivative Works that You distribute must
108
+ include a readable copy of the attribution notices contained
109
+ within such NOTICE file, excluding those notices that do not
110
+ pertain to any part of the Derivative Works, in at least one
111
+ of the following places: within a NOTICE text file distributed
112
+ as part of the Derivative Works; within the Source form or
113
+ documentation, if provided along with the Derivative Works; or,
114
+ within a display generated by the Derivative Works, if and
115
+ wherever such third-party notices normally appear. The contents
116
+ of the NOTICE file are for informational purposes only and
117
+ do not modify the License. You may add Your own attribution
118
+ notices within Derivative Works that You distribute, alongside
119
+ or as an addendum to the NOTICE text from the Work, provided
120
+ that such additional attribution notices cannot be construed
121
+ as modifying the License.
122
+
123
+ You may add Your own copyright statement to Your modifications and
124
+ may provide additional or different license terms and conditions
125
+ for use, reproduction, or distribution of Your modifications, or
126
+ for any such Derivative Works as a whole, provided Your use,
127
+ reproduction, and distribution of the Work otherwise complies with
128
+ the conditions stated in this License.
129
+
130
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
131
+ any Contribution intentionally submitted for inclusion in the Work
132
+ by You to the Licensor shall be under the terms and conditions of
133
+ this License, without any additional terms or conditions.
134
+ Notwithstanding the above, nothing herein shall supersede or modify
135
+ the terms of any separate license agreement you may have executed
136
+ with Licensor regarding such Contributions.
137
+
138
+ 6. Trademarks. This License does not grant permission to use the trade
139
+ names, trademarks, service marks, or product names of the Licensor,
140
+ except as required for reasonable and customary use in describing the
141
+ origin of the Work and reproducing the content of the NOTICE file.
142
+
143
+ 7. Disclaimer of Warranty. Unless required by applicable law or
144
+ agreed to in writing, Licensor provides the Work (and each
145
+ Contributor provides its Contributions) on an "AS IS" BASIS,
146
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147
+ implied, including, without limitation, any warranties or conditions
148
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149
+ PARTICULAR PURPOSE. You are solely responsible for determining the
150
+ appropriateness of using or redistributing the Work and assume any
151
+ risks associated with Your exercise of permissions under this License.
152
+
153
+ 8. Limitation of Liability. In no event and under no legal theory,
154
+ whether in tort (including negligence), contract, or otherwise,
155
+ unless required by applicable law (such as deliberate and grossly
156
+ negligent acts) or agreed to in writing, shall any Contributor be
157
+ liable to You for damages, including any direct, indirect, special,
158
+ incidental, or consequential damages of any character arising as a
159
+ result of this License or out of the use or inability to use the
160
+ Work (including but not limited to damages for loss of goodwill,
161
+ work stoppage, computer failure or malfunction, or any and all
162
+ other commercial damages or losses), even if such Contributor
163
+ has been advised of the possibility of such damages.
164
+
165
+ 9. Accepting Warranty or Additional Liability. While redistributing
166
+ the Work or Derivative Works thereof, You may choose to offer,
167
+ and charge a fee for, acceptance of support, warranty, indemnity,
168
+ or other liability obligations and/or rights consistent with this
169
+ License. However, in accepting such obligations, You may act only
170
+ on Your own behalf and on Your sole responsibility, not on behalf
171
+ of any other Contributor, and only if You agree to indemnify,
172
+ defend, and hold each Contributor harmless for any liability
173
+ incurred by, or claims asserted against, such Contributor by reason
174
+ of your accepting any such warranty or additional liability.
175
+
176
+ END OF TERMS AND CONDITIONS
177
+
178
+ APPENDIX: How to apply the Apache License to your work.
179
+
180
+ To apply the Apache License to your work, attach the following
181
+ boilerplate notice, with the fields enclosed by brackets "[]"
182
+ replaced with your own identifying information. (Don't include
183
+ the brackets!) The text should be enclosed in the appropriate
184
+ comment syntax for the file format. We also recommend that a
185
+ file or class name and description of purpose be included on the
186
+ same "printed page" as the copyright notice for easier
187
+ identification within third-party archives.
188
+
189
+ Copyright [yyyy] [name of copyright owner]
190
+
191
+ Licensed under the Apache License, Version 2.0 (the "License");
192
+ you may not use this file except in compliance with the License.
193
+ You may obtain a copy of the License at
194
+
195
+ http://www.apache.org/licenses/LICENSE-2.0
196
+
197
+ Unless required by applicable law or agreed to in writing, software
198
+ distributed under the License is distributed on an "AS IS" BASIS,
199
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
+ See the License for the specific language governing permissions and
201
+ limitations under the License.
202
+
heocr-0.1.0/NOTICE ADDED
@@ -0,0 +1,8 @@
1
+ heocr
2
+ Copyright the HeOCR maintainers. Licensed under the Apache License, Version 2.0 (see LICENSE).
3
+
4
+ This product includes code ported from internal tools of the Adanim Research Institute, with its permission, under the Apache License, Version 2.0.
5
+
6
+ heocr runs, when they are installed, these separate programs, which it does not bundle:
7
+ - Tesseract OCR (Apache License 2.0);
8
+ - LibreOffice (Mozilla Public License 2.0).
heocr-0.1.0/PKG-INFO ADDED
@@ -0,0 +1,172 @@
1
+ Metadata-Version: 2.4
2
+ Name: heocr
3
+ Version: 0.1.0
4
+ Summary: The best open pipeline for getting Hebrew text out of any document
5
+ Author: HeOCR maintainers
6
+ License-Expression: Apache-2.0
7
+ Project-URL: Source, https://github.com/HeOCR/heocr
8
+ Project-URL: Issues, https://github.com/HeOCR/heocr/issues
9
+ Project-URL: Documentation, https://github.com/HeOCR/heocr#readme
10
+ Keywords: hebrew,ocr,pdf,docx,text extraction,document
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Intended Audience :: Science/Research
14
+ Classifier: Natural Language :: Hebrew
15
+ Classifier: Operating System :: OS Independent
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3 :: Only
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Programming Language :: Python :: 3.13
21
+ Classifier: Topic :: Text Processing
22
+ Classifier: Topic :: Text Processing :: Linguistic
23
+ Requires-Python: >=3.11
24
+ Description-Content-Type: text/markdown
25
+ License-File: LICENSE
26
+ License-File: NOTICE
27
+ License-File: THIRD_PARTY_LICENSES.md
28
+ Requires-Dist: defusedxml>=0.7
29
+ Requires-Dist: pypdfium2>=4.30
30
+ Requires-Dist: Pillow>=10
31
+ Provides-Extra: test
32
+ Requires-Dist: jsonschema>=4.18; extra == "test"
33
+ Dynamic: license-file
34
+
35
+ # heocr
36
+
37
+ **The best open pipeline for getting Hebrew text out of any document.**
38
+
39
+ `heocr` looks at a document and works out what it is: its format, and for a PDF the state of each page's text layer. It then sends each part to the right tool:
40
+ - the text layer, where it can be trusted;
41
+ - our own readers for office files and web pages;
42
+ - OCR, where the text exists only as an image.
43
+
44
+ Every decision is recorded in the output. Text that could not be read is reported in `unread`, never dropped silently.
45
+
46
+ > Status: 0.1.0, the first release: exactly the release candidate the registered D-bench release gate judged (`0.1.0rc1`). Supported: DOCX, DOC, PDF T1–T3, spreads and scanned tables. Not supported: PDF T4, T5 and T6. See the [release notes](https://github.com/HeOCR/heocr/blob/main/docs/release-notes-0.1.0.md).
47
+
48
+ ## Quickstart
49
+
50
+ ```sh
51
+ pip install . # from a clone; the core needs only defusedxml, pypdfium2 and Pillow
52
+ ```
53
+
54
+ ```python
55
+ import heocr
56
+
57
+ doc = heocr.extract("tender.pdf") # a path or the file's bytes; any format below
58
+ doc.text # plain text in reading order; notes, headers, footers and navigation follow the body
59
+ doc.markdown # headings, numbered sections and tables kept
60
+ doc.to_json() # the heocr_document.v1 record: every block with its route, tool and flags
61
+ doc.unread # what was not read, and why
62
+ ```
63
+
64
+ ```sh
65
+ heocr tender.pdf # text to standard output
66
+ heocr tender.docx -o tender.md # the format follows the suffix: .md, .json, else text
67
+ heocr scan.pdf -f json --ocr-lang heb+eng
68
+ heocr report.pdf --no-ocr # never run OCR; pages that need it are reported unread
69
+ ```
70
+
71
+ The CLI prints each `unread` entry on stderr (`-q` silences them). It exits with 1 only when the file cannot be opened or an option is invalid. A document read only in part, or not at all, exits with 0, and its `unread` entries say why.
72
+
73
+ ## Formats and routes
74
+
75
+ The format is decided from the content, never from the file name. The file name is never recorded.
76
+
77
+ | Input | Route | Notes |
78
+ |---|---|---|
79
+ | **PDF**, page by page | The text layer, read with our own right-to-left line building, ruled and unruled tables, page furniture (running headers and footers) and repairs. | Each page gets a class from its signals; see the class table below. |
80
+ | | Region OCR for images on text-layer pages. | Ruled tables inside such images are read cell by cell. |
81
+ | | Whole-page OCR for scans. | Ruled tables are read cell by cell; amounts in cells get a second reading by `eng`. |
82
+ | | Spreads (two pages side by side) are read as two, the right half first in RTL. | A web page printed from a browser has its site menu given the `navigation` role. |
83
+ | **DOCX** | Our own WordprocessingML reader. | List numbering rendered, foot/endnotes tied to their place, hidden and white text dropped and flagged, embedded images OCR'd. |
84
+ | **DOC, RTF, ODT** | LibreOffice converts to DOCX (optional, see below), then the DOCX reader. | |
85
+ | **XLSX** | Our own reader: each sheet is a table of the rows and columns that hold a value. | Cached values only, never recalculated; numbers and dates as their format shows them; hidden sheets, rows and columns read and flagged. |
86
+ | **ODS, XLS** | LibreOffice converts to XLSX, then the XLSX reader. | |
87
+ | **PPTX** | Our own reader: slides in order, shapes in reading order by position, tables, speaker notes (role `notes`), pictures by OCR. | |
88
+ | **ODP, PPT** | LibreOffice converts to PPTX, then the PPTX reader. | |
89
+ | **HTML** | The standard library's parser. | Encoding from the BOM, `<meta>` or windows-1255. Menus, page headers and footers get the role `navigation`. A page filled in by script is reported unread. Legislation's section numbers and margin headings are kept with their sections. |
90
+ | **Images** (PNG, JPEG, TIFF with all frames, GIF, BMP, JPEG 2000, WebP) | OCR, each frame a page. | DPI from the file, else 300 (flagged). |
91
+
92
+ PDF page classes:
93
+
94
+ | Class | What the page is | Route |
95
+ |---|---|---|
96
+ | T1 | A good text layer | Text layer |
97
+ | T2 | A text layer in visual order | Text layer, reordered (or OCR when words are drawn reversed) |
98
+ | T3 | A broken font mapping | Repairs; OCR for the codes left |
99
+ | T4 | Someone else's OCR layer | Ignored; OCR |
100
+ | T5 | An image with no text layer (a scan) | OCR |
101
+ | T6 | A text layer with significant images | Text layer, plus region OCR |
102
+
103
+ ## What the record tells you
104
+
105
+ - **Every block** carries its `source` (`text_layer`, `office_xml`, `converted_docx`/`xlsx`/`pptx`, `html` or `ocr`) and its `tool`. OCR blocks are `unverified`, with their engine, settings and confidence.
106
+ - **Roles:** body text is `body`. `header`, `footer`, `footnote`, `endnote`, `comment`, `textbox`, `navigation` and `notes` are kept in the record and out of the body flow.
107
+ - **Repairs** fix characters the source meant but its font or producer broke, such as reversed brackets, `ð` for `נ`, symbol-font bullets, private-use codes and reversed digit groups. Each one fires only on evidence, and is logged with its counts. `--no-repairs` turns them all off: the output is then the extracted text, byte for byte.
108
+ - **Flags** say where the reading is a guess. For example:
109
+ - `reading_order_unresolved_columns`: side-by-side text whose order is unknown;
110
+ - `unruled_table`: a table inferred from layout alone, so its page is never `selection_safe`;
111
+ - `ocr_short_for_ink`: ink that OCR left unread.
112
+
113
+ ### `unread`: what was not read, and why
114
+
115
+ Every entry has a scope (`document`, `page` or `block`), a reason, and where possible the page and a box. The main reasons:
116
+
117
+ | Reason | When |
118
+ |---|---|
119
+ | `format_not_supported` | A format with no route yet (EPUB, email, archives) |
120
+ | `converter_unavailable`, `conversion_failed`, `conversion_timeout` | LibreOffice missing, refusing the file, or stopped at its time limit |
121
+ | `ocr_route_unavailable`, `ocr_failed`, `ocr_found_no_text`, `ocr_short_for_ink` | Tesseract missing or failing, or a page whose ink OCR did not read |
122
+ | `image_region_*`, `image_*` | Image regions and embedded images: not read, too small, or low confidence. A region whose text is already in the layer, or that is blank, is no loss: it is counted in the page's coverage (`image_regions_text_in_layer`, `image_regions_blank`), not here |
123
+ | `table_row_ink_unread` | A table row with ink in its cells that OCR read none of |
124
+ | `table_cell_unconfirmed` | A scanned table cell whose two readings (`heb` and the number model) differ, neither confident; both are kept in the cell |
125
+ | `rotated_text_not_read` | Characters drawn at an angle other than vertical. Vertical text (90 or 270 degrees: a margin note, a stamp's caption) is read as its own lines after the page's body, flagged `rotated_text` |
126
+ | `text_outside_page` | Characters drawn outside the page's visible box |
127
+ | `html_rendered_by_script` | A web page whose content is filled in by JavaScript; print it to PDF from a browser |
128
+ | `page_limit`, `time_limit`, `sheet_truncated`, `html_truncated`, `merges_not_applied`, `image_too_large` | A limit was reached (below) |
129
+ | `docx_unreadable`, `xlsx_unreadable`, `pptx_unreadable`, `html_unreadable`, `image_unreadable` | A file that does not parse |
130
+ | `chart_not_read`, `smartart_not_read`, `xlsx_drawing_not_read`, `embedded_content_not_read` | Charts, drawings, frames and embedded objects |
131
+
132
+ ## Limits
133
+
134
+ Each limit, when reached, gives an `unread` entry or a flag:
135
+
136
+ - **pages or image frames:** 2,000 (`max_pages`);
137
+ - **time per document:** none by default; `time_limit` (seconds, `--time-limit`) is a safety net, checked before each page, frame and embedded image, and the step under way finishes;
138
+ - **input size:** 512 MB;
139
+ - **LibreOffice:** 180 s per conversion (`convert_timeout`); the conversion's processes are stopped;
140
+ - **HTML:** 64M characters, 2M elements, nesting depth 100 (flattened, with no text lost);
141
+ - **XLSX:** 2M cells per sheet;
142
+ - **PPTX:** group nesting depth 100;
143
+ - **cell spans:** 1,000;
144
+ - **renders for OCR:** 300 dpi, capped at 40M pixels.
145
+
146
+ What is not detected:
147
+
148
+ - **Pointed text in a kept OCR layer.** Some scans carry someone else's OCR layer with niqqud or cantillation, with each mark in its own box, as Internet Archive's scans do. heocr keeps such a layer, but builds the marks into separate pieces, so pointed lines come out fragmented: on the pointed pages we tested, a median of under half of the layer's words came through. Unpointed layers are not affected. There is no switch to force OCR yet: convert those pages to images and read the images, which go through OCR. A fix is planned for the next release.
149
+ - **A partial OCR layer:** a scanned page under someone else's OCR layer that heocr keeps is flagged `ocr_short_for_ink` when at least 10% of the scan's ink lies away from the layer's lines and rulings. A line or two missing from a full page's layer stays under that, and is not detected.
150
+
151
+ Read once:
152
+
153
+ - **A repeated image:** an image drawn again in the same document (a letterhead, a logo, a stamp) is read by region OCR where it first appears. Each later page gets a copy of that reading in its own box, flagged `repeated_image_text` and pointing to the first page (`checks.repeated_image`), with no OCR: every page keeps its text, as running headers do. A first reading that was empty or unsure repeats as counts (`image_regions_repeated`), not as new `unread` entries.
154
+
155
+ ## Optional tools
156
+
157
+ The core runs without them. Without them, the parts that need them are reported in `unread`, never guessed.
158
+
159
+ - **Tesseract** (Apache-2.0), for OCR, with the `heb` model (and `eng` for the second reading of numbers). heocr finds it on the `PATH`, in `$HEOCR_TESSERACT`, or through `tesseract=` / `--tesseract`. Other options:
160
+ - `ocr_lang` / `--ocr-lang` for the languages (default `heb`);
161
+ - `--ocr-model` for a HeOCR model, kept in `$HEOCR_MODELS`.
162
+ - **LibreOffice** (MPL-2.0), for DOC, RTF, ODT, ODS, XLS, ODP and PPT. heocr finds it on the `PATH`, in `$HEOCR_SOFFICE`, or through `soffice=` / `--soffice`; on macOS also in `/Applications`. It runs with a throwaway profile and a time limit. `convert=False` / `--no-convert` turns conversion off.
163
+
164
+ Neither is bundled. heocr records the version and executable hash of each one it uses in the record's `producer.components`.
165
+
166
+ ## Licence and credits
167
+
168
+ Apache-2.0. The core depends only on permissive libraries: pypdfium2, Pillow and defusedxml. Copyleft tools are never in a default route. See [NOTICE](NOTICE), and [THIRD_PARTY_LICENSES.md](THIRD_PARTY_LICENSES.md) for the audit of the dependencies and the libraries they bundle.
169
+
170
+ ## Project
171
+
172
+ `heocr` is part of [HeOCR](https://github.com/HeOCR), an open Hebrew OCR project. Its plan is in [`planning/`](planning/). The project-wide decisions behind it (0019, 0020) are in [heocr-engine](https://github.com/HeOCR/heocr-engine/tree/main/planning/decisions).
heocr-0.1.0/README.md ADDED
@@ -0,0 +1,138 @@
1
+ # heocr
2
+
3
+ **The best open pipeline for getting Hebrew text out of any document.**
4
+
5
+ `heocr` looks at a document and works out what it is: its format, and for a PDF the state of each page's text layer. It then sends each part to the right tool:
6
+ - the text layer, where it can be trusted;
7
+ - our own readers for office files and web pages;
8
+ - OCR, where the text exists only as an image.
9
+
10
+ Every decision is recorded in the output. Text that could not be read is reported in `unread`, never dropped silently.
11
+
12
+ > Status: 0.1.0, the first release: exactly the release candidate the registered D-bench release gate judged (`0.1.0rc1`). Supported: DOCX, DOC, PDF T1–T3, spreads and scanned tables. Not supported: PDF T4, T5 and T6. See the [release notes](https://github.com/HeOCR/heocr/blob/main/docs/release-notes-0.1.0.md).
13
+
14
+ ## Quickstart
15
+
16
+ ```sh
17
+ pip install . # from a clone; the core needs only defusedxml, pypdfium2 and Pillow
18
+ ```
19
+
20
+ ```python
21
+ import heocr
22
+
23
+ doc = heocr.extract("tender.pdf") # a path or the file's bytes; any format below
24
+ doc.text # plain text in reading order; notes, headers, footers and navigation follow the body
25
+ doc.markdown # headings, numbered sections and tables kept
26
+ doc.to_json() # the heocr_document.v1 record: every block with its route, tool and flags
27
+ doc.unread # what was not read, and why
28
+ ```
29
+
30
+ ```sh
31
+ heocr tender.pdf # text to standard output
32
+ heocr tender.docx -o tender.md # the format follows the suffix: .md, .json, else text
33
+ heocr scan.pdf -f json --ocr-lang heb+eng
34
+ heocr report.pdf --no-ocr # never run OCR; pages that need it are reported unread
35
+ ```
36
+
37
+ The CLI prints each `unread` entry on stderr (`-q` silences them). It exits with 1 only when the file cannot be opened or an option is invalid. A document read only in part, or not at all, exits with 0, and its `unread` entries say why.
38
+
39
+ ## Formats and routes
40
+
41
+ The format is decided from the content, never from the file name. The file name is never recorded.
42
+
43
+ | Input | Route | Notes |
44
+ |---|---|---|
45
+ | **PDF**, page by page | The text layer, read with our own right-to-left line building, ruled and unruled tables, page furniture (running headers and footers) and repairs. | Each page gets a class from its signals; see the class table below. |
46
+ | | Region OCR for images on text-layer pages. | Ruled tables inside such images are read cell by cell. |
47
+ | | Whole-page OCR for scans. | Ruled tables are read cell by cell; amounts in cells get a second reading by `eng`. |
48
+ | | Spreads (two pages side by side) are read as two, the right half first in RTL. | A web page printed from a browser has its site menu given the `navigation` role. |
49
+ | **DOCX** | Our own WordprocessingML reader. | List numbering rendered, foot/endnotes tied to their place, hidden and white text dropped and flagged, embedded images OCR'd. |
50
+ | **DOC, RTF, ODT** | LibreOffice converts to DOCX (optional, see below), then the DOCX reader. | |
51
+ | **XLSX** | Our own reader: each sheet is a table of the rows and columns that hold a value. | Cached values only, never recalculated; numbers and dates as their format shows them; hidden sheets, rows and columns read and flagged. |
52
+ | **ODS, XLS** | LibreOffice converts to XLSX, then the XLSX reader. | |
53
+ | **PPTX** | Our own reader: slides in order, shapes in reading order by position, tables, speaker notes (role `notes`), pictures by OCR. | |
54
+ | **ODP, PPT** | LibreOffice converts to PPTX, then the PPTX reader. | |
55
+ | **HTML** | The standard library's parser. | Encoding from the BOM, `<meta>` or windows-1255. Menus, page headers and footers get the role `navigation`. A page filled in by script is reported unread. Legislation's section numbers and margin headings are kept with their sections. |
56
+ | **Images** (PNG, JPEG, TIFF with all frames, GIF, BMP, JPEG 2000, WebP) | OCR, each frame a page. | DPI from the file, else 300 (flagged). |
57
+
58
+ PDF page classes:
59
+
60
+ | Class | What the page is | Route |
61
+ |---|---|---|
62
+ | T1 | A good text layer | Text layer |
63
+ | T2 | A text layer in visual order | Text layer, reordered (or OCR when words are drawn reversed) |
64
+ | T3 | A broken font mapping | Repairs; OCR for the codes left |
65
+ | T4 | Someone else's OCR layer | Ignored; OCR |
66
+ | T5 | An image with no text layer (a scan) | OCR |
67
+ | T6 | A text layer with significant images | Text layer, plus region OCR |
68
+
69
+ ## What the record tells you
70
+
71
+ - **Every block** carries its `source` (`text_layer`, `office_xml`, `converted_docx`/`xlsx`/`pptx`, `html` or `ocr`) and its `tool`. OCR blocks are `unverified`, with their engine, settings and confidence.
72
+ - **Roles:** body text is `body`. `header`, `footer`, `footnote`, `endnote`, `comment`, `textbox`, `navigation` and `notes` are kept in the record and out of the body flow.
73
+ - **Repairs** fix characters the source meant but its font or producer broke, such as reversed brackets, `ð` for `נ`, symbol-font bullets, private-use codes and reversed digit groups. Each one fires only on evidence, and is logged with its counts. `--no-repairs` turns them all off: the output is then the extracted text, byte for byte.
74
+ - **Flags** say where the reading is a guess. For example:
75
+ - `reading_order_unresolved_columns`: side-by-side text whose order is unknown;
76
+ - `unruled_table`: a table inferred from layout alone, so its page is never `selection_safe`;
77
+ - `ocr_short_for_ink`: ink that OCR left unread.
78
+
79
+ ### `unread`: what was not read, and why
80
+
81
+ Every entry has a scope (`document`, `page` or `block`), a reason, and where possible the page and a box. The main reasons:
82
+
83
+ | Reason | When |
84
+ |---|---|
85
+ | `format_not_supported` | A format with no route yet (EPUB, email, archives) |
86
+ | `converter_unavailable`, `conversion_failed`, `conversion_timeout` | LibreOffice missing, refusing the file, or stopped at its time limit |
87
+ | `ocr_route_unavailable`, `ocr_failed`, `ocr_found_no_text`, `ocr_short_for_ink` | Tesseract missing or failing, or a page whose ink OCR did not read |
88
+ | `image_region_*`, `image_*` | Image regions and embedded images: not read, too small, or low confidence. A region whose text is already in the layer, or that is blank, is no loss: it is counted in the page's coverage (`image_regions_text_in_layer`, `image_regions_blank`), not here |
89
+ | `table_row_ink_unread` | A table row with ink in its cells that OCR read none of |
90
+ | `table_cell_unconfirmed` | A scanned table cell whose two readings (`heb` and the number model) differ, neither confident; both are kept in the cell |
91
+ | `rotated_text_not_read` | Characters drawn at an angle other than vertical. Vertical text (90 or 270 degrees: a margin note, a stamp's caption) is read as its own lines after the page's body, flagged `rotated_text` |
92
+ | `text_outside_page` | Characters drawn outside the page's visible box |
93
+ | `html_rendered_by_script` | A web page whose content is filled in by JavaScript; print it to PDF from a browser |
94
+ | `page_limit`, `time_limit`, `sheet_truncated`, `html_truncated`, `merges_not_applied`, `image_too_large` | A limit was reached (below) |
95
+ | `docx_unreadable`, `xlsx_unreadable`, `pptx_unreadable`, `html_unreadable`, `image_unreadable` | A file that does not parse |
96
+ | `chart_not_read`, `smartart_not_read`, `xlsx_drawing_not_read`, `embedded_content_not_read` | Charts, drawings, frames and embedded objects |
97
+
98
+ ## Limits
99
+
100
+ Each limit, when reached, gives an `unread` entry or a flag:
101
+
102
+ - **pages or image frames:** 2,000 (`max_pages`);
103
+ - **time per document:** none by default; `time_limit` (seconds, `--time-limit`) is a safety net, checked before each page, frame and embedded image, and the step under way finishes;
104
+ - **input size:** 512 MB;
105
+ - **LibreOffice:** 180 s per conversion (`convert_timeout`); the conversion's processes are stopped;
106
+ - **HTML:** 64M characters, 2M elements, nesting depth 100 (flattened, with no text lost);
107
+ - **XLSX:** 2M cells per sheet;
108
+ - **PPTX:** group nesting depth 100;
109
+ - **cell spans:** 1,000;
110
+ - **renders for OCR:** 300 dpi, capped at 40M pixels.
111
+
112
+ What is not detected:
113
+
114
+ - **Pointed text in a kept OCR layer.** Some scans carry someone else's OCR layer with niqqud or cantillation, with each mark in its own box, as Internet Archive's scans do. heocr keeps such a layer, but builds the marks into separate pieces, so pointed lines come out fragmented: on the pointed pages we tested, a median of under half of the layer's words came through. Unpointed layers are not affected. There is no switch to force OCR yet: convert those pages to images and read the images, which go through OCR. A fix is planned for the next release.
115
+ - **A partial OCR layer:** a scanned page under someone else's OCR layer that heocr keeps is flagged `ocr_short_for_ink` when at least 10% of the scan's ink lies away from the layer's lines and rulings. A line or two missing from a full page's layer stays under that, and is not detected.
116
+
117
+ Read once:
118
+
119
+ - **A repeated image:** an image drawn again in the same document (a letterhead, a logo, a stamp) is read by region OCR where it first appears. Each later page gets a copy of that reading in its own box, flagged `repeated_image_text` and pointing to the first page (`checks.repeated_image`), with no OCR: every page keeps its text, as running headers do. A first reading that was empty or unsure repeats as counts (`image_regions_repeated`), not as new `unread` entries.
120
+
121
+ ## Optional tools
122
+
123
+ The core runs without them. Without them, the parts that need them are reported in `unread`, never guessed.
124
+
125
+ - **Tesseract** (Apache-2.0), for OCR, with the `heb` model (and `eng` for the second reading of numbers). heocr finds it on the `PATH`, in `$HEOCR_TESSERACT`, or through `tesseract=` / `--tesseract`. Other options:
126
+ - `ocr_lang` / `--ocr-lang` for the languages (default `heb`);
127
+ - `--ocr-model` for a HeOCR model, kept in `$HEOCR_MODELS`.
128
+ - **LibreOffice** (MPL-2.0), for DOC, RTF, ODT, ODS, XLS, ODP and PPT. heocr finds it on the `PATH`, in `$HEOCR_SOFFICE`, or through `soffice=` / `--soffice`; on macOS also in `/Applications`. It runs with a throwaway profile and a time limit. `convert=False` / `--no-convert` turns conversion off.
129
+
130
+ Neither is bundled. heocr records the version and executable hash of each one it uses in the record's `producer.components`.
131
+
132
+ ## Licence and credits
133
+
134
+ Apache-2.0. The core depends only on permissive libraries: pypdfium2, Pillow and defusedxml. Copyleft tools are never in a default route. See [NOTICE](NOTICE), and [THIRD_PARTY_LICENSES.md](THIRD_PARTY_LICENSES.md) for the audit of the dependencies and the libraries they bundle.
135
+
136
+ ## Project
137
+
138
+ `heocr` is part of [HeOCR](https://github.com/HeOCR), an open Hebrew OCR project. Its plan is in [`planning/`](planning/). The project-wide decisions behind it (0019, 0020) are in [heocr-engine](https://github.com/HeOCR/heocr-engine/tree/main/planning/decisions).
@@ -0,0 +1,88 @@
1
+ # Third-party licences: an audit of what `pip install heocr` brings
2
+
3
+ heocr is Apache-2.0. **Its wheel is pure Python and bundles no third-party code or binary.** It depends on three packages, which bring their own compiled libraries. Decision 0020 keeps the default install free of copyleft. This file records how that was checked, so the claim is checked rather than assumed.
4
+
5
+ - **Audited:** 2026-11-17, with heocr 0.0.1.
6
+ - **Versions:** the versions pip resolved then, Pillow 13.0.0 and pypdfium2 5.15.2. heocr accepts `Pillow>=10` and `pypdfium2>=4.30`.
7
+ - **Wheels inspected:** macOS arm64, manylinux x86_64 and Windows amd64.
8
+ - **Licence texts:** taken from each wheel's own licence files (`*.dist-info/licenses/`).
9
+ - **In CI:** `scripts/package_smoke.py` checks every runtime dependency's declared licence against an allow-list on every CI run.
10
+
11
+ ## Runtime dependencies
12
+
13
+ | Package | Licence | What it bundles |
14
+ |---|---|---|
15
+ | **defusedxml** 0.7.1 | PSF-2.0 | Nothing (pure Python) |
16
+ | **Pillow** 13.0.0 | MIT-CMU (HPND) | Image libraries, listed below. They are shared libraries on macOS and Linux, and linked into the extension module on Windows |
17
+ | **pypdfium2** 5.15.2 | Apache-2.0 or BSD-3-Clause; the binary under its dependencies' licences | **PDFium** (`libpdfium.dylib` / `.so` / `pdfium.dll`), with the libraries listed below built in |
18
+
19
+ ## Libraries inside pypdfium2's PDFium
20
+
21
+ These are from the wheel's `BUILD_LICENSES`.
22
+
23
+ | Library | Licence |
24
+ |---|---|
25
+ | PDFium | BSD-3-Clause |
26
+ | Abseil, Highway | Apache-2.0 |
27
+ | AGG 2.3 (anti-aliasing) | AGG licence (permissive, BSD-style) |
28
+ | Dragonbox | Apache-2.0 with LLVM exception, or Boost |
29
+ | fast_float | MIT, Apache-2.0 or Boost |
30
+ | FreeType | The FreeType Project License (FTL), chosen over its GPLv2 alternative; its acknowledgement is below |
31
+ | HarfBuzz | MIT ("Old MIT") |
32
+ | ICU | Unicode License v3 |
33
+ | Little CMS | MIT |
34
+ | libjpeg-turbo | IJG, BSD-3-Clause and zlib |
35
+ | OpenJPEG | BSD-2-Clause |
36
+ | libpng | PNG Reference Library License v2 |
37
+ | libtiff | libtiff licence (BSD-style) |
38
+ | simdutf | Apache-2.0 or MIT |
39
+ | zlib | zlib |
40
+ | Brotli | MIT |
41
+ | libyuv | BSD-3-Clause |
42
+ | LLVM libc pieces | Apache-2.0 with LLVM exception |
43
+
44
+ ## Libraries inside Pillow
45
+
46
+ These are from the wheel's `LICENSE` and its bundled libraries.
47
+
48
+ | Library | Licence |
49
+ |---|---|
50
+ | libjpeg-turbo | IJG, BSD-3-Clause and zlib |
51
+ | libtiff | libtiff licence (BSD-style) |
52
+ | libwebp (with libsharpyuv, webpdemux, webpmux) | BSD-3-Clause |
53
+ | OpenJPEG | BSD-2-Clause |
54
+ | libpng | PNG Reference Library License v2 |
55
+ | zlib-ng / zlib | zlib |
56
+ | FreeType | FTL, chosen over GPLv2 |
57
+ | HarfBuzz | MIT |
58
+ | Little CMS | MIT |
59
+ | libavif | BSD-2-Clause |
60
+ | Brotli | MIT |
61
+ | liblzma (XZ Utils) | Public domain (0BSD). XZ's GPL and LGPL files are tools and build scripts that, as its notice says, do not end up in the binaries |
62
+ | Zstandard (Linux wheels) | BSD-3-Clause, chosen over GPLv2 |
63
+ | libxcb, libXau (macOS and Linux wheels) | MIT (X11) |
64
+
65
+ ## Copyleft terms seen, and why none apply
66
+
67
+ - **Dual licences.** FreeType and Zstandard are dual-licensed: their permissive licences (FTL, BSD-3-Clause) apply. Nothing requires the GPLv2 alternative.
68
+ - **XZ Utils.** It names GPL and LGPL files that are not in the shipped library.
69
+ - **Libraries Pillow can load, but doesn't ship.** Some optional libraries are LGPL, such as FriBiDi for complex text layout. Pillow loads them at run time only when the user installs them; its wheels do not ship them, and heocr does not use them.
70
+
71
+ ## Acknowledgements the licences ask for
72
+
73
+ The FreeType Project License asks for this line in the documentation of software that uses it. FreeType comes inside PDFium (pypdfium2) and inside Pillow:
74
+
75
+ > Portions of this software are copyright © 2025 The FreeType Project (www.freetype.org). All rights reserved.
76
+
77
+ ## External programs (not installed by pip)
78
+
79
+ heocr runs these when they are installed. It never bundles them. See NOTICE.
80
+
81
+ | Program | Licence | Used for |
82
+ |---|---|---|
83
+ | Tesseract OCR | Apache-2.0 | OCR (optional) |
84
+ | LibreOffice | MPL-2.0 | DOC, RTF, ODT, ODS, XLS, ODP and PPT conversion (optional) |
85
+
86
+ ## When to redo this audit
87
+
88
+ Redo it whenever a dependency is added, or a minimum version raised, and before each release (D5f). A new runtime dependency must also pass the allow-list in `scripts/package_smoke.py`.