nuqta-ocr 0.1.0__tar.gz

nuqta_ocr-0.1.0/.gitignore

@@ -0,0 +1,6 @@
+__pycache__/
+*.egg-info/
+dist/
+.venv/
+.pytest_cache/
+landing/

nuqta_ocr-0.1.0/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

nuqta_ocr-0.1.0/PKG-INFO

@@ -0,0 +1,134 @@
+Metadata-Version: 2.5
+Name: nuqta-ocr
+Version: 0.1.0
+Summary: Nuqta OCR: accurate Arabic document OCR with faithful tashkeel (open 0.8B and 4B models)
+Project-URL: Homepage, https://huggingface.co/collections/SalahALHaismawi/nuqta-ocr-6ad4af255195b706ac93d1d4
+Project-URL: Repository, https://github.com/SalahAlHaismawi/nuqta-ocr
+Project-URL: Demo, https://huggingface.co/spaces/SalahALHaismawi/Nuqta-OCR
+Project-URL: Model 4B, https://huggingface.co/SalahALHaismawi/Nuqta-OCR-4B
+Project-URL: Model 0.8B, https://huggingface.co/SalahALHaismawi/Nuqta-OCR-0.8B
+Author: Salah AlHaismawi, Mohamed Saleh Tulib, Ahmed ElSayed Mohamed Ibrahim
+License-Expression: Apache-2.0
+License-File: LICENSE
+Keywords: arabic,arabic-ocr,diacritics,document-ai,handwriting,mrz,ocr,passport,pdf-to-markdown,tashkeel,vlm
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Natural Language :: Arabic
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Scientific/Engineering :: Image Recognition
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.10
+Requires-Dist: accelerate>=1.0
+Requires-Dist: pillow>=10
+Requires-Dist: pypdfium2>=4
+Requires-Dist: torch>=2.4
+Requires-Dist: torchvision
+Requires-Dist: transformers>=4.57
+Provides-Extra: bench
+Requires-Dist: apted>=1.0.3; extra == 'bench'
+Requires-Dist: beautifulsoup4>=4.12; extra == 'bench'
+Requires-Dist: distance>=0.1.3; extra == 'bench'
+Requires-Dist: httpx>=0.27; extra == 'bench'
+Requires-Dist: huggingface-hub>=0.24; extra == 'bench'
+Requires-Dist: levenshtein>=0.25; extra == 'bench'
+Requires-Dist: lxml>=5; extra == 'bench'
+Requires-Dist: pyarrow>=14; extra == 'bench'
+Requires-Dist: sacrebleu<3,>=2.4; extra == 'bench'
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == 'dev'
+Provides-Extra: fast
+Requires-Dist: causal-conv1d>=1.5; extra == 'fast'
+Requires-Dist: flash-linear-attention>=0.5; extra == 'fast'
+Provides-Extra: vllm
+Requires-Dist: vllm>=0.19; extra == 'vllm'
+Description-Content-Type: text/markdown
+
+<p align="center"><img src="https://raw.githubusercontent.com/SalahAlHaismawi/nuqta-ocr/main/assets/nuqta_logo.png" alt="Nuqta OCR" width="440"></p>
+
+# Nuqta OCR
+
+Accurate Arabic document OCR with **faithful tashkeel**, in two open models: **Nuqta-OCR 4B** (#1 on KITAB-Bench and Misraj-DocOCR among published results) and **Nuqta-OCR 0.8B** (best for its size: ahead of Gemini-2.0-Flash, level with or slightly ahead of Baseer, runs on phones).
+
+- Reads books, scans, official documents, tables, IDs and passports
+- Copies the tashkeel printed on the page and almost never adds any (0.08–0.16 invented marks per 100 letters)
+- Markdown output with HTML tables, in reading order
+- Runs on-premises on a single GPU: your documents never leave your network
+
+## Install
+
+```bash
+pip install "nuqta-ocr[vllm]"   # recommended: fast, batched (Linux + NVIDIA GPU)
+pip install "nuqta-ocr[fast]"   # transformers + fast kernels (no vLLM needed)
+pip install nuqta-ocr           # transformers only (slow on long pages)
+```
+
+## Quick start
+
+```python
+from nuqta import OCR
+
+if __name__ == "__main__":          # required with the vLLM backend (it starts worker processes)
+    ocr = OCR("nuqta-0.8b")              # or "nuqta-4b" (most accurate)
+
+    print(ocr.read("page.jpg").text)                        # as written: tashkeel only where printed
+    print(ocr.read("scan.pdf", mode="markdown").text)       # headings, lists, HTML tables; every page
+    print(ocr.read("passport.jpg", mode="id").fields)       # MRZ fields + check digits
+
+    results = ocr.read_many(["a.jpg", "b.png", "c.pdf"])    # batched with vLLM
+```
+
+Command line:
+
+```bash
+nuqta read page.jpg
+nuqta read scan.pdf --mode markdown
+nuqta read passport.jpg --mode id --json
+```
+
+## Modes
+
+| mode | output |
+|---|---|
+| `as-written` (default) | the text exactly as printed, tashkeel only where it is printed |
+| `bare` | the text without any tashkeel |
+| `markdown` | headings, paragraphs, lists and tables (HTML) in reading order |
+| `id` | `markdown` reading + structured fields: passport / visa / ID-card MRZ with ICAO check digits; Egyptian national ID fields cross-checked against the national number |
+
+## What the library does for you
+
+- **Page rule:** full pages are shown to the model at its training resolution (2.4 MP); word, line and card crops keep their size. Small page renders (e.g. 72-dpi PDF pages) otherwise lose letters.
+- **Loop guard:** if a page reading repeats itself, the page is re-read at its native size; any repetition left is trimmed (`Page.looped` tells you).
+- **PDFs:** every page rendered at 200 dpi.
+- **ID checks:** a failed check digit or an inconsistent national number is reported, never silently "fixed". Send those fields to a human.
+
+## Speed (0.8B, one 24 GB GPU)
+
+| | vLLM | transformers + `[fast]` |
+|---|---|---|
+| text line | ~0.2 s | ~1 s |
+| book page | ~3 s | ~3.5 s |
+| markdown document page | ~3 s | ~6 s |
+
+## Results
+
+| | KITAB-Bench CER ↓ | Misraj-DocOCR MARS ↑ |
+|---|---|---|
+| **Nuqta-OCR 4B** | **0.106** | **82.8** |
+| **Nuqta-OCR 0.8B** | **0.117** | **77.1** |
+| Previous best (Gemini-2.0-Flash / Baseer) | 0.13 | 76.9 |
+
+Recommended input size (what the library does). The 0.8B's lead over Baseer is within the benchmark's noise.
+
+Full tables, confidence intervals and disclosures are in the model cards; the full benchmarks are
+[KITAB-Bench](https://github.com/mbzuai-oryx/KITAB-Bench) and [Misraj-DocOCR](https://huggingface.co/datasets/Misraj/Misraj-DocOCR).
+
+## License
+
+Code: Apache-2.0. Models: see their model cards.
+
+## Contributors
+
+Salah AlHaismawi · Mohamed Saleh Tulib · Ahmed ElSayed Mohamed Ibrahim

nuqta_ocr-0.1.0/README.md

@@ -0,0 +1,86 @@
+<p align="center"><img src="https://raw.githubusercontent.com/SalahAlHaismawi/nuqta-ocr/main/assets/nuqta_logo.png" alt="Nuqta OCR" width="440"></p>
+
+# Nuqta OCR
+
+Accurate Arabic document OCR with **faithful tashkeel**, in two open models: **Nuqta-OCR 4B** (#1 on KITAB-Bench and Misraj-DocOCR among published results) and **Nuqta-OCR 0.8B** (best for its size: ahead of Gemini-2.0-Flash, level with or slightly ahead of Baseer, runs on phones).
+
+- Reads books, scans, official documents, tables, IDs and passports
+- Copies the tashkeel printed on the page and almost never adds any (0.08–0.16 invented marks per 100 letters)
+- Markdown output with HTML tables, in reading order
+- Runs on-premises on a single GPU: your documents never leave your network
+
+## Install
+
+```bash
+pip install "nuqta-ocr[vllm]"   # recommended: fast, batched (Linux + NVIDIA GPU)
+pip install "nuqta-ocr[fast]"   # transformers + fast kernels (no vLLM needed)
+pip install nuqta-ocr           # transformers only (slow on long pages)
+```
+
+## Quick start
+
+```python
+from nuqta import OCR
+
+if __name__ == "__main__":          # required with the vLLM backend (it starts worker processes)
+    ocr = OCR("nuqta-0.8b")              # or "nuqta-4b" (most accurate)
+
+    print(ocr.read("page.jpg").text)                        # as written: tashkeel only where printed
+    print(ocr.read("scan.pdf", mode="markdown").text)       # headings, lists, HTML tables; every page
+    print(ocr.read("passport.jpg", mode="id").fields)       # MRZ fields + check digits
+
+    results = ocr.read_many(["a.jpg", "b.png", "c.pdf"])    # batched with vLLM
+```
+
+Command line:
+
+```bash
+nuqta read page.jpg
+nuqta read scan.pdf --mode markdown
+nuqta read passport.jpg --mode id --json
+```
+
+## Modes
+
+| mode | output |
+|---|---|
+| `as-written` (default) | the text exactly as printed, tashkeel only where it is printed |
+| `bare` | the text without any tashkeel |
+| `markdown` | headings, paragraphs, lists and tables (HTML) in reading order |
+| `id` | `markdown` reading + structured fields: passport / visa / ID-card MRZ with ICAO check digits; Egyptian national ID fields cross-checked against the national number |
+
+## What the library does for you
+
+- **Page rule:** full pages are shown to the model at its training resolution (2.4 MP); word, line and card crops keep their size. Small page renders (e.g. 72-dpi PDF pages) otherwise lose letters.
+- **Loop guard:** if a page reading repeats itself, the page is re-read at its native size; any repetition left is trimmed (`Page.looped` tells you).
+- **PDFs:** every page rendered at 200 dpi.
+- **ID checks:** a failed check digit or an inconsistent national number is reported, never silently "fixed". Send those fields to a human.
+
+## Speed (0.8B, one 24 GB GPU)
+
+| | vLLM | transformers + `[fast]` |
+|---|---|---|
+| text line | ~0.2 s | ~1 s |
+| book page | ~3 s | ~3.5 s |
+| markdown document page | ~3 s | ~6 s |
+
+## Results
+
+| | KITAB-Bench CER ↓ | Misraj-DocOCR MARS ↑ |
+|---|---|---|
+| **Nuqta-OCR 4B** | **0.106** | **82.8** |
+| **Nuqta-OCR 0.8B** | **0.117** | **77.1** |
+| Previous best (Gemini-2.0-Flash / Baseer) | 0.13 | 76.9 |
+
+Recommended input size (what the library does). The 0.8B's lead over Baseer is within the benchmark's noise.
+
+Full tables, confidence intervals and disclosures are in the model cards; the full benchmarks are
+[KITAB-Bench](https://github.com/mbzuai-oryx/KITAB-Bench) and [Misraj-DocOCR](https://huggingface.co/datasets/Misraj/Misraj-DocOCR).
+
+## License
+
+Code: Apache-2.0. Models: see their model cards.
+
+## Contributors
+
+Salah AlHaismawi · Mohamed Saleh Tulib · Ahmed ElSayed Mohamed Ibrahim

nuqta_ocr-0.1.0/llms.txt

@@ -0,0 +1,11 @@
+# Nuqta OCR
+
+> Nuqta OCR is an open-weights Arabic OCR system: Nuqta-OCR 4B (#1 on KITAB-Bench and Misraj-DocOCR among published results) and the compact Nuqta-OCR 0.8B (runs on phones as GGUF). It converts Arabic scans, photos and PDFs (books, official documents, tables, IDs and passports) into text or Markdown with HTML tables, and copies the tashkeel (diacritics) printed on the page, almost never adding any. Apache-2.0.
+
+- Install: `pip install "nuqta-ocr[vllm]"`; use: `from nuqta import OCR; OCR("nuqta-4b").read("page.jpg").text`
+- Modes: as-written (tashkeel as printed), bare, markdown (tables as HTML), id (MRZ + ID fields with check digits)
+- Results 4B: KITAB-Bench CER 0.106 (Gemini-2.0-Flash 0.13); Misraj-DocOCR MARS 82.8 (Baseer 76.9), TEDS 76.1, CER 0.121
+- Results 0.8B: KITAB-Bench CER 0.117; Misraj-DocOCR MARS 77.1 (within noise of Baseer)
+- Models: https://huggingface.co/SalahALHaismawi/Nuqta-OCR-4B, https://huggingface.co/SalahALHaismawi/Nuqta-OCR-0.8B
+- GGUF (llama.cpp, offline): https://huggingface.co/SalahALHaismawi/Nuqta-OCR-0.8B-GGUF, https://huggingface.co/SalahALHaismawi/Nuqta-OCR-4B-GGUF
+- Demo: https://huggingface.co/spaces/SalahALHaismawi/Nuqta-OCR · Code: https://github.com/SalahAlHaismawi/nuqta-ocr

nuqta_ocr-0.1.0/pyproject.toml

@@ -0,0 +1,65 @@
+[project]
+name = "nuqta-ocr"
+version = "0.1.0"
+description = "Nuqta OCR: accurate Arabic document OCR with faithful tashkeel (open 0.8B and 4B models)"
+readme = "README.md"
+license = "Apache-2.0"
+requires-python = ">=3.10"
+authors = [{ name = "Salah AlHaismawi" }, { name = "Mohamed Saleh Tulib" }, { name = "Ahmed ElSayed Mohamed Ibrahim" }]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Intended Audience :: Developers",
+  "Intended Audience :: Science/Research",
+  "License :: OSI Approved :: Apache Software License",
+  "Natural Language :: Arabic",
+  "Programming Language :: Python :: 3",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+  "Topic :: Scientific/Engineering :: Image Recognition",
+  "Topic :: Text Processing :: Linguistic",
+]
+keywords = ["arabic", "ocr", "arabic-ocr", "tashkeel", "diacritics", "document-ai", "pdf-to-markdown", "handwriting", "passport", "mrz", "vlm"]
+dependencies = [
+  "pillow>=10",
+  "pypdfium2>=4",
+  "torch>=2.4",
+  "torchvision",
+  "transformers>=4.57",
+  "accelerate>=1.0",
+]
+
+[project.optional-dependencies]
+vllm = ["vllm>=0.19"]
+fast = ["flash-linear-attention>=0.5", "causal-conv1d>=1.5"]  # fast kernels for the transformers backend
+dev = ["pytest>=8"]
+# benchmarks/ (run.py + score.py): dataset download, endpoint client and the official KITAB-Bench / Misraj scoring
+bench = [
+  "huggingface_hub>=0.24",
+  "pyarrow>=14",
+  "httpx>=0.27",
+  "levenshtein>=0.25",       # python-Levenshtein's maintained name, same distance
+  "sacrebleu>=2.4,<3",       # KITAB-Bench pins 2.5.1; the published numbers were scored with 2.6.0
+  "apted>=1.0.3",            # TEDS (KITAB-Bench pdfs-eval/teds_metric.py)
+  "distance>=0.1.3",
+  "lxml>=5",
+  "beautifulsoup4>=4.12",
+]
+
+[project.urls]
+Homepage = "https://huggingface.co/collections/SalahALHaismawi/nuqta-ocr-6ad4af255195b706ac93d1d4"
+Repository = "https://github.com/SalahAlHaismawi/nuqta-ocr"
+Demo = "https://huggingface.co/spaces/SalahALHaismawi/Nuqta-OCR"
+"Model 4B" = "https://huggingface.co/SalahALHaismawi/Nuqta-OCR-4B"
+"Model 0.8B" = "https://huggingface.co/SalahALHaismawi/Nuqta-OCR-0.8B"
+
+[project.scripts]
+nuqta = "nuqta.cli:main"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/nuqta"]
+
+[tool.hatch.build.targets.sdist]
+include = ["/src/nuqta", "/tests", "/README.md", "/LICENSE", "/pyproject.toml", "/llms.txt"]

nuqta_ocr-0.1.0/src/nuqta/__init__.py

@@ -0,0 +1,7 @@
+"""Nuqta OCR: accurate Arabic document OCR with faithful tashkeel."""
+from .ocr import OCR, Page, Result, read_fields
+from .preprocess import is_page, prepare
+from .text import MODES, PROMPTS, looped, trim_loops
+
+__all__ = ["OCR", "Page", "Result", "read_fields", "is_page", "prepare", "MODES", "PROMPTS", "looped", "trim_loops"]
+__version__ = "0.1.0"

nuqta_ocr-0.1.0/src/nuqta/backends.py

@@ -0,0 +1,79 @@
+"""Generation backends: Hugging Face transformers (default) and vLLM (fast batched serving, optional).
+
+Both decode greedily (temperature 0, no repetition penalty), with thinking disabled: the settings the published
+benchmark numbers were measured with.
+"""
+from __future__ import annotations
+
+import base64
+import io
+
+from PIL import Image
+
+MAX_NEW_TOKENS = 8192
+
+
+def _data_url(im: Image.Image) -> str:
+    buf = io.BytesIO()
+    im.save(buf, format="PNG")
+    return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode()
+
+
+class TransformersBackend:
+    def __init__(self, model: str, device_map: str = "auto", dtype: str = "bfloat16"):
+        import torch
+        from transformers import AutoModelForImageTextToText, AutoProcessor
+
+        self.torch = torch
+        self.processor = AutoProcessor.from_pretrained(model)
+        self.model = AutoModelForImageTextToText.from_pretrained(model, dtype=getattr(torch, dtype),
+                                                                 device_map=device_map)
+        self.model.eval()
+        # stop at the end of the answer turn: the checkpoints' generation_config names only <|endoftext|>, so without
+        # this generate() runs on into new chat turns up to max_new_tokens (vLLM stops via the chat template)
+        tok = self.processor.tokenizer
+        self.eos = [i for i in {tok.convert_tokens_to_ids("<|im_end|>"), tok.eos_token_id,
+                                tok.convert_tokens_to_ids("<|endoftext|>")} if isinstance(i, int) and i >= 0]
+
+    def generate(self, items: list[tuple[Image.Image, str]], max_new_tokens: int = MAX_NEW_TOKENS) -> list[str]:
+        out = []
+        for im, prompt in items:  # one at a time: pages differ a lot in size, padding a batch wastes memory
+            msgs = [{"role": "user", "content": [{"type": "image", "image": im}, {"type": "text", "text": prompt}]}]
+            inputs = self.processor.apply_chat_template(msgs, add_generation_prompt=True, tokenize=True,
+                                                        return_dict=True, return_tensors="pt",
+                                                        enable_thinking=False).to(self.model.device)
+            with self.torch.inference_mode():
+                ids = self.model.generate(**inputs, max_new_tokens=max_new_tokens, do_sample=False, eos_token_id=self.eos,
+                                         pad_token_id=self.eos[0])
+            out.append(self.processor.decode(ids[0, inputs["input_ids"].shape[1]:], skip_special_tokens=True))
+        return out
+
+
+class VLLMBackend:
+    def __init__(self, model: str, gpu_memory_utilization: float = 0.85, max_model_len: int = 16384, **kw):
+        from vllm import LLM, SamplingParams
+
+        self.llm = LLM(model=model, gpu_memory_utilization=gpu_memory_utilization, max_model_len=max_model_len,
+                       limit_mm_per_prompt={"image": 1}, **kw)
+        self.SamplingParams = SamplingParams
+
+    def generate(self, items: list[tuple[Image.Image, str]], max_new_tokens: int = MAX_NEW_TOKENS) -> list[str]:
+        convs = [[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": _data_url(im)}},
+                                                {"type": "text", "text": prompt}]}] for im, prompt in items]
+        sp = self.SamplingParams(temperature=0.0, max_tokens=max_new_tokens, repetition_penalty=1.0)
+        res = self.llm.chat(convs, sp, chat_template_kwargs={"enable_thinking": False}, use_tqdm=False)
+        return [r.outputs[0].text for r in res]
+
+
+def make_backend(model: str, backend: str = "auto", **kw):
+    if backend == "auto":
+        try:
+            import vllm  # noqa: F401
+            backend = "vllm"
+        except ImportError:
+            backend = "transformers"
+    if backend == "vllm":
+        return VLLMBackend(model, **kw)
+    if backend == "transformers":
+        return TransformersBackend(model, **kw)
+    raise ValueError(f"unknown backend {backend!r} (auto, transformers, vllm)")

nuqta_ocr-0.1.0/src/nuqta/cli.py

@@ -0,0 +1,42 @@
+"""nuqta read FILE [FILE ...] [--mode as-written|bare|markdown|id] [--model PATH_OR_HUB_ID] [--json]"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from .text import MODES
+
+DEFAULT_MODEL = "nuqta-0.8b"
+
+
+def main(argv: list[str] | None = None) -> None:
+    ap = argparse.ArgumentParser(prog="nuqta", description="Nuqta OCR: Arabic document OCR with faithful tashkeel")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    r = sub.add_parser("read", help="read images / PDFs and print the text")
+    r.add_argument("files", nargs="+")
+    r.add_argument("--mode", default="as-written", choices=MODES + ("id",))
+    r.add_argument("--model", default=DEFAULT_MODEL)
+    r.add_argument("--backend", default="auto", choices=["auto", "transformers", "vllm"])
+    r.add_argument("--no-enlarge", action="store_true", help="read pages at their native size")
+    r.add_argument("--json", action="store_true", help="one JSON object per file (text, pages, fields)")
+    a = ap.parse_args(argv)
+
+    from .ocr import OCR
+
+    ocr = OCR(a.model, backend=a.backend, enlarge_pages=not a.no_enlarge)
+    for f, res in zip(a.files, ocr.read_many(a.files, mode=a.mode)):
+        if a.json:
+            print(json.dumps({"file": f, "mode": res.mode, "text": res.text, "fields": res.fields,
+                              "pages": [vars(p) for p in res.pages]}, ensure_ascii=False, default=str))
+        else:
+            if len(a.files) > 1:
+                print(f"==> {f} <==")
+            print(res.text)
+            if res.fields:
+                print(json.dumps(res.fields, ensure_ascii=False, indent=1, default=str))
+    sys.stdout.flush()
+
+
+if __name__ == "__main__":
+    main()

nuqta_ocr-0.1.0/src/nuqta/egyid.py

@@ -0,0 +1,151 @@
+"""Egyptian national ID card (بطاقة تحقيق الشخصية) reading -> fields (2026-10-14).
+
+The card prints no field labels: the front is header, first name, the rest of the name, address lines, (birth date),
+the 14-digit national number and the card serial; the back is the national number, the issue month, profession lines,
+sex, religion, marital status and «البطاقة سارية حتى <expiry>». Fields are taken by line position and shape.
+
+The national number is self-describing: digit 1 the century (2 = 19xx, 3 = 20xx), 2-7 the birth date YYMMDD, 8-9 the
+governorate of birth, 13 odd for a man, even for a woman. `nid_info` decodes it; the decoded birth date / sex /
+governorate cross-check the printed values (a misread digit rarely keeps them consistent; the address is where the
+holder lives now, so a different governorate is no error).
+"""
+import re
+from datetime import date
+
+_AI = str.maketrans("٠١٢٣٤٥٦٧٨٩۰۱۲۳۴۵۶۷۸۹", "01234567890123456789")
+_TO_AI = str.maketrans("0123456789", "٠١٢٣٤٥٦٧٨٩")
+GOV = {"01": "القاهرة", "02": "الإسكندرية", "03": "بورسعيد", "04": "السويس", "11": "دمياط", "12": "الدقهلية",
+       "13": "الشرقية", "14": "القليوبية", "15": "كفر الشيخ", "16": "الغربية", "17": "المنوفية", "18": "البحيرة",
+       "19": "الإسماعيلية", "21": "الجيزة", "22": "بني سويف", "23": "الفيوم", "24": "المنيا", "25": "أسيوط",
+       "26": "سوهاج", "27": "قنا", "28": "أسوان", "29": "الأقصر", "31": "البحر الأحمر", "32": "الوادي الجديد",
+       "33": "مطروح", "34": "شمال سيناء", "35": "جنوب سيناء", "88": "خارج الجمهورية"}
+# spellings on cards / in addresses that differ from GOV («اسكندرية», «الاسماعيلية», «بنى سويف», ...)
+_GOV_KEYS = {g: {re.sub("[أإآ]", "ا", g).replace("ى", "ي"), re.sub("^ال", "", re.sub("[أإآ]", "ا", g)).replace("ى", "ي")}
+             for g in GOV.values()}
+_HEAD = re.compile(r"جمهورية\s+مصر\s+العربية|بطاقة\s+تحقيق\s+الشخصية")
+_NID = re.compile(r"(?<![\d٠-٩])[23٢٣](?:\s?[\d٠-٩]){13}(?![\d٠-٩])")
+_DATE = re.compile(r"[\d٠-٩]{4}\s*/\s*[\d٠-٩]{1,2}(?:\s*/\s*[\d٠-٩]{1,2})?")
+_SEX = {"ذكر", "أنثى", "انثى", "انثي"}
+_RELIGION = re.compile(r"^(?:مسلم|مسلمة|مسلمه|مسيحي|مسيحى|مسيحية|مسيحيه|يهودي|يهودى)$")
+_MARITAL = re.compile(r"^(?:أعزب|اعزب|عزباء|آنسة|انسة|آنسه|انسه|متزوج|متزوجة|متزوجه|مطلق|مطلقة|مطلقه|أرمل|ارمل|أرملة|ارملة|ارمله)$")
+_SERIAL = re.compile(r"^[A-Z]{1,3}\s?[\d٠-٩]{6,8}$|^[\d٠-٩]{8}[\d٠-٩A-Z]?$")
+
+
+def looks_egyptian(text: str) -> bool:
+    return bool(_HEAD.search(text) or ("سارية حتى" in text and _NID.search(text)))
+
+
+def nid_info(nid: str) -> dict | None:
+    """Decode a 14-digit national number; None when it cannot be one (bad century / date / governorate)."""
+    d = re.sub(r"\D", "", nid.translate(_AI))
+    if len(d) != 14 or d[0] not in "23":
+        return None
+    try:
+        dob = date((1900 if d[0] == "2" else 2000) + int(d[1:3]), int(d[3:5]), int(d[5:7]))
+    except ValueError:
+        return None
+    if d[7:9] not in GOV:
+        return None
+    return {"dob": dob, "gov": GOV[d[7:9]], "sex": "ذكر" if int(d[12]) % 2 else "أنثى"}
+
+
+def _date(v: str):
+    m = re.fullmatch(r"(\d{4})/(\d{1,2})/(\d{1,2})", re.sub(r"\s", "", v.translate(_AI)))
+    try:
+        return date(int(m[1]), int(m[2]), int(m[3])) if m else None
+    except ValueError:
+        return None
+
+
+def gov_in(gov: str, text: str) -> bool:
+    t = re.sub("[أإآ]", "ا", text).replace("ى", "ي")
+    return any(k in t for k in _GOV_KEYS.get(gov, ()))
+
+
+def parse(text: str) -> dict:
+    """Fields of an Egyptian ID front or back reading; {} when it is not one."""
+    if not looks_egyptian(text):
+        return {}
+    lines = [re.sub(r"\*\*|</?t[dhr]>|^#+\s*", "", x).strip() for x in text.split("\n")]
+    lines = [x for x in lines if x and not re.fullmatch(r"<img>.*?</img>|image here", x) and not _HEAD.fullmatch(x)]
+    f: dict[str, str] = {}
+    words, addr = [], []
+    back = any("سارية حتى" in x for x in lines) or any(x in _SEX or _RELIGION.match(x) for x in lines)
+    for x in lines:
+        if _HEAD.search(x):
+            continue
+        m = _NID.search(x)
+        if m and "nid" not in f:
+            f["nid"] = m.group(0)
+            continue
+        if "سارية حتى" in x:
+            d = _DATE.search(x)
+            if d:
+                f["doe"] = d.group(0)
+            else:
+                f["_doe_next"] = "1"
+            continue
+        d = _DATE.fullmatch(x)
+        if d:
+            if f.pop("_doe_next", None):
+                f["doe"] = x
+            elif back and x.count("/") + x.count("／") == 1:
+                f.setdefault("doi", x)  # the back prints the issue month «٢٠١٥/١١»
+            elif not back:
+                # newer fronts print the expiry too («٢٠٣٠/٠٢»): a date after the card could have been issued
+                y = int(re.sub(r"\D", "", x.translate(_AI))[:4])
+                f.setdefault("doe" if y > 2010 or x.count("/") == 1 else "dob", x)
+            else:
+                f.setdefault("doe" if "doi" in f else "doi", x)
+            continue
+        if _SERIAL.match(x) and "nid" in f and not back:
+            f.setdefault("no", x)
+            continue
+        if x in _SEX:
+            f["sex"] = x
+            continue
+        if _RELIGION.match(x):
+            f["religion"] = x
+            continue
+        if _MARITAL.match(x):
+            f["marital"] = x
+            continue
+        if re.fullmatch(r"[\d٠-٩]{1,2}", x):  # husband's name line / card copy number «١»
+            continue
+        if re.search("[ء-ي]", x):
+            (words if back else (words if len(words) < 2 and "nid" not in f and not addr else addr)).append(x)
+    f.pop("_doe_next", None)
+    if back:
+        if words:
+            f["prof"] = " ".join(words)
+    else:
+        if words:
+            f["name"] = " ".join(words)
+        if addr:
+            f["address"] = " ".join(addr)
+        f.setdefault("nat", "مصري")
+    return f
+
+
+def check(f: dict, conf: dict) -> None:
+    """Cross-check / fill from the national number: birth date, sex, governorate of birth (front address)."""
+    info = nid_info(f.get("nid", ""))
+    if not f.get("nid"):
+        return
+    if not info:
+        conf["nid"] = "check"  # cannot be a national number: a digit is misread
+        return
+    ok = True
+    if f.get("dob"):
+        ok &= _date(f["dob"]) == info["dob"]
+    else:
+        f["dob"] = info["dob"].strftime("%Y/%m/%d").translate(_TO_AI)
+        conf["dob"] = "nid"
+    if not f.get("pob"):
+        f["pob"], conf["pob"] = info["gov"], "nid"  # governorate of birth
+    if f.get("sex"):
+        ok &= (f["sex"] == "ذكر") == (info["sex"] == "ذكر")
+    else:
+        f["sex"], conf["sex"] = info["sex"], "nid"
+    # the governorate of birth is often the address one too, but people move: agreement is evidence, not a rule
+    conf["nid"] = ("verified" if conf.get("dob") != "nid" or gov_in(info["gov"], f.get("address", "")) else "agreed") if ok else "check"

nuqta_ocr-0.1.0/src/nuqta/mrz.py

@@ -0,0 +1,130 @@
+"""Passport / ID machine-readable zone (ICAO 9303) from an OCR reading (2026-10-13).
+
+parse_mrz(text) finds the MRZ lines in a model reading (TD3 passports 2 x 44, TD2 2 x 36, TD1 ID cards 3 x 30),
+(visas MRV-A 2 x 44 / MRV-B 2 x 36: document code V, no composite check digit)
+splits them into fields, checks every check digit and, given the whole reading, whether the number and dates also
+appear in the printed (visual) zone. Nothing is corrected silently: a failed check digit is reported as failed.
+The model may drop trailing '<' filler, so lines are padded to the format length; a field that falls in the
+missing part is reported as unread.
+"""
+import re
+from datetime import date
+
+_LINE = re.compile(r"^[A-Z0-9<]{20,50}$")
+_WEIGHTS = (7, 3, 1)
+_MONTHS = ["JAN", "FEB", "MAR", "APR", "MAY", "JUN", "JUL", "AUG", "SEP", "OCT", "NOV", "DEC"]
+_TAG = re.compile(r"</?(?:td|th|tr|p|br|div|span|b|strong|em|i|code)\b[^<>]*>", re.IGNORECASE)
+
+
+def check_digit(s: str) -> str:
+    """ICAO 9303 check digit: weights 7, 3, 1; '<' = 0, A-Z = 10-35."""
+    total = 0
+    for i, c in enumerate(s):
+        v = int(c) if c.isdigit() else (ord(c) - 55 if "A" <= c <= "Z" else 0)
+        total += v * _WEIGHTS[i % 3]
+    return str(total % 10)
+
+
+def _candidates(text: str) -> list[str]:
+    out = []
+    for raw in text.splitlines():
+        s = _TAG.sub("", raw)  # markup the reading may wrap around the line (<td>, <p>); MRZ '<' is not a tag
+        s = re.sub(r"\s+", "", s).upper().replace("«", "<").replace("‹", "<")
+        # a line with no '<' (TD3 line 2 with a full optional field) counts only when long and number-heavy
+        ok = _LINE.match(s) and ("<" in s or (len(s) >= 28 and sum(c.isdigit() for c in s) >= 6))
+        out.append(s if ok else "")
+    return out
+
+
+def _groups(lines: list[str]) -> list[list[str]]:
+    groups, cur = [], []
+    for s in lines + [""]:
+        if s:
+            cur.append(s)
+        elif cur:
+            if any("<" in x for x in cur):
+                groups.append(cur)
+            cur = []
+    return groups
+
+
+def _fmt(lines: list[str]) -> tuple[str, int] | None:
+    if len(lines) >= 3:
+        return "TD1", 30
+    if len(lines) == 2:
+        n = max(len(x) for x in lines)
+        return ("TD2", 36) if abs(n - 36) < abs(n - 44) and not lines[0].startswith("P") else ("TD3", 44)
+    return None
+
+
+def _names(s: str) -> tuple[str, str]:
+    s = s.rstrip("<")
+    sur, _, given = s.partition("<<")
+    return sur.replace("<", " ").strip(), given.replace("<", " ").strip()
+
+
+def _date(yymmdd: str, future: bool) -> str | None:
+    if not re.fullmatch(r"\d{6}", yymmdd):
+        return None
+    yy, mm, dd = int(yymmdd[:2]), int(yymmdd[2:4]), int(yymmdd[4:])
+    now = date.today().year % 100
+    year = 2000 + yy if (future or yy <= now) else 1900 + yy
+    try:
+        return date(year, mm, dd).strftime("%d-%m-%Y")
+    except ValueError:
+        return None
+
+
+def _check(value: str, digit: str) -> bool | None:
+    if not value or not digit or "<" * len(value) == value and digit == "<":
+        return None  # unread (missing filler part) or an empty optional field
+    return check_digit(value) == digit
+
+
+def parse_mrz(text: str) -> dict | None:
+    """The first MRZ block in `text` as fields + check-digit results, or None."""
+    for g in _groups(_candidates(text)):
+        f = _fmt(g)
+        if not f:
+            continue
+        kind, n = f
+        ls = [x[:n].ljust(n, "<") for x in g[: 3 if kind == "TD1" else 2]]
+        cut = [len(x) for x in g]  # how much of each line was actually read
+        if kind == "TD1":
+            a, b, c = ls
+            num, nd, dob, dd, sex, exp, ed = a[5:14], a[14], b[0:6], b[6], b[7], b[8:14], b[14]
+            out = {"format": "TD1 (ID card)", "doc_type": a[0:2].strip("<"), "issuer": a[2:5], "nationality": b[15:18]}
+            sur, given = _names(c)
+            comp = (a[5:30] + b[0:7] + b[8:15] + b[18:29], b[29]) if cut[1] >= 30 else None
+        else:
+            a, b = ls
+            num, nd, dob, dd, sex, exp, ed = b[0:9], b[9], b[13:19], b[19], b[20], b[21:27], b[27]
+            visa = a[0] == "V"  # MRV: the optional data runs to the end of line 2, there is no composite digit
+            out = {"format": ("MRV-A (visa)" if n == 44 else "MRV-B (visa)") if visa else "TD3 (passport)" if kind == "TD3" else "TD2",
+                   "doc_type": a[0:2].strip("<"), "issuer": a[2:5], "nationality": b[10:13]}
+            sur, given = _names(a[5:])
+            last = n - 1
+            comp = (b[0:10] + b[13:20] + b[21:last], b[last]) if cut[1] >= n and not visa else None
+        out.update({"surname": sur, "given_names": given, "number": num.rstrip("<"), "sex": sex.replace("<", "X"),
+                    "birth": _date(dob, False), "expiry": _date(exp, True)})
+        checks = {"number": _check(num, nd), "birth": _check(dob, dd), "expiry": _check(exp, ed)}
+        if comp:
+            checks["composite"] = _check(*comp)
+        out["checks"] = checks
+        out["valid"] = all(v is not False for v in checks.values()) and any(v for v in checks.values())
+        out["lines"] = g
+        # cross-check with the printed (visual) zone of the same reading (the MRZ lines themselves left out)
+        vis = "\n".join(x for x in text.splitlines() if re.sub(r"\s+", "", _TAG.sub("", x)).upper() not in g)
+        flat = re.sub(r"\s+", "", vis)
+        seen = {}
+        if out["number"]:
+            seen["number"] = out["number"] in flat
+        for k in ("birth", "expiry"):
+            if out[k]:
+                d, m, y = out[k].split("-")
+                mon = _MONTHS[int(m) - 1]
+                seen[k] = any(v in flat.upper() for v in (f"{d}-{m}-{y}", f"{d}/{m}/{y}", f"{d}.{m}.{y}", f"{y}-{m}-{d}",
+                                                          f"{y}/{m}/{d}", f"{d}{mon}{y}", f"{d}{mon}{y[2:]}"))
+        out["printed_match"] = seen
+        return out
+    return None

nuqta_ocr-0.1.0/src/nuqta/ocr.py

@@ -0,0 +1,123 @@
+"""Nuqta OCR: the main entry point."""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from PIL import Image
+
+from . import egyid, mrz
+from .backends import MAX_NEW_TOKENS, make_backend
+from .preprocess import load_image, plain, prepare
+from .text import MODES, PROMPTS, clean, looped, trim_loops
+
+
+@dataclass
+class Page:
+    text: str
+    looped: bool = False          # even the fallback reading repeated itself (text was trimmed)
+    fallback: bool = False        # the page was re-read at native size because the first reading looped
+
+
+@dataclass
+class Result:
+    pages: list[Page]
+    mode: str
+    fields: dict = field(default_factory=dict)  # mode="id" only
+
+    @property
+    def text(self) -> str:
+        return "\n\n".join(p.text for p in self.pages)
+
+    def __str__(self) -> str:
+        return self.text
+
+
+def _pdf_pages(path: Path, dpi: int = 200) -> list[Image.Image]:
+    import pypdfium2 as pdfium
+
+    doc = pdfium.PdfDocument(str(path))
+    return [doc[i].render(scale=dpi / 72).to_pil().convert("RGB") for i in range(len(doc))]
+
+
+MODELS = {"nuqta-0.8b": "SalahALHaismawi/Nuqta-OCR-0.8B", "nuqta-4b": "SalahALHaismawi/Nuqta-OCR-4B"}
+
+
+class OCR:
+    """Read Arabic documents.
+
+        ocr = OCR("nuqta-0.8b")                   # or "nuqta-4b", a hub id or a local path
+        ocr.read("page.jpg").text                 # as written: tashkeel only where printed
+        ocr.read("scan.pdf", mode="markdown")     # headings, lists, HTML tables, every page
+        ocr.read("passport.jpg", mode="id").fields
+
+    Modes: as-written (default), bare (no tashkeel), markdown, id.
+    backend: "auto" (vLLM when installed, else transformers), "transformers" or "vllm".
+    """
+
+    def __init__(self, model: str, backend: str = "auto", enlarge_pages: bool = True, **backend_kw):
+        self.backend = make_backend(MODELS.get(model.lower(), model), backend, **backend_kw)
+        self.enlarge_pages = enlarge_pages
+
+    # -- reading
+    def _read_images(self, images: list[Image.Image], prompt: str, max_new_tokens: int) -> list[Page]:
+        first = [prepare(im, self.enlarge_pages) for im in images]
+        outs = [clean(t) for t in self.backend.generate([(im, prompt) for im in first], max_new_tokens)]
+        pages = [Page(t) for t in outs]
+        # loop fallback: re-read a looping page at its native size (if that is a different input)
+        redo = [i for i, t in enumerate(outs) if looped(t) and first[i].size != plain(images[i]).size]
+        if redo:
+            again = [clean(t) for t in self.backend.generate([(plain(images[i]), prompt) for i in redo], max_new_tokens)]
+            for i, t in zip(redo, again):
+                if not looped(t):
+                    pages[i] = Page(t, fallback=True)
+        for p in pages:
+            if looped(p.text):
+                p.text, p.looped = trim_loops(p.text), True
+        return pages
+
+    def read(self, src, mode: str = "as-written", max_new_tokens: int = MAX_NEW_TOKENS) -> Result:
+        """Read one image (path, bytes or PIL image) or a PDF (every page)."""
+        return self.read_many([src], mode, max_new_tokens)[0]
+
+    def read_many(self, srcs: list, mode: str = "as-written", max_new_tokens: int = MAX_NEW_TOKENS) -> list[Result]:
+        """Read several files; with the vLLM backend they are batched."""
+        if mode not in MODES + ("id",):
+            raise ValueError(f"mode must be one of {MODES + ('id',)}")
+        prompt = PROMPTS["markdown" if mode == "id" else mode]
+        images, owner = [], []
+        for k, src in enumerate(srcs):
+            if isinstance(src, (str, Path)) and str(src).lower().endswith(".pdf"):
+                ims = _pdf_pages(Path(src))
+            else:
+                ims = [load_image(src)]
+            images += ims
+            owner += [k] * len(ims)
+        pages = self._read_images(images, prompt, max_new_tokens)
+        results = [Result([], mode) for _ in srcs]
+        for k, p in zip(owner, pages):
+            results[k].pages.append(p)
+        if mode == "id":
+            for r in results:
+                r.fields = read_fields(r.text)
+        return results
+
+
+def read_fields(text: str) -> dict:
+    """Structured fields from an ID / passport reading, with validity checks.
+
+    mrz:      the machine-readable zone (passports, visas, TD1 cards): fields + ICAO check digits ("valid") and
+              whether the number and dates also appear in the printed zone.
+    egyptian: Egyptian national ID fields; the 14-digit national number is decoded and cross-checked against the
+              printed birth date and sex ("nid": verified / agreed / check). "check" = a digit is probably misread.
+    A failed check means: send this field to a human. Nothing is corrected silently."""
+    out: dict = {}
+    m = mrz.parse_mrz(text)
+    if m:
+        out["mrz"] = {k: v for k, v in m.items() if k != "lines"}
+    e = egyid.parse(text)
+    if e:
+        conf: dict = {}
+        egyid.check(e, conf)
+        out["egyptian"] = {"fields": e, "checks": conf}
+    return out

nuqta_ocr-0.1.0/src/nuqta/preprocess.py

@@ -0,0 +1,60 @@
+"""Image input policy: what the model saw in training, so what it should see at inference.
+
+* Pages (page-shaped images: aspect <= 2 and short side >= 450 px) are enlarged to the training size, 2.4 MP.
+  Small page renders (e.g. 612x792 PDF pages with ~10 px text) otherwise lose letters and loop; on our held-out
+  page sets this never hurt and cut handwritten-page errors by 41%.
+* Crops (words, lines, paragraphs, cards: everything not page-shaped) keep their size. Enlarging crops hurt
+  (word crops: +90% CER at 2.4 MP).
+* Every image is capped at 2.4 MP, the training cap.
+* Transparent PNGs are composited onto white (a plain RGB conversion turns transparent pixels black).
+"""
+from __future__ import annotations
+
+import io
+from pathlib import Path
+
+from PIL import Image
+
+TRAIN_PIXELS = 2_400_000
+PAGE_MIN_SHORT_SIDE = 450
+PAGE_MAX_ASPECT = 2.0
+
+
+def load_image(src) -> Image.Image:
+    """A path, bytes or a PIL image -> an RGB PIL image."""
+    if isinstance(src, Image.Image):
+        im = src
+    elif isinstance(src, (bytes, bytearray)):
+        im = Image.open(io.BytesIO(src))
+    else:
+        im = Image.open(Path(src))
+    if im.mode in ("RGBA", "LA", "PA") or (im.mode == "P" and "transparency" in im.info):
+        im = im.convert("RGBA")
+        bg = Image.new("RGBA", im.size, (255, 255, 255, 255))
+        im = Image.alpha_composite(bg, im)
+    return im.convert("RGB")
+
+
+def is_page(im: Image.Image) -> bool:
+    w, h = im.size
+    return max(w, h) / max(min(w, h), 1) <= PAGE_MAX_ASPECT and min(w, h) >= PAGE_MIN_SHORT_SIDE
+
+
+def _scale_to(im: Image.Image, pixels: int) -> Image.Image:
+    k = (pixels / (im.width * im.height)) ** 0.5
+    return im.resize((max(32, int(im.width * k)), max(32, int(im.height * k))), Image.LANCZOS)
+
+
+def prepare(im: Image.Image, enlarge_pages: bool = True) -> Image.Image:
+    """The release input policy (see the module docstring)."""
+    area = im.width * im.height
+    if enlarge_pages and is_page(im) and area < TRAIN_PIXELS:
+        return _scale_to(im, TRAIN_PIXELS)
+    if area > TRAIN_PIXELS:
+        return _scale_to(im, TRAIN_PIXELS)
+    return im
+
+
+def plain(im: Image.Image) -> Image.Image:
+    """The fallback reading's input: native size, only capped."""
+    return _scale_to(im, TRAIN_PIXELS) if im.width * im.height > TRAIN_PIXELS else im

nuqta_ocr-0.1.0/src/nuqta/text.py

@@ -0,0 +1,64 @@
+"""Output clean-up and loop detection."""
+from __future__ import annotations
+
+import re
+
+PROMPTS = {
+    "as-written": "Transcribe the Arabic text in this image exactly as written. Include only the diacritics (tashkeel) "
+                  "that are visible in the image.",
+    "bare": "Transcribe the Arabic text in this image without any diacritics.",
+    "markdown": "Convert this document to Markdown: headings, paragraphs, lists and tables in reading order. Include "
+                "only the diacritics visible in the image.",
+}
+MODES = tuple(PROMPTS)
+
+_THINK = re.compile(r"<think>.*?</think>", re.S)
+_FENCE = re.compile(r"^```\w*\n?|\n?```$")
+_CHAR_RUN = re.compile(r"([^\s<])\1{199,}")  # 200+ of one character: dotted / underlined form lines are shorter
+_CYCLE = re.compile(r"(.{8,200}?)(?:\s*\1){2,}", re.S)
+LOOP_CHARS = 200  # a reading that loses more than this to trim_loops counts as looped
+END_REPEATS = 5  # a span repeated this often up to the very end of the output (cut off by the token limit) is a loop
+RUNAWAY_REPEATS = 20  # a span repeated this often is a loop even mid-text; fewer repeats only count at the very end
+LINE_REPEATS = 10  # a long line re-appearing this often is a loop (refrains and repeated table rows stay)
+
+
+def clean(text: str) -> str:
+    text = _THINK.sub("", text or "")
+    return _FENCE.sub("", text.strip()).strip()
+
+
+def trim_loops(text: str) -> str:
+    """Remove runaway repetition only. Genuine repeats stay: identical table rows, dotted form lines, refrains and MRZ
+    filler («<<<<»). A runaway is a span repeated back to back that either runs on to the end of the output (where
+    generation was cut off) END_REPEATS+ times or repeats RUNAWAY_REPEATS+ times anywhere; it keeps one copy. A run of 200+ identical characters
+    keeps one, and a long line (>= 40 chars) that appears LINE_REPEATS+ times is kept only at its first occurrence."""
+    text = _CHAR_RUN.sub(r"\1", text)
+
+    def cut(m):
+        unit = m.group(1)
+        if not unit.replace("<", "").strip():  # MRZ filler
+            return m.group(0)
+        reps = len(m.group(0)) / max(len(unit), 1)
+        at_end = m.end() >= len(m.string.rstrip()) - 1
+        return unit if ((at_end and reps >= END_REPEATS) or reps >= RUNAWAY_REPEATS) else m.group(0)
+
+    prev = None
+    while prev != text:
+        prev = text
+        text = _CYCLE.sub(cut, text)
+    lines = text.split("\n")
+    keys = [re.sub(r"\s+", " ", line).strip() for line in lines]
+    count = {}
+    for k in keys:
+        count[k] = count.get(k, 0) + 1
+    seen, out = set(), []
+    for line, k in zip(lines, keys):
+        if len(k) >= 40 and count[k] >= LINE_REPEATS and k in seen:
+            continue
+        seen.add(k)
+        out.append(line)
+    return "\n".join(out)
+
+
+def looped(text: str) -> bool:
+    return len(text) - len(trim_loops(text)) > LOOP_CHARS

nuqta_ocr-0.1.0/tests/test_basic.py

@@ -0,0 +1,49 @@
+from PIL import Image
+
+from nuqta import is_page, looped, prepare, read_fields, trim_loops
+from nuqta.preprocess import TRAIN_PIXELS, plain
+from nuqta.text import clean
+
+
+def test_page_rule_enlarges_small_pages_only():
+    page = Image.new("RGB", (612, 792), "white")  # a 72-dpi PDF page render
+    big = prepare(page)
+    assert is_page(page) and abs(big.width * big.height - TRAIN_PIXELS) / TRAIN_PIXELS < 0.01
+    line = Image.new("RGB", (1200, 80), "white")  # a text line crop: never enlarged
+    assert not is_page(line) and prepare(line).size == line.size
+    word = Image.new("RGB", (300, 120), "white")
+    assert prepare(word).size == word.size
+
+
+def test_cap_at_training_size():
+    huge = Image.new("RGB", (4000, 3000), "white")
+    for im in (prepare(huge), plain(huge)):
+        assert im.width * im.height <= TRAIN_PIXELS * 1.01
+
+
+def test_loops():
+    page = "السطر الأول من الصفحة\n" + "١ | ١ | " * 200
+    assert looped(page) and len(trim_loops(page)) < 100
+    assert not looped("نص عادي بلا تكرار " * 3)
+    assert trim_loops("P<UTO<<<<<<<<<<<<<<<<<<<<<<") == "P<UTO<<<<<<<<<<<<<<<<<<<<<<"  # MRZ filler is kept
+
+
+def test_clean():
+    assert clean("<think>x</think>```markdown\n# عنوان\n```") == "# عنوان"
+
+
+def test_mrz_icao_specimen():
+    text = "PASSPORT\nP<UTOERIKSSON<<ANNA<MARIA<<<<<<<<<<<<<<<<<<<\nL898902C36UTO7408122F1204159ZE184226B<<<<<10"
+    m = read_fields(text)["mrz"]
+    assert m["valid"] and m["number"] == "L898902C3" and m["surname"] == "ERIKSSON"
+
+
+def test_trim_loops_keeps_real_repeats_and_cuts_runaways():
+    from nuqta.text import looped, trim_loops
+    row = "<tr><td>قلم</td><td>1</td><td>5</td></tr>"
+    for keep in ("<table>" + row * 9 + "</table>\nالمجموع 45",            # identical table rows
+                 "الاسم: ...................................... التاريخ: ____________________",  # form lines
+                 "\n".join(["يا ليلُ الصبُّ متى غدُه أقيامُ الساعةِ موعدُهُ"] * 4)):  # refrain
+        assert trim_loops(keep) == keep and not looped(keep)
+    runaway = "مقدمة النص. " + "وهذا مثال متكرر " * 60
+    assert looped(runaway) and len(trim_loops(runaway)) < 60