unshape 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- unshape/__init__.py +7 -0
- unshape/data/ATTRIBUTION.md +13 -0
- unshape/data/train_bn.txt +3180 -0
- unshape/data/train_gu.txt +320 -0
- unshape/data/train_hi.txt +2399 -0
- unshape/data/train_kn.txt +425 -0
- unshape/data/train_ml.txt +1919 -0
- unshape/data/train_or.txt +1198 -0
- unshape/data/train_pa.txt +1623 -0
- unshape/data/train_si.txt +1664 -0
- unshape/data/train_ta.txt +2415 -0
- unshape/data/train_te.txt +2100 -0
- unshape/decode.py +553 -0
- unshape/fontfree.py +425 -0
- unshape/route.py +123 -0
- unshape/unshape_emb.py +233 -0
- unshape/unshape_emb3.py +93 -0
- unshape/unshape_emb4.py +88 -0
- unshape/unshape_multi4.py +326 -0
- unshape/unshape_multi4.rules.json +32 -0
- unshape/validity.py +118 -0
- unshape-0.5.0.dist-info/METADATA +216 -0
- unshape-0.5.0.dist-info/RECORD +28 -0
- unshape-0.5.0.dist-info/WHEEL +5 -0
- unshape-0.5.0.dist-info/entry_points.txt +4 -0
- unshape-0.5.0.dist-info/licenses/LICENSE +661 -0
- unshape-0.5.0.dist-info/licenses/NOTICE +16 -0
- unshape-0.5.0.dist-info/top_level.txt +1 -0
unshape/__init__.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# Copyright (c) 2026 Novain Shibu. AGPL-3.0-or-later, or a commercial licence (see COMMERCIAL.md).
|
|
2
|
+
"""unshape: exact Unicode text from complex-script PDFs with broken text layers, without OCR."""
|
|
3
|
+
import sys as _sys
|
|
4
|
+
from pathlib import Path as _Path
|
|
5
|
+
|
|
6
|
+
_sys.path.insert(0, str(_Path(__file__).resolve().parent)) # the research modules import each other by plain name
|
|
7
|
+
__version__ = "0.5.0"
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Attribution for unshape/data/train_<lang>.txt
|
|
2
|
+
|
|
3
|
+
These files contain plain text taken from articles of the Wikipedia editions in Bengali, Gujarati, Hindi, Kannada,
|
|
4
|
+
Malayalam, Odia, Punjabi, Sinhala, Tamil and Telugu, written by **Wikipedia contributors**.
|
|
5
|
+
|
|
6
|
+
- Source: https://www.wikipedia.org/ (the respective language editions), taken from the Hugging Face dataset
|
|
7
|
+
`wikimedia/wikipedia`, configuration `20231101.<lang>` (train split, rows 3000-3059 per language)
|
|
8
|
+
- Licence: Creative Commons Attribution-ShareAlike 4.0 International (CC BY-SA 4.0),
|
|
9
|
+
https://creativecommons.org/licenses/by-sa/4.0/
|
|
10
|
+
- Changes: article text was extracted as plain text, concatenated and normalised; markup and non-text elements were removed.
|
|
11
|
+
|
|
12
|
+
These files are distributed under CC BY-SA 4.0 (not under the AGPL). Unshape uses them only to build a list of common
|
|
13
|
+
syllables and a character language model that break ties between candidate readings.
|