ocr-util 2.0.1__tar.gz

ocr_util-2.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Universitäts- und Landesbibliothek Sachsen-Anhalt
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ocr_util-2.0.1/PKG-INFO

@@ -0,0 +1,136 @@
+Metadata-Version: 2.4
+Name: ocr-util
+Version: 2.0.1
+Summary: OCR Utils
+Author-email: Universitäts- und Landesbibliothek Sachsen-Anhalt <development@bibliothek.uni-halle.de>
+Project-URL: Homepage, https://github.com/ulb-sachsen-anhalt/ocr-util
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: rapidfuzz>3
+Requires-Dist: nltk
+Requires-Dist: requests
+Requires-Dist: docker
+Requires-Dist: numpy
+Requires-Dist: shapely
+Requires-Dist: lxml
+Requires-Dist: opencv-python-headless
+Requires-Dist: exifread
+Requires-Dist: Pillow
+Provides-Extra: dev
+Requires-Dist: black; extra == "dev"
+Requires-Dist: pylint; extra == "dev"
+Provides-Extra: test
+Requires-Dist: pytest-cov; extra == "test"
+Requires-Dist: coverage-badge; extra == "test"
+Requires-Dist: pytest-xdist; extra == "test"
+Requires-Dist: flake8; extra == "test"
+Requires-Dist: lxml-stubs; extra == "test"
+Requires-Dist: opencv-stubs; extra == "test"
+Dynamic: license-file
+
+# OCR Util
+
+![python-app](https://github.com/ulb-sachsen-anhalt/ocr-util/actions/workflows/python-app.yml/badge.svg) [![Coverage](./coverage.svg)](https://github.com/ulb-sachsen-anhalt/ocr-util/actions/workflows/python-app.yml) [![PyPi version](https://badgen.net/pypi/v/digital-eval/)](https://pypi.org/project/digital-eval) ![PyPI - Downloads](https://img.shields.io/pypi/dm/digital-eval) ![PyPI - License](https://img.shields.io/pypi/l/digital-eval) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/digital-eval)
+
+
+Collection of utils to 
+* evaluation of OCR data for the masses
+* generation of extended OCR-Evaluation Corpora
+* generation of pair-wise Trainingdata for OCR-Backends
+
+## Requirements
+
+* recent *nix-OS
+* Python3.10+ Environment
+
+## Usage
+
+Each section contains detailed usage help instructions:
+
+```bash
+# evaluation
+ocr eval --help
+
+# corpus management
+ocr corpus --help
+
+# slice image by image + input OCR
+ocr slice --help
+
+# render image + input OCR
+ocr show --help
+```
+
+### Data problems
+
+Inconsistent OCR Groundtruth with empty texts (ALTO String elements missing CONTENT or PAGE without TextEquiv) or invalid geometrical coordinates (less than 3 points or even empty) will lead to evaluation errors if geometry must be respected.
+
+_Please note_:  
+Invalid data files are tried(!) to be excluded from evaluation.
+
+### Evaluation Filter-Then-Aggregate
+
+The evaluation CLI supports a single pre-aggregation filter using metadata extractors.
+
+Example: keep only entries where MODS language is exactly German, then aggregate by publication century:
+
+```bash
+ocr eval <candidates> \
+	--reference <groundtruth> \
+	--mets-file <mets.xml> \
+	--filter-by "mods:language=ger" \
+	--aggregate-by "mods:dateIssued:century"
+```
+
+Multi-language filter values are interpreted as sets:
+
+```bash
+ocr eval <candidates> \
+	--reference <groundtruth> \
+	--mets-file <mets.xml> \
+	--filter-by "mods:language=ger+eng" \
+	--aggregate-by "mods:dateIssued:century"
+```
+
+Behavior:
+* single filter value -> exact match (e.g. `ger` does not match `ger+eng`)
+* multi-value filter -> all filter values must be present in any order
+* entries missing the filter criterion are reported as WARNING and discarded
+
+## Development
+
+Plattform: Intel(R) Core(TM) i5-6500 CPU@3.20GHz, 16GB RAM, Ubuntu 22.04 LTS, Python 3.10+
+
+```bash
+# clone local
+git clone <repository-url> <local-dir>
+cd <local-dir>
+
+# enable virtual python 3 environment (linux)
+# and update pip itself
+python3.10 -m venv venv
+. venv/bin/activate
+python -m pip install -U pip
+
+# install with dev dependencies
+python -m pip install -e ".[dev,test]"
+
+# run tests with coverage
+python -m pytest --cov=src
+
+# run tests faster (parallel, auto worker count)
+python -m pytest -q -n auto
+```
+
+## Contribution
+
+Contributions, suggestions and proposals welcome!
+
+## License
+
+Under terms of the [MIT license](https://opensource.org/licenses/MIT).
+
+**NOTE**: This software depends on packages that _might_ be licensed under different terms.

ocr_util-2.0.1/README.md

@@ -0,0 +1,103 @@
+# OCR Util
+
+![python-app](https://github.com/ulb-sachsen-anhalt/ocr-util/actions/workflows/python-app.yml/badge.svg) [![Coverage](./coverage.svg)](https://github.com/ulb-sachsen-anhalt/ocr-util/actions/workflows/python-app.yml) [![PyPi version](https://badgen.net/pypi/v/digital-eval/)](https://pypi.org/project/digital-eval) ![PyPI - Downloads](https://img.shields.io/pypi/dm/digital-eval) ![PyPI - License](https://img.shields.io/pypi/l/digital-eval) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/digital-eval)
+
+
+Collection of utils to 
+* evaluation of OCR data for the masses
+* generation of extended OCR-Evaluation Corpora
+* generation of pair-wise Trainingdata for OCR-Backends
+
+## Requirements
+
+* recent *nix-OS
+* Python3.10+ Environment
+
+## Usage
+
+Each section contains detailed usage help instructions:
+
+```bash
+# evaluation
+ocr eval --help
+
+# corpus management
+ocr corpus --help
+
+# slice image by image + input OCR
+ocr slice --help
+
+# render image + input OCR
+ocr show --help
+```
+
+### Data problems
+
+Inconsistent OCR Groundtruth with empty texts (ALTO String elements missing CONTENT or PAGE without TextEquiv) or invalid geometrical coordinates (less than 3 points or even empty) will lead to evaluation errors if geometry must be respected.
+
+_Please note_:  
+Invalid data files are tried(!) to be excluded from evaluation.
+
+### Evaluation Filter-Then-Aggregate
+
+The evaluation CLI supports a single pre-aggregation filter using metadata extractors.
+
+Example: keep only entries where MODS language is exactly German, then aggregate by publication century:
+
+```bash
+ocr eval <candidates> \
+	--reference <groundtruth> \
+	--mets-file <mets.xml> \
+	--filter-by "mods:language=ger" \
+	--aggregate-by "mods:dateIssued:century"
+```
+
+Multi-language filter values are interpreted as sets:
+
+```bash
+ocr eval <candidates> \
+	--reference <groundtruth> \
+	--mets-file <mets.xml> \
+	--filter-by "mods:language=ger+eng" \
+	--aggregate-by "mods:dateIssued:century"
+```
+
+Behavior:
+* single filter value -> exact match (e.g. `ger` does not match `ger+eng`)
+* multi-value filter -> all filter values must be present in any order
+* entries missing the filter criterion are reported as WARNING and discarded
+
+## Development
+
+Plattform: Intel(R) Core(TM) i5-6500 CPU@3.20GHz, 16GB RAM, Ubuntu 22.04 LTS, Python 3.10+
+
+```bash
+# clone local
+git clone <repository-url> <local-dir>
+cd <local-dir>
+
+# enable virtual python 3 environment (linux)
+# and update pip itself
+python3.10 -m venv venv
+. venv/bin/activate
+python -m pip install -U pip
+
+# install with dev dependencies
+python -m pip install -e ".[dev,test]"
+
+# run tests with coverage
+python -m pytest --cov=src
+
+# run tests faster (parallel, auto worker count)
+python -m pytest -q -n auto
+```
+
+## Contribution
+
+Contributions, suggestions and proposals welcome!
+
+## License
+
+Under terms of the [MIT license](https://opensource.org/licenses/MIT).
+
+**NOTE**: This software depends on packages that _might_ be licensed under different terms.

ocr_util-2.0.1/pyproject.toml

@@ -0,0 +1,81 @@
+[project]
+name = "ocr-util"
+dynamic = ["version"]
+description = "OCR Utils"
+readme = "README.md"
+requires-python = ">=3.10"
+authors = [{name = "Universitäts- und Landesbibliothek Sachsen-Anhalt",email = "development@bibliothek.uni-halle.de"}]
+classifiers = [
+    "Programming Language :: Python :: 3",
+	"License :: OSI Approved :: MIT License"
+]
+dependencies = [
+	"rapidfuzz>3",
+    "nltk",
+    "requests",
+    "docker",
+    "numpy",
+    "shapely",
+    "lxml",
+    "opencv-python-headless",
+    "exifread",
+    "Pillow"
+]
+
+[project.optional-dependencies]
+dev = [
+    "black",
+    "pylint",
+]
+test = [
+    "pytest-cov",
+    "coverage-badge",
+    "pytest-xdist",
+    "flake8",
+    "lxml-stubs",
+    "opencv-stubs",
+]
+
+[project.urls]
+Homepage = "https://github.com/ulb-sachsen-anhalt/ocr-util"
+
+[project.scripts]
+ocr-util = "ocr_util.cli:start"
+
+[build-system]
+requires = ["setuptools>=61.0.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.dynamic]
+version = {attr = "ocr_util.__version__"}
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+ocr_util = ["corpus/*.xml"]
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.black]
+line-length = 120
+target-version = ["py310", "py311", "py312"]
+include = '\.pyi?$'
+
+[tool.pylint.main]
+extension-pkg-allow-list=["lxml.etree"]
+
+[tool.pylint.messages_control]
+reportAttributeAccessIssue = "none"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+addopts = "-v --tb=short --cov-report=xml"
+
+[tool.coverage.run]
+source = ["ocr_util"]
+omit = ["*/tests/*", "*/__pycache__/*", "*/venv/*", "*/build/*"]

ocr_util-2.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

ocr_util-2.0.1/src/ocr_util/__init__.py

@@ -0,0 +1,3 @@
+"""main API"""
+
+__version__ = "2.0.1"

ocr_util-2.0.1/src/ocr_util/cli.py

@@ -0,0 +1,311 @@
+# -*- coding: utf-8 -*-
+"""OCR Utils"""
+
+import argparse
+import logging
+import os
+import re
+from pathlib import Path, PurePath
+
+import ocr_util
+import ocr_util.eval.model as do
+import ocr_util.eval.model.filter as dofi
+import ocr_util.eval.cli as eval_cli
+import ocr_util.slice.cli as slice_cli
+import ocr_util.show.cli as show_cli
+import ocr_util.corpus.generate_corpus as gc
+
+from ocr_util.corpus.common import CorpusArgs
+
+# script constants
+DEFAULT_VERBOSITY = 0
+SUB_CMD_FRAME = "frame"
+SUB_CMD_GROUNDTRUTH_CORPUS = "corpus"
+CORPUS_CACHE_DIR_NAME = "ocr_util_corpus_mets_cache"
+CORPUS_CACHE_DIR = os.path.join(
+    os.path.expanduser("~"), ".cache", CORPUS_CACHE_DIR_NAME
+)
+
+SUB_CMD_EVALUATE = "eval"
+SUB_CMD_SLICE = "slice"
+SUB_CMD_SHOW = "show"
+
+# Remove this constant as it's now managed by show_cli
+# SUB_CMD_SHOW is kept for backward compatibility with other references
+
+
+def points_type(points: str) -> str:
+    match: re.Match = re.match(dofi.PolygonFrameFilterUtil.POINT_LIST_PATTERN, points)
+    if not match:
+        raise argparse.ArgumentTypeError(f"Invalid point coordinates: '{points}'")
+    return points
+
+
+def start() -> None:
+    # Configure logging once, centrally
+    logging.basicConfig(
+        level=logging.INFO,
+        format='%(asctime)s [%(levelname)s][%(name)s] %(message)s',
+        datefmt='%Y-%m-%d %H:%M:%S'
+    )
+    arg_parser: argparse.ArgumentParser = argparse.ArgumentParser(
+        prog="ocr-util",
+        description=f"OCR Util {ocr_util.__version__} of ULB Sachsen-Anhalt",
+    )
+    sub_arg_parsers = arg_parser.add_subparsers(
+        title="Subkommandos",
+        dest="subcommand",
+        required=True,
+    )
+
+    # frames subcommand
+    frame_arg_parser = sub_arg_parsers.add_parser(
+        SUB_CMD_FRAME,
+        help="Filter Contents of provided ALTO-v3-Data by provided Coordinates, where Coordinates span a rectangular"
+        " box with",
+    )
+    frame_arg_parser.add_argument(
+        "-v",
+        "--verbosity",
+        action="count",
+        default=DEFAULT_VERBOSITY,
+        required=False,
+        help=f"Verbosity flag. To increase, append multiple 'v's (optional; default: '{DEFAULT_VERBOSITY}')",
+    )
+    frame_arg_parser.add_argument(
+        "-i", "--input-ocr-file", help="Path of OCR-Data file to process", required=True
+    )
+    frame_arg_parser.add_argument(
+        "-o",
+        "--output-ocr-file",
+        help="Path of resulting OCR-Data file",
+        required=False,
+        default=None,
+    )
+    frame_arg_parser.add_argument(
+        "-p",
+        "--points",
+        required=True,
+        type=points_type,
+        help="""
+        Frame to slice words/lines/regions from input OCR-Data
+        f.e.: --frame "2892,2480 5072,2480 5072,5148 2892,5148"
+        """,
+    )
+
+    # groundtruth-corpus subcommand
+    groundtruth_corpus_arg_parser = sub_arg_parsers.add_parser(
+        SUB_CMD_GROUNDTRUTH_CORPUS,
+        help="Create METS file from N ground truth PAGE-XML files with URN identifiers",
+    )
+    groundtruth_corpus_arg_parser.add_argument(
+        "-i",
+        "--input",
+        dest="input_dir",
+        help="Path to the input directory containing GT PAGE-XML files",
+        required=True,
+    )
+    groundtruth_corpus_arg_parser.add_argument(
+        "-o",
+        "--output",
+        dest="output_dir",
+        help="Path to the output directory for generated corpus",
+        required=True,
+    )
+    groundtruth_corpus_arg_parser.add_argument(
+        "-l",
+        "--limit",
+        type=int,
+        default=0,
+        help="Number of files to process (default: 0 = unlimited)",
+        required=False,
+    )
+    groundtruth_corpus_arg_parser.add_argument(
+        "-t",
+        "--temp-dir",
+        dest="temp_dir",
+        default=CORPUS_CACHE_DIR,
+        help=f"Path to temporary directory for caching METS files (default: {CORPUS_CACHE_DIR})",
+        required=False,
+    )
+    groundtruth_corpus_arg_parser.add_argument(
+        "-v",
+        "--verbosity",
+        action="count",
+        default=DEFAULT_VERBOSITY,
+        required=False,
+        help=f"Verbosity flag. To increase, append multiple 'v's (optional; default: '{DEFAULT_VERBOSITY}')",
+    )
+    groundtruth_corpus_arg_parser.add_argument(
+        "--corpus-label",
+        dest="corpus_label",
+        default="Ground Truth Corpus",
+        help="Label for the corpus in the METS logical structure (default: 'Ground Truth Corpus')",
+        required=False,
+    )
+    groundtruth_corpus_arg_parser.add_argument(
+        "--oai-base-url",
+        dest="oai_base_url",
+        help="Base URL for OAI-PMH requests",
+        required=False,
+    )
+    groundtruth_corpus_arg_parser.add_argument(
+        "--clear-cache",
+        dest="clear_cache",
+        action="store_true",
+        default=False,
+        help="Clear the local cache directory before processing (default: False)",
+        required=False,
+    )
+
+    # evaluate subcommand
+    evaluate_arg_parser = sub_arg_parsers.add_parser(
+        SUB_CMD_EVALUATE,
+        help="Evaluate OCR candidates against ground truth data",
+        add_help=True,
+    )
+    eval_cli.register_arguments(evaluate_arg_parser)
+
+    # slice subcommand
+    slice_arg_parser = sub_arg_parsers.add_parser(
+        SUB_CMD_SLICE,
+        help="Generate pairs of textlines and image frames from OCR and image data",
+        add_help=True,
+    )
+    slice_arg_parser.add_argument(
+        "data",
+        type=str,
+        help="path to local alto|page file corresponding to image",
+    )
+    slice_arg_parser.add_argument(
+        "-i",
+        "--image",
+        required=True,
+        help="path to local image file tif|jpg|png corresponding to ocr",
+    )
+    slice_arg_parser.add_argument(
+        "-o",
+        "--output_dir",
+        default=slice_cli.DEFAULT_OUTDIR_PREFIX,
+        help=f"output directory, re-created if already exists (default: {slice_cli.DEFAULT_OUTDIR_PREFIX})",
+    )
+    slice_arg_parser.add_argument(
+        "--prefix-output",
+        required=False,
+        help="optional: prefix each pair using this arg (default: '')",
+    )
+    slice_arg_parser.add_argument(
+        "-m",
+        "--minchars",
+        required=False,
+        type=int,
+        default=int(slice_cli.DEFAULT_MIN_CHARS),
+        help=f"optional: minimum printable chars required for a line to be included into set (default: {slice_cli.DEFAULT_MIN_CHARS})",
+    )
+    slice_arg_parser.add_argument(
+        "-s",
+        "--summary",
+        required=False,
+        action="store_true",
+        default=slice_cli.DEFAULT_USE_SUMMARY,
+        help=f"optional: print all lines in additional file (default: {slice_cli.DEFAULT_USE_SUMMARY})",
+    )
+    slice_arg_parser.add_argument(
+        "-r",
+        "--reorder",
+        required=False,
+        action="store_true",
+        default=slice_cli.DEFAULT_USE_REORDER,
+        help=f"optional: re-order word tokens from right-to-left (default: {slice_cli.DEFAULT_USE_REORDER})",
+    )
+    slice_arg_parser.add_argument(
+        "--binarize",
+        required=False,
+        action="store_true",
+        default=slice_cli.DEFAULT_BINARIZE,
+        help=f"optional: binarize textline images (default: {slice_cli.DEFAULT_BINARIZE})",
+    )
+    slice_arg_parser.add_argument(
+        "--sanitize",
+        required=False,
+        type=bool,
+        default=slice_cli.DEFAULT_SANITIZE,
+        help=f"optional: sanitize textline images (default: {slice_cli.DEFAULT_SANITIZE})",
+    )
+    slice_arg_parser.add_argument(
+        "--no-sanitize", dest="sanitize", action="store_false"
+    )
+    slice_arg_parser.add_argument(
+        "--intrusion-ratio",
+        required=False,
+        default=slice_cli.DEFAULT_INTRUSION_RATIO,
+        help=f"optional: alter threshold for top and bottom ratios for intrusion detection for sanitizing (default: {slice_cli.DEFAULT_INTRUSION_RATIO})",
+    )
+    slice_arg_parser.add_argument(
+        "--rotation-threshold",
+        required=False,
+        type=float,
+        default=slice_cli.DEFAULT_ROTATION_THRESH,
+        help=f"optional: alter threshold for rotation of textline image (default: {slice_cli.DEFAULT_ROTATION_THRESH})",
+    )
+    slice_arg_parser.add_argument(
+        "-p",
+        "--padding",
+        required=False,
+        type=int,
+        default=slice_cli.DEFAULT_PADDING,
+        help=f"optional: additional padding for existing textline image (default: {slice_cli.DEFAULT_PADDING})",
+    )
+
+    # show subcommand
+    show_cli.register_arguments(sub_arg_parsers)
+
+    args = arg_parser.parse_args()
+
+    verbosity: int = getattr(args, "verbosity", DEFAULT_VERBOSITY)
+
+    if args.subcommand == SUB_CMD_FRAME:
+        input_ocr_file: str = args.input_ocr_file
+        output_ocr_file: str = args.output_ocr_file
+        points: str = args.points
+        if verbosity > 1:
+            print(
+                f"[DEBUG] args: {input_ocr_file}, {output_ocr_file}, {points}, {verbosity}"
+            )
+        polygon_frame_filter: dofi.PolygonFrameFilter = dofi.PolygonFrameFilter(
+            input_ocr_file, points, verbosity
+        )
+        piece_result: do.DigitalObjectTree = polygon_frame_filter.process()
+        file_result: PurePath = do.from_digital_object(piece_result, output_ocr_file)
+        if verbosity > 0:
+            print("[INFO ] file_result", file_result)
+
+    elif args.subcommand == SUB_CMD_GROUNDTRUTH_CORPUS:
+        corpus_args = CorpusArgs(
+            input_dir=Path(args.input_dir).absolute(),
+            output_dir=Path(args.output_dir).absolute(),
+            local_cache_dir=Path(args.temp_dir).absolute(),
+            limit=int(args.limit),
+            corpus_label=args.corpus_label,
+            clear_cache=args.clear_cache
+        )
+        gc.generate(corpus_args)
+
+    elif args.subcommand == SUB_CMD_EVALUATE:
+        eval_args = vars(args)
+        eval_args.pop("subcommand", None)
+        eval_cli.start_evaluation(eval_args)
+
+    elif args.subcommand == SUB_CMD_SLICE:
+        slice_args = vars(args)
+        slice_args.pop("subcommand", None)
+        slice_cli.start_slice(slice_args)
+
+    elif args.subcommand == SUB_CMD_SHOW:
+        show_args = vars(args)
+        show_args.pop("subcommand", None)
+        show_cli.start_show(show_args)
+
+
+if __name__ == "__main__":
+    start()

ocr_util-2.0.1/src/ocr_util/corpus/__init__.py

@@ -0,0 +1,17 @@
+"""Public API for the OCR ground truth corpus generation package.
+
+Import :func:`~ocr_util.corpus.generate_corpus.generate` and
+:class:`~ocr_util.corpus.common.CorpusArgs` to create a METS-based corpus
+from a directory of PAGE-XML ground truth files::
+
+    from ocr_util.corpus.generate_corpus import generate
+    from ocr_util.corpus.common import CorpusArgs
+    from pathlib import Path
+
+    result = generate(CorpusArgs(
+        input_dir=Path("gt/"),
+        output_dir=Path("corpus/"),
+        local_cache_dir=Path("/tmp/mets_cache"),
+    ))
+    print(result.file_path, result.n_pages)
+"""