ocr-util 2.1.1__tar.gz → 2.2.1__tar.gz

{ocr_util-2.1.1/src/ocr_util.egg-info → ocr_util-2.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ocr-util
-Version: 2.1.1
+Version: 2.2.1
 Summary: OCR Utils
 Author-email: Universitäts- und Landesbibliothek Sachsen-Anhalt <development@bibliothek.uni-halle.de>
 Project-URL: Homepage, https://github.com/ulb-sachsen-anhalt/ocr-util
@@ -57,6 +57,9 @@ ocr-util eval --help
 # corpus management
 ocr-util corpus --help
 
+# analyse an existing METS corpus
+ocr-util corpus-analyse --help
+
 # slice image by image + input OCR
 ocr-util slice --help
 
@@ -100,6 +103,38 @@ Behavior:
 * single filter value -> exact match (e.g. `ger` does not match `ger+eng`)
 * multi-value filter -> all filter values must be present in any order
 * entries missing the filter criterion are reported as WARNING and discarded
+* METS coverage reports how many evaluation pairs map to full-text references
+	and how many METS references have no evaluation pair
+* aggregation coverage reports how many evaluated pairs provide each dimension;
+	reported `items` count candidate/ground-truth pairs, not MODS elements
+
+### Corpus Analysis
+
+Existing METS corpora can be filtered without evaluation candidate data. Matching
+full-text file references are printed one per line:
+
+```bash
+ocr-util corpus-analyse <mets.xml> \
+	--filter-by "mods:dateIssued:century=16th"
+```
+
+Repeat `--filter-by` to combine criteria with AND:
+
+```bash
+ocr-util corpus-analyse <mets.xml> \
+	--filter-by "mods:dateIssued:century=16th" \
+	--filter-by "mods:language=ger"
+```
+
+Check that every local file referenced by any METS `FLocat` exists:
+
+```bash
+ocr-util corpus-analyse <mets.xml> --check
+```
+
+Relative paths are resolved from the METS directory. Missing files and remote
+references that cannot be checked locally produce a non-zero exit status. This
+is a file-presence check; it does not perform XML schema validation.
 
 ## Development
 

{ocr_util-2.1.1 → ocr_util-2.2.1}/README.md

@@ -24,6 +24,9 @@ ocr-util eval --help
 # corpus management
 ocr-util corpus --help
 
+# analyse an existing METS corpus
+ocr-util corpus-analyse --help
+
 # slice image by image + input OCR
 ocr-util slice --help
 
@@ -67,6 +70,38 @@ Behavior:
 * single filter value -> exact match (e.g. `ger` does not match `ger+eng`)
 * multi-value filter -> all filter values must be present in any order
 * entries missing the filter criterion are reported as WARNING and discarded
+* METS coverage reports how many evaluation pairs map to full-text references
+	and how many METS references have no evaluation pair
+* aggregation coverage reports how many evaluated pairs provide each dimension;
+	reported `items` count candidate/ground-truth pairs, not MODS elements
+
+### Corpus Analysis
+
+Existing METS corpora can be filtered without evaluation candidate data. Matching
+full-text file references are printed one per line:
+
+```bash
+ocr-util corpus-analyse <mets.xml> \
+	--filter-by "mods:dateIssued:century=16th"
+```
+
+Repeat `--filter-by` to combine criteria with AND:
+
+```bash
+ocr-util corpus-analyse <mets.xml> \
+	--filter-by "mods:dateIssued:century=16th" \
+	--filter-by "mods:language=ger"
+```
+
+Check that every local file referenced by any METS `FLocat` exists:
+
+```bash
+ocr-util corpus-analyse <mets.xml> --check
+```
+
+Relative paths are resolved from the METS directory. Missing files and remote
+references that cannot be checked locally produce a non-zero exit status. This
+is a file-presence check; it does not perform XML schema validation.
 
 ## Development
 

ocr_util-2.2.1/src/ocr_util/__init__.py

@@ -0,0 +1,3 @@
+"""main API"""
+
+__version__ = "2.2.1"

{ocr_util-2.1.1 → ocr_util-2.2.1}/src/ocr_util/cli.py

@@ -13,6 +13,7 @@ import ocr_util.eval.model.filter as dofi
 import ocr_util.eval.cli as eval_cli
 import ocr_util.slice.cli as slice_cli
 import ocr_util.show.cli as show_cli
+import ocr_util.corpus.analyse as corpus_analyse
 import ocr_util.corpus.generate_corpus as gc
 
 from ocr_util.corpus.common import CorpusArgs
@@ -21,10 +22,9 @@ from ocr_util.corpus.common import CorpusArgs
 DEFAULT_VERBOSITY = 0
 SUB_CMD_FRAME = "frame"
 SUB_CMD_GROUNDTRUTH_CORPUS = "corpus"
+SUB_CMD_CORPUS_ANALYSE = "corpus-analyse"
 CORPUS_CACHE_DIR_NAME = "ocr_util_corpus_mets_cache"
-CORPUS_CACHE_DIR = os.path.join(
-    os.path.expanduser("~"), ".cache", CORPUS_CACHE_DIR_NAME
-)
+CORPUS_CACHE_DIR = os.path.join(os.path.expanduser("~"), ".cache", CORPUS_CACHE_DIR_NAME)
 
 SUB_CMD_EVALUATE = "eval"
 SUB_CMD_SLICE = "slice"
@@ -44,9 +44,7 @@ def points_type(points: str) -> str:
 def start() -> None:
     # Configure logging once, centrally
     logging.basicConfig(
-        level=logging.INFO,
-        format='%(asctime)s [%(levelname)s][%(name)s] %(message)s',
-        datefmt='%Y-%m-%d %H:%M:%S'
+        level=logging.INFO, format="%(asctime)s [%(levelname)s][%(name)s] %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
     )
     arg_parser: argparse.ArgumentParser = argparse.ArgumentParser(
         prog="ocr-util",
@@ -72,9 +70,7 @@ def start() -> None:
         required=False,
         help=f"Verbosity flag. To increase, append multiple 'v's (optional; default: '{DEFAULT_VERBOSITY}')",
     )
-    frame_arg_parser.add_argument(
-        "-i", "--input-ocr-file", help="Path of OCR-Data file to process", required=True
-    )
+    frame_arg_parser.add_argument("-i", "--input-ocr-file", help="Path of OCR-Data file to process", required=True)
     frame_arg_parser.add_argument(
         "-o",
         "--output-ocr-file",
@@ -158,6 +154,12 @@ def start() -> None:
         required=False,
     )
 
+    corpus_analyse_parser = sub_arg_parsers.add_parser(
+        SUB_CMD_CORPUS_ANALYSE,
+        help="List files in an existing METS corpus that match metadata filters",
+    )
+    corpus_analyse.register_arguments(corpus_analyse_parser)
+
     # evaluate subcommand
     evaluate_arg_parser = sub_arg_parsers.add_parser(
         SUB_CMD_EVALUATE,
@@ -232,9 +234,7 @@ def start() -> None:
         default=slice_cli.DEFAULT_SANITIZE,
         help=f"optional: sanitize textline images (default: {slice_cli.DEFAULT_SANITIZE})",
     )
-    slice_arg_parser.add_argument(
-        "--no-sanitize", dest="sanitize", action="store_false"
-    )
+    slice_arg_parser.add_argument("--no-sanitize", dest="sanitize", action="store_false")
     slice_arg_parser.add_argument(
         "--intrusion-ratio",
         required=False,
@@ -269,12 +269,8 @@ def start() -> None:
         output_ocr_file: str = args.output_ocr_file
         points: str = args.points
         if verbosity > 1:
-            print(
-                f"[DEBUG] args: {input_ocr_file}, {output_ocr_file}, {points}, {verbosity}"
-            )
-        polygon_frame_filter: dofi.PolygonFrameFilter = dofi.PolygonFrameFilter(
-            input_ocr_file, points, verbosity
-        )
+            print(f"[DEBUG] args: {input_ocr_file}, {output_ocr_file}, {points}, {verbosity}")
+        polygon_frame_filter: dofi.PolygonFrameFilter = dofi.PolygonFrameFilter(input_ocr_file, points, verbosity)
         piece_result: do.DigitalObjectTree = polygon_frame_filter.process()
         file_result: PurePath = do.from_digital_object(piece_result, output_ocr_file)
         if verbosity > 0:
@@ -287,10 +283,17 @@ def start() -> None:
             local_cache_dir=Path(args.temp_dir).absolute(),
             limit=int(args.limit),
             corpus_label=args.corpus_label,
-            clear_cache=args.clear_cache
+            clear_cache=args.clear_cache,
         )
         gc.generate(corpus_args)
 
+    elif args.subcommand == SUB_CMD_CORPUS_ANALYSE:
+        analyse_args = vars(args)
+        analyse_args.pop("subcommand", None)
+        result = corpus_analyse.start_analysis(analyse_args)
+        if isinstance(result, corpus_analyse.CorpusCheckResult) and not result.is_valid:
+            raise SystemExit(1)
+
     elif args.subcommand == SUB_CMD_EVALUATE:
         eval_args = vars(args)
         eval_args.pop("subcommand", None)

ocr_util-2.2.1/src/ocr_util/corpus/analyse.py

@@ -0,0 +1,173 @@
+"""Analyse a METS corpus with evaluation-compatible filters."""
+
+from __future__ import annotations
+
+import argparse
+import dataclasses
+import re
+import typing
+from pathlib import Path
+from urllib.parse import unquote, urlparse
+
+import lxml.etree as ET
+
+import ocr_util.eval as digev
+from ocr_util.eval.cli import _build_filter_spec, _filter_value_matches
+
+METS_NAMESPACES = digev.METSModsExtractor.DEFAULT_NAMESPACES
+
+
+@dataclasses.dataclass(frozen=True)
+class CorpusCheckResult:
+    """summarize local file references found in a METS corpus."""
+
+    checked: tuple[Path, ...]
+    missing: tuple[Path, ...]
+    uncheckable: tuple[str, ...]
+
+    @property
+    def is_valid(self) -> bool:
+        """return whether every METS file reference is locally present."""
+        return not self.missing and not self.uncheckable
+
+
+def _groundtruth_type(path: Path) -> typing.Optional[str]:
+    """infer the evaluation ground-truth type from a corpus file name."""
+    match = re.match(r".*(?:gt\.(\w{3,})|\.(\w{3,})\.gt)\.xml$", path.name)
+    label = next((group for group in match.groups() if group), None) if match else None
+    if label and label.startswith("art"):
+        return "article"
+    if label and label.startswith("ann"):
+        return "announcement"
+    return None
+
+
+def _corpus_entries(mets_file: Path) -> list[digev.EvalEntry]:
+    """create entry-like objects for full-text files referenced by a METS corpus."""
+    tree = ET.parse(str(mets_file))
+    entries: list[digev.EvalEntry] = []
+
+    for file_group in tree.xpath("//mets:fileGrp", namespaces=METS_NAMESPACES):
+        if "FULLTEXT" not in (file_group.get("USE") or "").upper():
+            continue
+
+        for href in file_group.xpath("./mets:file/mets:FLocat/@xlink:href", namespaces=METS_NAMESPACES):
+            referenced_path = Path(str(href))
+            entry = digev.EvalEntry(referenced_path)
+            entry.path_groundtruth = referenced_path
+            entry.domain_directories = list(reversed(referenced_path.parent.parts))
+            entry.gt_type = _groundtruth_type(referenced_path) or entry.gt_type
+            entries.append(entry)
+
+    return entries
+
+
+def check_corpus(mets_file: Path) -> CorpusCheckResult:
+    """check whether every file referenced by the METS corpus is present locally."""
+    mets_file = Path(mets_file)
+    if not mets_file.is_file():
+        raise FileNotFoundError(f"METS file not found: {mets_file}")
+
+    tree = ET.parse(str(mets_file))
+    hrefs = tree.xpath("//mets:FLocat/@xlink:href", namespaces=METS_NAMESPACES)
+    checked: list[Path] = []
+    missing: list[Path] = []
+    uncheckable: list[str] = []
+
+    for href_value in hrefs:
+        href = str(href_value)
+        parsed = urlparse(href)
+        if parsed.scheme not in ("", "file") or (parsed.netloc not in ("", "localhost")):
+            uncheckable.append(href)
+            continue
+
+        path = Path(unquote(parsed.path))
+        if not path.is_absolute():
+            path = mets_file.parent / path
+        checked.append(path)
+        if not path.is_file():
+            missing.append(path)
+
+    return CorpusCheckResult(tuple(checked), tuple(missing), tuple(uncheckable))
+
+
+def analyse(mets_file: Path, filters: typing.Sequence[str]) -> list[str]:
+    """return corpus file references matching all evaluation filter specifications."""
+    mets_file = Path(mets_file)
+    if not mets_file.is_file():
+        raise FileNotFoundError(f"METS file not found: {mets_file}")
+
+    entries = _corpus_entries(mets_file)
+    for filter_by in filters:
+        filter_spec = _build_filter_spec(filter_by, mets_file, strict=True)
+        if filter_spec is None:
+            raise ValueError(f"Invalid filter specification: '{filter_by}'")
+        _, extractor, expected_value = filter_spec
+        entries = [
+            entry
+            for entry in entries
+            if (value := extractor(entry)) is not None
+            and str(value).strip()
+            and _filter_value_matches(str(value), expected_value)
+        ]
+
+    return [str(entry.path_groundtruth) for entry in entries]
+
+
+def register_arguments(parser: argparse.ArgumentParser) -> None:
+    """register command-line arguments for corpus analysis."""
+    parser.add_argument("mets_file", type=Path, help="Path to the corpus METS file")
+    parser.add_argument(
+        "--filter-by",
+        action="append",
+        default=[],
+        help=(
+            "Evaluation-compatible filter; repeat to combine filters with AND, "
+            "for example 'mods:dateIssued:century=16th'"
+        ),
+    )
+    parser.add_argument(
+        "--check",
+        action="store_true",
+        help="Check that every local file referenced by the METS corpus exists",
+    )
+
+
+def start_analysis(
+    args: typing.Mapping[str, typing.Any],
+) -> typing.Union[list[str], CorpusCheckResult]:
+    """analyse a corpus and print each matching file reference to stdout."""
+    if args.get("check"):
+        result = check_corpus(Path(args["mets_file"]))
+        missing = set(result.missing)
+        for path in result.checked:
+            status = "MISSING" if path in missing else "OK"
+            print(f"{status} {path}")
+        for href in result.uncheckable:
+            print(f"UNCHECKABLE {href}")
+        print(
+            f"SUMMARY checked={len(result.checked)} missing={len(result.missing)} "
+            f"uncheckable={len(result.uncheckable)}"
+        )
+        return result
+
+    if not args.get("filter_by"):
+        raise ValueError("at least one --filter-by or --check is required")
+
+    matches = analyse(Path(args["mets_file"]), args["filter_by"])
+    for match in matches:
+        print(match)
+    return matches
+
+
+def start() -> None:
+    """run the standalone corpus analysis CLI."""
+    parser = argparse.ArgumentParser()
+    register_arguments(parser)
+    result = start_analysis(vars(parser.parse_args()))
+    if isinstance(result, CorpusCheckResult) and not result.is_valid:
+        raise SystemExit(1)
+
+
+if __name__ == "__main__":
+    start()

{ocr_util-2.1.1 → ocr_util-2.2.1}/src/ocr_util/corpus/common.py

@@ -419,9 +419,7 @@ class MetsResourceFile(MetsFile):
         assert file_fulltext is not None, f"No fulltext file pointer for page with CONTENTIDS='{page_urn}'"
         return file_image, file_fulltext
 
-
-    def _create_fulltext_element(self, new_id: str, gt_file_path: pathlib.Path,
-                                 out_dir: pathlib.Path) -> ET._Element:
+    def _create_fulltext_element(self, new_id: str, gt_file_path: pathlib.Path, out_dir: pathlib.Path) -> ET._Element:
         """Create a new file element for the fulltext file with the appropriate FLocat child."""
         file_fulltext = ET.Element(
             f'{{{self.nsmap["mets"]}}}file', attrib={"ID": new_id, "MIMETYPE": "application/vnd.prima.page+xml"}

{ocr_util-2.1.1 → ocr_util-2.2.1}/src/ocr_util/corpus/load_metadata.py

@@ -37,7 +37,6 @@ FALLBACK_IDX_BASE_URLS = {
 }
 
 
-
 @dataclasses.dataclass(frozen=True)
 class RecordResolutionResult:
     """Normalized result of a handle lookup attempt.

{ocr_util-2.1.1 → ocr_util-2.2.1}/src/ocr_util/eval/aggregation.py

@@ -146,9 +146,7 @@ def century_transform(value: str) -> typing.Optional[str]:
     try:
         year = int(str(value).strip()[:4])
         century = year // 100 + 1
-        suffix = {1: "st", 2: "nd", 3: "rd"}.get(
-            century % 10 if century % 100 not in (11, 12, 13) else 0, "th"
-        )
+        suffix = {1: "st", 2: "nd", 3: "rd"}.get(century % 10 if century % 100 not in (11, 12, 13) else 0, "th")
         return f"{century}{suffix}"
     except (ValueError, TypeError):
         return None
@@ -247,9 +245,7 @@ class METSDivAttrExtractor:
 
         # physical div ID → [file hrefs]
         phys_to_hrefs: typing.Dict[str, typing.List[str]] = {}
-        for pdiv in tree.xpath(
-            '//mets:structMap[@TYPE="PHYSICAL"]//mets:div[@ID]', namespaces=ns
-        ):
+        for pdiv in tree.xpath('//mets:structMap[@TYPE="PHYSICAL"]//mets:div[@ID]', namespaces=ns):
             pid = pdiv.get("ID")
             hrefs = [
                 file_id_to_href[fid]
@@ -263,9 +259,7 @@ class METSDivAttrExtractor:
         log_to_attr: typing.Dict[str, str] = {}
         # also: DMDID → attribute value (fallback)
         dmdid_to_attr: typing.Dict[str, str] = {}
-        for ldiv in tree.xpath(
-            '//mets:structMap[@TYPE="LOGICAL"]//mets:div[@ID]', namespaces=ns
-        ):
+        for ldiv in tree.xpath('//mets:structMap[@TYPE="LOGICAL"]//mets:div[@ID]', namespaces=ns):
             lid = ldiv.get("ID")
             attr_val = ldiv.get(self.attribute)
             if lid and attr_val:
@@ -412,9 +406,7 @@ class METSModsExtractor:
         files = tree.xpath("//mets:file", namespaces=self.namespaces)
         for file_elem in files:
             file_id = file_elem.get("ID")
-            flocat = file_elem.xpath(
-                "./mets:FLocat/@xlink:href", namespaces=self.namespaces
-            )
+            flocat = file_elem.xpath("./mets:FLocat/@xlink:href", namespaces=self.namespaces)
             if not flocat:
                 continue
 
@@ -456,9 +448,7 @@ class METSModsExtractor:
                 continue
 
             hrefs = []
-            for file_id in physical_div.xpath(
-                "./mets:fptr/@FILEID", namespaces=self.namespaces
-            ):
+            for file_id in physical_div.xpath("./mets:fptr/@FILEID", namespaces=self.namespaces):
                 file_href = file_id_to_href.get(file_id)
                 if file_href:
                     hrefs.append(file_href)
@@ -467,9 +457,7 @@ class METSModsExtractor:
                 physical_div_to_hrefs[physical_id] = hrefs
 
         xlink_ns = self.namespaces.get("xlink", "http://www.w3.org/1999/xlink")
-        for link in tree.xpath(
-            "//mets:structLink/mets:smLink", namespaces=self.namespaces
-        ):
+        for link in tree.xpath("//mets:structLink/mets:smLink", namespaces=self.namespaces):
             logical_id = link.get(f"{{{xlink_ns}}}from")
             physical_id = link.get(f"{{{xlink_ns}}}to")
             if not logical_id or not physical_id:
@@ -510,9 +498,7 @@ class METSModsExtractor:
         # Apply user's XPath expression to MODS section
         for mods_section in mods_sections:
             try:
-                results = mods_section.xpath(
-                    self.xpath_expression, namespaces=self.namespaces
-                )
+                results = mods_section.xpath(self.xpath_expression, namespaces=self.namespaces)
                 if results:
                     values = []
                     for result in results:
@@ -583,15 +569,11 @@ class AggregationStrategy:
         hierarchical: If True, creates hierarchical keys combining all dimensions
     """
 
-    def __init__(
-        self, dimensions: typing.List[AggregationDimension], hierarchical: bool = False
-    ):
+    def __init__(self, dimensions: typing.List[AggregationDimension], hierarchical: bool = False):
         self.dimensions = dimensions
         self.hierarchical = hierarchical
 
-    def generate_keys(
-        self, entry: typing.Any, metric: digem.OCRMetric
-    ) -> typing.List[str]:
+    def generate_keys(self, entry: typing.Any, metric: digem.OCRMetric) -> typing.List[str]:
         """Generate aggregation keys for an evaluation entry
 
         Args: