synthocr 2.1.0__py3-none-any.whl

synthocr/fonts.py

@@ -0,0 +1,225 @@
+"""Shared font discovery, bounded coverage caches, and verified font downloads."""
+
+import hashlib
+import json
+import logging
+from collections import OrderedDict
+from dataclasses import dataclass
+from importlib.resources import files
+from pathlib import Path
+from urllib.request import urlopen
+
+from fontTools.ttLib import TTCollection, TTFont, TTLibError
+from PIL import ImageFont
+
+from .downloads import download, within
+from .text import required_codepoints as required_codepoints
+
+LOG = logging.getLogger(__name__)
+FONT_SUFFIXES = {".ttf", ".otf", ".ttc", ".otc"}
+# Pinned open-font downloads, shipped with the package so installed copies can fetch them.
+CATALOG = Path(__file__).with_name("font_catalog.json")
+MAX_FONT_BYTES = 64 * 1024 * 1024
+PROFILES = ("sorani", "kurmanji", "kurdish")
+
+
+@dataclass(frozen=True)
+class Font:
+    path: str
+    index: int = 0
+
+    @property
+    def name(self):
+        return f"{Path(self.path).name}#{self.index}"
+
+
+def discover(directories: list[Path]) -> list[Font]:
+    result = []
+    seen = set()
+    for directory in directories:
+        for path in sorted(directory.rglob("*")):
+            if not path.is_file() or path.suffix.lower() not in FONT_SUFFIXES:
+                continue
+            if path.stat().st_size > MAX_FONT_BYTES:
+                LOG.warning("Skipping font larger than 64 MiB: %s", path)
+                continue
+            with path.open("rb") as stream:
+                digest = hashlib.file_digest(stream, "sha256").hexdigest()
+            if digest in seen:
+                continue
+            seen.add(digest)
+            try:
+                if path.suffix.lower() in {".ttc", ".otc"}:
+                    with TTCollection(path, lazy=True) as collection:
+                        faces = len(collection.fonts)
+                else:
+                    faces = 1
+                for index in range(faces):
+                    font = Font(str(path.resolve()), index)
+                    ImageFont.truetype(font.path, 12, index=index)
+                    result.append(font)
+            except (OSError, ValueError, TTLibError) as error:
+                LOG.warning("Skipping unreadable font %s: %s", path, error)
+    if not result:
+        raise ValueError("No usable TTF/OTF/TTC/OTC fonts found in the configured directories")
+    return result
+
+
+def font_ids(fonts: list[Font]) -> dict[Font, str]:
+    """Return each face's stable ID: the file's SHA-256 and the face index."""
+    digests = {}
+    result = {}
+    for font in fonts:
+        if font.path not in digests:
+            with Path(font.path).open("rb") as stream:
+                digests[font.path] = hashlib.file_digest(stream, "sha256").hexdigest()
+        result[font] = f"{digests[font.path]}:{font.index}"
+    return result
+
+
+def select(fonts: list[Font], include: list[str], ids: dict[Font, str] | None = None) -> list[Font]:
+    """Keep the faces named in include (file name, "name#face" or ID); empty keeps all."""
+    if not include:
+        return fonts
+    wanted = set(include)
+    ids = ids or font_ids(fonts)
+    chosen = [font for font in fonts if wanted & {ids[font], Path(font.path).name, font.name}]
+    if not chosen:
+        raise ValueError("None of the chosen fonts are available; choose at least one font")
+    return chosen
+
+
+class Coverage:
+    def __init__(self, limit: int = 64):
+        self.limit = limit
+        self.cache: OrderedDict[Font, bytes] = OrderedDict()
+
+    def _mask(self, font: Font) -> bytes:
+        if font in self.cache:
+            self.cache.move_to_end(font)
+            return self.cache[font]
+        with TTFont(font.path, fontNumber=font.index, lazy=True) as face:
+            cmap = face.getBestCmap() or {}
+            maximum = min(max(cmap, default=0), 0x10FFFF)
+            bits = bytearray((maximum >> 3) + 1)
+            for cp, glyph in cmap.items():
+                if 0 <= cp <= 0x10FFFF and glyph != ".notdef":
+                    bits[cp >> 3] |= 1 << (cp & 7)
+            value = bytes(bits)
+        self.cache[font] = value
+        if len(self.cache) > self.limit:
+            self.cache.popitem(last=False)
+        return value
+
+    def missing(self, font: Font, needed: frozenset[int]) -> frozenset[int]:
+        bits = self._mask(font)
+        return frozenset(
+            cp for cp in needed if cp >> 3 >= len(bits) or not bits[cp >> 3] & (1 << (cp & 7))
+        )
+
+    def codepoints(self, font: Font) -> frozenset[int]:
+        """Expanded inspection view; generation keeps only compact bitmap caches."""
+        return frozenset(
+            i * 8 + b
+            for i, byte in enumerate(self._mask(font))
+            if byte
+            for b in range(8)
+            if byte & (1 << b)
+        )
+
+    def choose(self, fonts: list[Font], text: str, offset: int, variant: int = 0) -> Font | None:
+        needed = required_codepoints(text)
+        matches = 0
+        for step in range(len(fonts)):
+            font = fonts[(offset + step) % len(fonts)]
+            if not self.missing(font, needed):
+                if matches == variant:
+                    return font
+                matches += 1
+        return None
+
+
+def load_characters(path: Path) -> str:
+    """Accept the original font-checking JSON mapping/list and plain UTF-8 text."""
+    if path.suffix.lower() != ".json":
+        characters = set()
+        with path.open(encoding="utf-8") as stream:
+            while chunk := stream.read(65536):
+                characters.update(chunk)
+        return "".join(sorted(characters))
+    return _characters(json.loads(path.read_text(encoding="utf-8")))
+
+
+def load_profile(name: str) -> str:
+    """Load a bundled reference without a checkout or companion repository."""
+    if name not in PROFILES:
+        raise ValueError(f"Unknown font profile: {name}")
+    if name == "kurdish":
+        resource = files("synthocr").joinpath("kurdish_characters.json")
+        return _characters(json.loads(resource.read_text(encoding="utf-8")))
+    profiles = json.loads(
+        files("synthocr").joinpath("font_profiles.json").read_text(encoding="utf-8")
+    )
+    profile = profiles[name]
+    return profile["letters"] + profile["digits"] + profile["punctuation"]
+
+
+def _characters(data) -> str:
+    if isinstance(data, dict):
+        values = list(data.values())
+        if all(isinstance(v, dict) for v in values):
+            return "".join(v["char"] for v in values if "char" in v)
+        return "".join(data)
+    if isinstance(data, list):
+        return "".join(v["char"] if isinstance(v, dict) else v for v in data)
+    raise ValueError("Character JSON must contain a mapping or a list")
+
+
+def check(fonts: list[Font], text: str, cache_size: int = 64) -> dict:
+    coverage = Coverage(cache_size)
+    needed = required_codepoints(text)
+    results = []
+    for font in fonts:
+        missing = sorted(coverage.missing(font, needed))
+        results.append(
+            {
+                "font": font.name,
+                "path": font.path,
+                "face_index": font.index,
+                "supported": len(needed) - len(missing),
+                "required": len(needed),
+                "missing": [f"U+{cp:04X}" for cp in missing],
+                "missing_text": "".join(map(chr, missing)),
+            }
+        )
+    return {
+        "fonts": len(fonts),
+        "characters": len(needed),
+        "fully_supported": sum(not row["missing"] for row in results),
+        "results": results,
+    }
+
+
+def _download(url: str, digest: str, target: Path, max_bytes: int):
+    return download(url, digest, target, max_bytes, {"raw.githubusercontent.com"}, opener=urlopen)
+
+
+def fetch(catalog: Path, destination: Path, families: list[str] | None = None) -> dict:
+    entries = json.loads(catalog.read_text(encoding="utf-8"))["fonts"]
+    available = {entry["family"] for entry in entries}
+    if families and set(families) - available:
+        raise ValueError(f"Unknown families: {sorted(set(families) - available)}")
+    installed = []
+    for entry in entries:
+        if families and entry["family"] not in families:
+            continue
+        for key in ("file", "license_file"):
+            value = entry[key]
+            _download(
+                value["url"],
+                value["sha256"],
+                within(destination, value["name"]),
+                MAX_FONT_BYTES if key == "file" else 512 * 1024,
+            )
+        installed.append(entry["family"])
+    return {"families": installed, "destination": str(destination)}

synthocr/home.py

@@ -0,0 +1,150 @@
+"""A personal workspace for installed copies that run outside a project folder."""
+
+from __future__ import annotations
+
+import logging
+import os
+import sys
+from datetime import date
+from pathlib import Path
+
+from .config import Config, load_config
+from .corpora import snapshot
+from .fonts import CATALOG, fetch
+from .text import generate_text, prepare
+
+LOG = logging.getLogger(__name__)
+
+# Open fonts that draw Sorani, Kurmanji, Arabic or English completely (about 9 MiB).
+SETUP_FONTS = (
+    "notosans",
+    "notoserif",
+    "notonaskharabic",
+    "notosansarabic",
+    "notonastaliqurdu",
+    "amiri",
+    "harmattan",
+    "ibmplexsansarabic",
+    "lateef",
+    "scheherazadenew",
+    "reemkufi",
+    "cairo",
+    "markazitext",
+    "readexpro",
+    "almarai",
+    "tajawal",
+)
+# Tatoeba codes for the app's four languages, and the offline template languages.
+SETUP_TATOEBA = ("ckb", "kmr", "ara", "eng")
+SETUP_TEMPLATES = ("ckb", "ar", "en")
+CONFIG = """\
+# Created by `synthocr setup`. Paths are relative to this file.
+[data]
+raw = "data/raw"
+processed = "data/processed"
+
+[fonts]
+directories = ["fonts"]
+
+[generation]
+output = "dataset/default"
+"""
+
+
+class NotSetUp(ValueError):
+    """No config.toml here and no personal workspace yet."""
+
+
+def home() -> Path:
+    """The workspace folder: SYNTHOCR_HOME, or the platform's usual data folder."""
+    if value := os.environ.get("SYNTHOCR_HOME"):
+        return Path(value).expanduser().resolve()
+    if sys.platform == "win32":
+        base = Path(os.environ.get("LOCALAPPDATA") or Path.home() / "AppData" / "Local")
+        return base / "SynthOCR"
+    if sys.platform == "darwin":
+        return Path.home() / "Library" / "Application Support" / "SynthOCR"
+    base = Path(os.environ.get("XDG_DATA_HOME") or Path.home() / ".local" / "share")
+    return base / "synthocr"
+
+
+def resolve_config(path: Path | None, loader=load_config) -> tuple[Config, bool]:
+    """Load an explicit or local config.toml; otherwise the personal workspace's.
+
+    The second value says whether the personal workspace is used.
+    """
+    if path is not None:
+        return loader(path), False
+    if Path("config.toml").is_file():
+        return loader(Path("config.toml")), False
+    workspace = home() / "config.toml"
+    if not workspace.is_file():
+        raise NotSetUp("SynthOCR is not set up yet. Run: synthocr setup")
+    return loader(workspace), True
+
+
+def ready(config: Config) -> bool:
+    return any(config.path("fonts").rglob("OFL.txt")) and any(
+        config.path(config.data.raw).rglob("*.jsonl")
+    )
+
+
+def setup(
+    root: Path | None = None,
+    *,
+    all_fonts: bool = False,
+    limit: int = 20_000,
+    tatoeba: bool = True,
+) -> dict:
+    """Create the workspace, fetch open fonts and text, and prepare the text index.
+
+    Safe to run again: downloads are checked against pinned checksums and kept.
+    """
+    root = (root or home()).resolve()
+    root.mkdir(parents=True, exist_ok=True)
+    config_path = root / "config.toml"
+    if not config_path.exists():
+        config_path.write_text(CONFIG, encoding="utf-8")
+    config = load_config(config_path)
+    LOG.info("Workspace: %s", root)
+
+    LOG.info("Downloading open fonts…")
+    fonts = fetch(CATALOG, config.path("fonts"), None if all_fonts else list(SETUP_FONTS))
+
+    raw = config.path(config.data.raw)
+    templates = []
+    for language in SETUP_TEMPLATES:
+        output = raw / "synthetic" / f"{language}.jsonl"
+        if not output.exists():
+            LOG.info("Writing offline %s text templates…", language)
+            generate_text(output, limit, 42, "mixed", language)
+        templates.append(language)
+
+    sentences, skipped = 0, None
+    # One snapshot is enough; running setup again on another day adds no copies.
+    if tatoeba and not (raw / "tatoeba").exists():
+        LOG.info("Downloading Tatoeba sentences (CC BY 2.0 France)…")
+        try:
+            sentences = snapshot(
+                raw,
+                root / "data" / "corpora.json",
+                root / ".cache" / "corpora",
+                list(SETUP_TATOEBA),
+                date.today().isoformat(),
+                limit,
+            )["records"]
+        except (OSError, ValueError) as error:
+            # Templates and the bundled spelling cases still work offline.
+            skipped = str(error)
+            LOG.warning("Tatoeba download skipped: %s", error)
+
+    LOG.info("Preparing the text index…")
+    prepared = prepare(config)
+    return {
+        "workspace": str(root),
+        "fonts": fonts["families"],
+        "templates": templates,
+        "tatoeba_sentences": sentences,
+        "tatoeba_skipped": skipped,
+        "texts": prepared["records"],
+    }