hyperglot 0.8.0__tar.gz → 0.8.2__tar.gz

{hyperglot-0.8.0 → hyperglot-0.8.2}/PKG-INFO

@@ -1,15 +1,13 @@
 Metadata-Version: 2.4
 Name: hyperglot
-Version: 0.8.0
+Version: 0.8.2
 Summary: Detect language support for font binaries
-Home-page: https://github.com/rosettatype/hyperglot
-Author: Johannes Neumeier - Rosetta
-Author-email: johannes@rosettatype.com
-License: Apache License 2.0
+Author-email: Johannes Neumeier - Rosetta <johannes@rosettatype.com>
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://github.com/rosettatype/hyperglot
 Project-URL: Hyperglot web interface, https://hyperglot.rosettatype.com
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Operating System :: OS Independent
-Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Intended Audience :: End Users/Desktop
 Classifier: Intended Audience :: Information Technology
 Classifier: Topic :: Text Processing :: Fonts
@@ -19,7 +17,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: packaging>=26.0
 Requires-Dist: click>=7.0
-Requires-Dist: fonttools>=4.50.0
+Requires-Dist: fonttools[woff]>=4.50.0
 Requires-Dist: uharfbuzz>=0.39.0
 Requires-Dist: pyyaml>=6.0.0
 Requires-Dist: colorlog>=6.4.1
@@ -29,19 +27,7 @@ Requires-Dist: tox; extra == "dev"
 Provides-Extra: build
 Requires-Dist: build; extra == "build"
 Requires-Dist: twine; extra == "build"
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: license
 Dynamic: license-file
-Dynamic: project-url
-Dynamic: provides-extra
-Dynamic: requires-dist
-Dynamic: requires-python
-Dynamic: summary
 
 # Hyperglot – a database and tools for detecting language support in fonts
 
@@ -56,7 +42,14 @@ Hyperglot is available as:
 📖 [Learn more about Hyperglot](https://hyperglot.rosettatype.com/about)  
 🙋 [Read the FAQ](FAQ.md)
 
-💰 [Sponsor via GitHub](https://github.com/sponsors/rosettatype) or directly via [Hyperglot sponsorship](https://hyperglot.rosettatype.com/sponsor). Any and all contributions are much appreciated! 🙏
+## 💰 Sponsor
+
+We welcome contributions to support our editorial work via Rosetta’s [GitHub](https://github.com/sponsors/rosettatype) or [Stripe](https://donate.stripe.com/3cI00j3PD0w53fwdx8dfG0f). Any level of support is greatly appreciated and helps sustain the project. 🙏
+
+- Contributions above €/$1,000 will be acknowledged below and on the website (About page) for a minimum of five years.
+- Contributions above €/$3,000 will also be featured in the website header as “Funded by…” for one year.
+
+For more details, feel free to get in touch via email.
 
 ## Data validity & contributing
 
@@ -119,6 +112,8 @@ Besides the main `hyperglot` command used for font inspection, the package also
 
 ### Basic usage
 
+Font inspection supports TTF, OTF, WOFF and WOFF2 files.
+
 Use:
 
 ```shell

{hyperglot-0.8.0 → hyperglot-0.8.2}/README.md

@@ -11,7 +11,14 @@ Hyperglot is available as:
 📖 [Learn more about Hyperglot](https://hyperglot.rosettatype.com/about)  
 🙋 [Read the FAQ](FAQ.md)
 
-💰 [Sponsor via GitHub](https://github.com/sponsors/rosettatype) or directly via [Hyperglot sponsorship](https://hyperglot.rosettatype.com/sponsor). Any and all contributions are much appreciated! 🙏
+## 💰 Sponsor
+
+We welcome contributions to support our editorial work via Rosetta’s [GitHub](https://github.com/sponsors/rosettatype) or [Stripe](https://donate.stripe.com/3cI00j3PD0w53fwdx8dfG0f). Any level of support is greatly appreciated and helps sustain the project. 🙏
+
+- Contributions above €/$1,000 will be acknowledged below and on the website (About page) for a minimum of five years.
+- Contributions above €/$3,000 will also be featured in the website header as “Funded by…” for one year.
+
+For more details, feel free to get in touch via email.
 
 ## Data validity & contributing
 
@@ -74,6 +81,8 @@ Besides the main `hyperglot` command used for font inspection, the package also
 
 ### Basic usage
 
+Font inspection supports TTF, OTF, WOFF and WOFF2 files.
+
 Use:
 
 ```shell

{hyperglot-0.8.0 → hyperglot-0.8.2}/lib/hyperglot/__init__.py

@@ -1,13 +1,14 @@
 """
 Gather a few package wide constants.
 """
+
 import re
 from os import path
 from enum import Enum
 
 from hyperglot.utils import AllChoicesEnumMixin, ConvenientEnumMixin
 
-__version__ = "0.8.0"
+__version__ = "0.8.2"
 
 DB = path.abspath(path.join(path.dirname(__file__), "data"))
 DB_EXTRA = path.abspath(path.join(path.dirname(__file__), "extra_data"))
@@ -15,8 +16,9 @@ DB_CHECKS = path.abspath(path.join(path.dirname(__file__), "checks"))
 
 LANGUAGE_CACHE_FILE = ".hyperglot-cache"
 
-# ~~DONE Refactor these levels and status as Enum's~~
-# TODO Eventaully remove deprecated "CONSTANTS"
+# Only show a cache rebuild warning once per session, not once per language!
+LANGUAGE_CACHE_MISMATCH_WARNING_SHOWN = False
+
 
 class SupportLevel(AllChoicesEnumMixin, Enum):
     """
@@ -117,6 +119,7 @@ RE_MULTIPLE_SPACES = re.compile(r"\s{2,}")
 from hyperglot.languages import Languages
 from hyperglot.language import Language
 from hyperglot.orthography import Orthography
+from hyperglot.checker import FontChecker, CharsetChecker
 
 __all__ = [
     Languages,
@@ -126,4 +129,6 @@ __all__ = [
     LanguageValidity,
     LanguageStatus,
     OrthographyStatus,
+    FontChecker,
+    CharsetChecker,
 ]

{hyperglot-0.8.0 → hyperglot-0.8.2}/lib/hyperglot/checkbase.py

@@ -1,50 +1,33 @@
+from __future__ import annotations
+
 import logging
+from typing import TYPE_CHECKING
+
+from hyperglot import SupportLevel
 
-from hyperglot import SupportLevel, LanguageValidity
-from hyperglot.shaper import Shaper
+if TYPE_CHECKING:
+    from hyperglot.orthography import Orthography
+    from hyperglot.checker import Checker
+    from hyperglot.shaper import Shaper
 
 
 class CheckBase:
 
     # Unicode constants used in some of the checks
-    VIRAMA = chr(0x094D)
+    # Futher script-specific constant can be added in subclasses
     ZWJ = chr(0x200D)
     ZWNJ = chr(0x200C)
     DOTTED_CIRCLE = chr(0x25CC)
-    BRAHMIC_CATEGORIES = {
-        "+": [chr(0x094D)],
-        "V": [
-            chr(c)
-            for c in list(range(0x0904, 0x0914 + 1))
-            + [0x0960, 0x0961]
-            + list(range(0x0973, 0x0977 + 1))
-        ],
-        "C": [
-            chr(c)
-            for c in list(range(0x0915, 0x0939 + 1))
-            + list(range(0x0958, 0x095F + 1))
-            + list(range(0x0978, 0x097F + 1))
-        ],
-        "D": [
-            chr(c)
-            for c in [0x093A, 0x093B]
-            + list(range(0x093E, 0x094C + 1))
-            + [0x094E, 0x094F]
-            + [0x0955, 0x0956, 0x0957]
-            + [0x0962, 0x0963]
-        ],
-        "M": [chr(0x093C)],
-        "m": [chr(c) for c in list(range(0x0900, 0x0903 + 1))],
-        "P": [chr(c) for c in [0x093D, 0x0964, 0x0965]],
-        "Z": [chr(0x200C)],
-        "z": [chr(0x200D)],
-    }
 
     # Conditions by which to determined if a check should run for a given
-    # orthography
+    # orthography.
+    # Multiple condition attributes are INTERSECTed, e.g. if both "scripts" and
+    # "attributes" are provided, the check will run if an orthography matches
+    # any of the scripts AND has any of the attributes.
     conditions = {
-        # e.g.
-        # "script": "Devanagari",
+        # Match any of the given scripts
+        # "scripts": ("Devanagari",),
+        # Match any of the given orthography attributes, e.g. "base", "auxiliary", "mark", "combinations"
         # "attributes": ("combinations",),
     }
 
@@ -58,17 +41,60 @@ class CheckBase:
 
     def __init__(self):
         # Use any module logger to output code issues, append to self.logs
-        # reporting entries that should get output "higher" up in the check run.
+        # reporting entries that should get output "higher" up in the check
+        # run.
         self.logs = []
 
-    def check(self):
-        raise NotImplementedError("Checks need to implement check method!")
+    def precheck(
+        self,
+        orthography: Orthography,
+        checker: Checker,
+        **kwargs,
+    ) -> bool:
+        """
+        Optional subclass method to precheck the check for a given orthography
+        and checker, e.g. by precomputing some data or doing some pre-checks.
+
+        Return True if the check should run, False if it should be skipped as
+        unnecessary.
+
+        When subclassing, run this method first, and return False to skip the
+        check early if it is not necessary.
+        """
+
+        # The compiled options for this check, after a orthography and checker
+        # have been passed.
+        self.options = self._get_options(**kwargs)
+
+        # Subclassing this method must return a bool here.
+        return True
+
+    def check(
+        self,
+        orthography: Orthography,
+        checker: Checker,
+        **kwargs,
+    ) -> bool:
+        """
+        Run the check, return True if it passes, False if it fails.
+
+        Subclasses should implement the actual check logic here, and call
+        this super class first, optionally overwriting also precheck to
+        prepare data.
+        """
+
+        # Run precheck to prepare check and determine early on if it indicated
+        # it can be skipped altogether (passes).
+        if not self.precheck(orthography, checker, **kwargs):
+            return True
+
+        return False
 
     def check_all_render(self, input: str, shaper: Shaper) -> bool:
         """
         Check an input string renders in the font without leaving any notdef or
-        dotted circles. As a fairly general check this may be useful in multiple
-        check implementations.
+        dotted circles. As a fairly general check this may be useful in
+        multiple check implementations.
         """
         dotted_circle_cp = shaper.font.get_nominal_glyph(ord(self.DOTTED_CIRCLE))
 
@@ -95,7 +121,6 @@ class CheckBase:
         """
         options = {
             "check": [SupportLevel.BASE.value],
-            "validity": LanguageValidity.DRAFT.value,
             "decomposed": False,
             "marks": False,
             "report_missing": -1,
@@ -117,3 +142,58 @@ class CheckBase:
                 return category
 
         return None
+
+
+class BrahmiBaseCheck(CheckBase):
+    """
+    A more specific base check for Brahmi scripts.
+    """
+
+    VIRAMA = chr(0x094D)
+    BRAHMIC_CATEGORIES = {
+        "+": [chr(0x094D)],
+        "V": [
+            chr(c)
+            for c in list(range(0x0904, 0x0914 + 1))
+            + [0x0960, 0x0961]
+            + list(range(0x0973, 0x0977 + 1))
+        ],
+        "C": [
+            chr(c)
+            for c in list(range(0x0915, 0x0939 + 1))
+            + list(range(0x0958, 0x095F + 1))
+            + list(range(0x0978, 0x097F + 1))
+        ],
+        "D": [
+            chr(c)
+            for c in [0x093A, 0x093B]
+            + list(range(0x093E, 0x094C + 1))
+            + [0x094E, 0x094F]
+            + [0x0955, 0x0956, 0x0957]
+            + [0x0962, 0x0963]
+        ],
+        "M": [chr(0x093C)],
+        "m": [chr(c) for c in list(range(0x0900, 0x0903 + 1))],
+        "P": [chr(c) for c in [0x093D, 0x0964, 0x0965]],
+        "Z": [chr(0x200C)],
+        "z": [chr(0x200D)],
+    }
+
+    conditions = {
+        "scripts": ("Devanagari",),
+        "attributes": ("combinations",),
+    }
+
+    requires_font = True
+
+    def precheck(
+        self,
+        orthography: Orthography,
+        checker: Checker,
+        **kwargs,
+    ) -> bool:
+        super().precheck(orthography, checker, **kwargs)
+        if not orthography.combinations:
+            return False
+
+        return True

{hyperglot-0.8.0 → hyperglot-0.8.2}/lib/hyperglot/checker.py

@@ -3,10 +3,17 @@ from fontTools.ttLib import TTFont
 import os
 import sys
 import importlib.util
+import functools
 from typing import List, Set
 from collections.abc import Iterable
 
-from hyperglot import DB_CHECKS, SupportLevel, LanguageValidity, OrthographyStatus, LanguageStatus
+from hyperglot import (
+    DB_CHECKS,
+    SupportLevel,
+    LanguageValidity,
+    OrthographyStatus,
+    LanguageStatus,
+)
 from hyperglot.checkbase import CheckBase
 from hyperglot.shaper import Shaper
 from hyperglot.languages import Languages
@@ -30,6 +37,43 @@ def format_missing_unicodes(codepoints: Set[str], reference) -> str:
         return (" ".join(["%s (%s)" % (c, str(ord(c))) for c in diff]),)
 
 
+@functools.cache
+def _load_check_modules() -> tuple:
+    """
+    Import every check module in lib/hyperglot/checks, once per process.
+
+    Checks are matched against each orthography of each language, and importing
+    the modules anew every time made loading them the bulk of a check's run
+    time. The modules only hold classes; each run still gets fresh Check
+    instances, so no per-run state is shared.
+    """
+    modules = []
+    check_files = [
+        f for f in os.listdir(os.path.join(DB_CHECKS)) if f.endswith(".py")
+    ]
+
+    for f in check_files:
+        module_file = os.path.join(DB_CHECKS, f)
+        module_name = os.path.splitext(f)[0]
+
+        try:
+            spec = importlib.util.spec_from_file_location(module_name, module_file)
+            c = importlib.util.module_from_spec(spec)
+            sys.modules[module_name] = c
+            spec.loader.exec_module(c)
+            check = c.Check()
+        except Exception as e:
+            log.error(f"Failed to instantiate check {module_name}. Fix the below:")
+            raise e
+
+        if not isinstance(check, CheckBase):
+            raise Exception(f"Check {module_name} needs to subclass CheckBase!")
+
+        modules.append((module_name, c))
+
+    return tuple(modules)
+
+
 class Checker:
     """
     A base class for CharsetChecker and FontChecker encapsulating language
@@ -38,53 +82,40 @@ class Checker:
 
     def __init__(self, fontpath: str = None, characters: List = None):
         self.fontpath = fontpath
-        self.characters = characters
+        self.characters = characters if characters is not None else []
         self.font = None
         self.shaper = None
 
-    def _get_checks_for_orthography(self, orthography: Orthography):
+    @classmethod
+    def _get_checks_for_orthography(
+        cls, orthography: Orthography, perform_shaping_checks: bool = False
+    ) -> List[tuple[str, int, CheckBase]]:
         """
         Parse lib/hyperglot/checks and return all Check classes where
         conditions are satisfied for this language.
         """
         checks = []
 
-        check_files = [
-            f for f in os.listdir(os.path.join(DB_CHECKS)) if f.endswith(".py")
-        ]
-
-        for f in check_files:
-            module_file = os.path.join(DB_CHECKS, f)
-            module_name = os.path.splitext(f)[0]
-
-            try:
-                spec = importlib.util.spec_from_file_location(module_name, module_file)
-                c = importlib.util.module_from_spec(spec)
-                sys.modules[module_name] = c
-                spec.loader.exec_module(c)
-                check = c.Check()
-            except Exception as e:
-                log.error(f"Failed to instantiate check {module_name}. Fix the below:")
-                raise e
-
-            if not isinstance(check, CheckBase):
-                raise Exception(f"Check {module_name} needs to subclass CheckBase!")
+        for module_name, c in _load_check_modules():
+            check = c.Check()
 
             # Ignore checks that only run when checking with a font
-            if check.requires_font and self.shaper is None:
+            if check.requires_font and not perform_shaping_checks:
                 continue
 
             fulfills = True
 
-            if "script" in check.conditions:
-                if "script" in check.conditions:
-                    if check.conditions["script"] != orthography["script"]:
-                        fulfills = False
-            if "attributes" in check.conditions["attributes"]:
-                for a in check.conditions["attributes"]:
-                    if a not in orthography:
+            if "scripts" in check.conditions:
+                if "scripts" in check.conditions:
+                    if orthography["script"] not in check.conditions["scripts"]:
                         fulfills = False
 
+            if fulfills and "attributes" in check.conditions:
+                # If _none_ of the attributes exist, the check is not required
+                # to run.
+                if not any([a in orthography for a in check.conditions["attributes"]]):
+                    fulfills = False
+
             if fulfills:
                 checks.append((module_name, check.priority, c.Check()))
 
@@ -272,16 +303,16 @@ class Checker:
         if orthographies == []:
             return {} if return_script_object else False
 
-        self.shaper = None
-        if shaping:
-            # Setup a reusable shaper to run checks with
+        if shaping and self.shaper is None:
             self.shaper = Shaper(self.fontpath)
 
         for ort in orthographies:
             supported = True
-            checks = self._get_checks_for_orthography(ort)
+            checks = self._get_checks_for_orthography(
+                orthography=ort, perform_shaping_checks=shaping
+            )
 
-            # FIXME TBD Run all checks, even after one has failed, to get 
+            # FIXME TBD Run all checks, even after one has failed, to get
             # logging output
             for check_name, _priority, c in checks:
                 result = c.check(
@@ -352,7 +383,6 @@ class FontChecker(Checker):
         super().__init__(fontpath=fontpath)
 
         self.font = TTFont(fontpath, lazy=True)
-        self.shaper = Shaper(fontpath)
         self.characters = self._parse_font_chars()
 
     def get_supported_languages(self, **kwargs):

{hyperglot-0.8.0 → hyperglot-0.8.2}/lib/hyperglot/checks/check_brahmi_conjuncts.py

@@ -2,23 +2,42 @@ import re
 import logging
 from typing import Tuple
 
-from hyperglot.checkbase import CheckBase
+from hyperglot.checkbase import BrahmiBaseCheck
+from hyperglot.checker import Checker
 from hyperglot.orthography import Orthography
 from hyperglot.shaper import Shaper
 
 log = logging.getLogger(__name__)
 
 
-class Check(CheckBase):
+class Check(BrahmiBaseCheck):
+    """
+    Check to confirm conjunct formation for unencoded combinations, e.g.
+    consonant + virama + consonant in Devanagari, based off the 'combinations'
+    orthography attribute.
+
+    The threshold option can be used to fine tune the granularity of the check,
+    e.g. allowing for less frequently used conjuncts to not cause the check to
+    fail, if below this threshold (based on 'combinations' frequency data from
+    a parsed language corpus)
+    """
+
+    # conditions and requires_font from BrahmiBaseCheck
 
-    conditions = {
-        "script": "Devanagari",
-        "attributes": ("combinations",),
-    }
-    requires_font = True
     priority = 50
     logger = logging.getLogger("hyperglot.reporting.conjuncts")
 
+    def precheck(self, orthography: Orthography, checker: Checker, **kwargs) -> bool:
+        super().precheck(orthography, checker, **kwargs)
+
+        self.conjuncts = dict(
+            filter(self.filter_conjuncts, orthography.combinations.items())
+        )
+        if self.conjuncts == {}:
+            return False
+
+        return True
+
     def check(self, orthography: Orthography, checker, **kwargs) -> bool:
         """
         the sequence of the consonant Ka, virama, and consonant Pa will result in a conjunct KPa
@@ -38,24 +57,16 @@ class Check(CheckBase):
         Ka + virama + Ka → KKa
         ZWJ ensures a half form is rendered instead:
         Ka + virama + ZWJ + Ka → K- + Ka
-
         """
-        options = self._get_options(**kwargs)
 
-        if not orthography.combinations:
-            return True
-        conjuncts = dict(
-            filter(self.filter_conjuncts, orthography.combinations.items())
-        )
-        if conjuncts == {}:
-            return True
+        super().check(orthography, checker, **kwargs)
 
         # Iterate all syllables to provide reporting.
         # TODO exit early if no reporting is set
 
         fails_over_threshold = False
 
-        for conjunct, frequency in conjuncts.items():
+        for conjunct, frequency in self.conjuncts.items():
             fails = False
             # Ensure no .notdef are left over
             if not self.check_all_render(conjunct, checker.shaper):
@@ -65,9 +76,9 @@ class Check(CheckBase):
                 fails = True
 
             if fails:
-                fmt_threshold = str(options["threshold"]).format(".5f")
+                fmt_threshold = str(self.options["threshold"]).format(".5f")
 
-                if frequency > options["threshold"]:
+                if frequency > self.options["threshold"]:
                     log.error(
                         f"Conjunct '{conjunct}' ({frequency:.5f}) does not shape "
                         f"but is required by frequency threshold ({fmt_threshold})."

{hyperglot-0.8.0 → hyperglot-0.8.2}/lib/hyperglot/checks/check_brahmi_halfforms.py

@@ -1,24 +1,28 @@
 import logging
 
-from hyperglot.checkbase import CheckBase
+from hyperglot.checkbase import BrahmiBaseCheck
 from hyperglot.shaper import Shaper
 
 log = logging.getLogger(__name__)
 
 
-class Check(CheckBase):
+class Check(BrahmiBaseCheck):
+    """
+    Check to confirm halfform shaping for unencoded combinations, e.g.
+    consonant + virama in Devanagari.
+
+    To find the half form constituents the checks looks at the 'combinations'
+    orthography attribute and filters for clusters of the form
+    consonant + virama.
+    """
+
+    # conditions and requires_font from BrahmiBaseCheck
 
-    conditions = {
-        "script": "Devanagari",
-        "attributes": ("combinations",),
-    }
-    requires_font = True
     priority = 50
     logger = logging.getLogger("hyperglot.reporting.halfforms")
 
     def check(self, orthography, checker, **kwargs):
-        if not orthography.combinations:
-            return True
+        super().check(orthography, checker, **kwargs)
 
         for h in filter(self.filter_halfforms, orthography.combinations.keys()):
             if not self.check_halfform(h, checker.shaper):

{hyperglot-0.8.0 → hyperglot-0.8.2}/lib/hyperglot/checks/check_combination_marks.py

@@ -1,25 +1,26 @@
 import logging
 import unicodedata
 
-from hyperglot.checkbase import CheckBase
+from hyperglot.checkbase import BrahmiBaseCheck
 from hyperglot.shaper import Shaper
 
 log = logging.getLogger(__name__)
 
 
-class Check(CheckBase):
+class Check(BrahmiBaseCheck):
+    """
+    Check mark attachment for unencoded combinations, e.g.
+    consonant + virama + consonant in Devanagari.
+    """
+
+    # conditions and requires_font from BrahmiBaseCheck
 
-    conditions = {
-        "script": "Devanagari",
-        "attributes": ("combinations",),
-    }
-    requires_font = True
     priority = 75
     logger = logging.getLogger("hyperglot.reporting.marks")
 
     def check(self, orthography, checker, **kwargs):
-        if not orthography.combinations:
-            return True
+
+        super().check(orthography, checker, **kwargs)
 
         for c in orthography.combinations.keys():
             if not self.check_cluster_mark_attachment(c, checker.shaper):