pylelemmatize 0.1.0__py3-none-any.whl

pylelemmatize/__init__.py

@@ -0,0 +1,52 @@
+from typing import Dict, Literal, Optional
+from .char_distance import char_similarity
+from .main_scripts import main_alphabet_extract_corpus_alphabet, main_alphabet_evaluate_merges
+from .abstract_mapper import AbstractLemmatizer, GenericLemmatizer, fast_cer, fast_numpy_to_str, fast_str_to_numpy
+from .fast_mapper import LemmatizerBMP
+#from .charset import allbmp_encoding_alphabet_strings, allnonbmp_encoding_alphabet_strings, main_map_test_corpus_on_alphabets
+
+#from .charset import chr_iso, chr_mes, chr_ascii, chr_mufi_bmp, chr_mufi_nonbmp
+from .all_charsets import Charsets
+charsets = Charsets()
+
+from .util import extract_transcription_from_page_xml, main_extract_transcription_from_page_xml, print_err
+import sys
+from .philogeny import main_char_similarity_tree
+from .version import version
+from .substitution_augmenter import CharConfusionMatrix
+__version__ = version
+
+
+default_unknown_chr = "�"
+
+def create_lemmatizer(src_alphabet_str: str, dst_alphabet_str: Optional[str]=None,
+                  mapper_type: Literal["fast", "generic", "guess"] = "guess", unknown_chr: str = default_unknown_chr,
+                  override_map: Optional[Dict[str, str]] = None, min_similarity: float = .3) -> GenericLemmatizer:
+    if mapper_type == "guess":
+        if LemmatizerBMP.alphabet_in_bmp(src_alphabet_str)  and LemmatizerBMP.alphabet_in_bmp(dst_alphabet_str):
+            mapper_type = "fast"
+        else:
+            mapper_type = "generic"
+    if mapper_type == "fast":
+        return LemmatizerBMP.from_alphabet_mapping(src_alphabet_str, dst_alphabet_str, unknown_chr=unknown_chr,
+                                                   override_map=override_map, min_similarity=min_similarity)
+    elif mapper_type == "generic":
+        return GenericLemmatizer.from_alphabet_mapping(src_alphabet_str, dst_alphabet_str, unknown_chr=unknown_chr, 
+                                                       override_map=override_map, min_similarity=min_similarity)
+    else:
+        raise ValueError(f"Unknown mapper type: {mapper_type}")
+
+
+__all__ = ["create_lemmatizer", "AbstractLemmatizer", "GenericLemmatizer", "LemmatizerBMP",
+           "char_similarity", "fast_cer", "fast_numpy_to_str", "fast_str_to_numpy",
+           "Charsets", "charset",
+           "extract_transcription_from_page_xml", "print_err",
+           "CharConfusionMatrix", "__version__"]
+
+if "torch" in sys.modules or "sphinx" in sys.modules:  # to allow doc generation without torch
+    from .mapper_ds import Seq2SeqDs
+    from .demapper_lstm import DemapperLSTM, main_train_one2one, main_infer_one2one, main_report_demapper
+    __all__.extend(["Seq2SeqDs", "DemapperLSTM", "main_train_one2one", "main_infer_one2one", "main_report_demapper"])
+else:
+    print("Warning: Torch is not loaded. Seq2SeqDs will not be available.", file=sys.stderr)
+

pylelemmatize/abstract_mapper.py

@@ -0,0 +1,345 @@
+from difflib import SequenceMatcher
+import sys
+
+from typing import Any, Dict, Generator, Literal, Optional, Set, Tuple, Union
+from collections import defaultdict
+import numpy as np
+from abc import ABC, abstractmethod
+import unicodedata
+from .char_distance import char_similarity
+
+
+def fast_str_to_numpy(s: str, dtype=np.uint16) -> np.ndarray:
+    if dtype == np.uint16:
+        return np.frombuffer(s.encode('utf-16le'), dtype=dtype)
+    elif dtype == np.uint32:
+        return np.frombuffer(s.encode('utf-32le'), dtype=dtype)
+    elif dtype == np.uint64:
+        return np.frombuffer(s.encode('utf-64le'), dtype=dtype)
+    elif dtype == np.uint8:
+        return np.frombuffer(s.encode('utf-8'), dtype=dtype)
+    else:
+        raise ValueError(f"Unsupported dtype: {dtype}")
+
+
+def fast_numpy_to_str(np_arr: np.ndarray) -> str:
+    if np_arr.dtype == np.uint16:
+        return np_arr.tobytes().decode('utf-16le')
+    elif np_arr.dtype == np.uint32:
+        return np_arr.tobytes().decode('utf-32le')
+    elif np_arr.dtype == np.uint64:
+        return np_arr.tobytes().decode('utf-64le')
+    elif np_arr.dtype == np.uint8:
+        return np_arr.tobytes().decode('utf-8')
+    else:
+        raise ValueError(f"Unsupported dtype: {np_arr.dtype}")
+
+
+def fast_cer(pred: str, true: str) -> float:
+    np_pred = fast_str_to_numpy(pred)
+    np_true = fast_str_to_numpy(true)
+    return np.mean(np_pred != np_true)
+
+
+
+
+class AbstractLemmatizer(ABC):
+    """Abstract base class for lemmatizers that map characters from a source alphabet to a destination alphabet.
+    
+    Attributes:
+        src_alphabet_str (str): The source alphabet string.
+        dst_alphabet_str (str): The destination alphabet string.
+        unknown_chr (str): The character used for unknown mappings. Default is "�".
+        normalize_unicode (Callable[[str], str]): Function to normalize Unicode strings.
+    """
+    def _densify_unicode(self: Any, text: str) -> str:
+        """
+        Convert a Unicode string to its composed (dense) form using NFC normalization.
+        
+        Args:
+            text (str): The input Unicode string.
+        
+        Returns:
+            str: The composed (dense) Unicode string.
+        """
+        return unicodedata.normalize('NFC', text)
+
+    def _decompose_unicode(self: Any, text: str) -> str:
+        """
+        Convert a Unicode string to its decomposed (sparse) form using NFD normalization.
+        
+        Args:
+            text (str): The input Unicode string.
+        
+        Returns:
+            str: The decomposed (sparse) Unicode string.
+        """
+        return unicodedata.normalize('NFD', text)
+
+
+    def _null_unicode(self: Any, text: str) -> str:
+        """
+        Normalize a Unicode string to its composed (dense) form using NFC normalization.
+        
+        Args:
+            text (str): The input Unicode string.
+        
+        Returns:
+            str: The normalized Unicode string.
+        """
+        return text
+
+    @classmethod
+    def __create_mappers(cls, mapping_dict, unknown_chr) -> Tuple[defaultdict, np.ndarray, Dict[int, str], Dict[str, int], np.ndarray, np.ndarray, np.ndarray]:
+        max_ord = max(ord(c) for c in mapping_dict.keys() | mapping_dict.values() | {unknown_chr})
+        assert max_ord < 256**3, f"All characters must be whithin resoble size (256^3). found {max_ord}."
+
+        if unknown_chr in mapping_dict:
+            assert mapping_dict[unknown_chr] == unknown_chr, "unknown_chr must map to itself in the mapping_dict."
+            del mapping_dict[unknown_chr]  # Remove the unknown character from the mapping to avoid confusion
+
+        src_alphabet_str = ''.join(sorted(mapping_dict.keys()))
+        dst_alphabet_str = ''.join(sorted(set(mapping_dict.values())))
+
+        chr2chr = defaultdict(lambda: unknown_chr)
+        chr2chr.update(mapping_dict)
+        
+        dense2src_dst = [(n + 1, (c, chr2chr[c])) for n, c in enumerate(src_alphabet_str)]
+
+        src_str = [(s,s) for _, (s, _) in dense2src_dst]
+        src_str = ''.join([s for _, s in sorted(src_str)])
+
+        dst_str = [(s, d) for _, (s, d) in dense2src_dst]
+        dst_str = [(s, d) for s, d in dst_str if d != unknown_chr]  # Remove unknown characters from the destination string
+        dst_str = ''.join([d for _, d in sorted(dst_str)])
+        
+        src_full_str = unknown_chr + src_str
+        
+        srcchr2dense = {s: n for n, s in enumerate(src_full_str)}
+
+        np_chrord2dense = np.zeros(max_ord, dtype=np.uint16)
+        np_dense2chrord = np.zeros(max_ord, dtype=np.uint16)
+        for c, n in srcchr2dense.items():
+            np_chrord2dense[ord(c)] = n
+            np_dense2chrord[n] = ord(chr2chr[c])
+        return src_alphabet_str, dst_alphabet_str, np_chrord2dense, np_dense2chrord
+
+    @classmethod
+    def fast_alphabet_extraction(cls, text: str) -> str:
+        np_text = fast_str_to_numpy(text)
+        uniq = np.unique(np_text)
+        return fast_numpy_to_str(uniq)
+    
+    def __set_unicode_normalization(self, normalize_unicode: Literal["Dense", "Composite", None] = "Dense"):
+        """
+        np_text = fast_str_to_numpy(text)
+        Set the Unicode normalization method.
+        
+        Args:
+            normalize_unicode (Literal["Dense", "Composite", None]): The normalization type.
+                - "Dense" for NFC normalization (composed form).
+                - "Composite" for NFD normalization (decomposed form).
+                - None for no normalization.
+        """
+        self.__unicode_normalization = normalize_unicode
+        if normalize_unicode.lower() == "dense":
+            self.normalize_unicode = self._densify_unicode
+        elif normalize_unicode == "composite":
+            self.normalize_unicode = self._decompose_unicode
+        elif normalize_unicode is None:
+            self.normalize_unicode = self._null_unicode
+        else:
+            raise ValueError(f"Unknown normalization type: {normalize_unicode}")
+
+    def __init__(self, unicode_normalization: Literal["Dense", "Composite", None] = "Dense", unknown_chr: str = "�"):
+        super().__init__()
+        self.__set_unicode_normalization(unicode_normalization)
+        self.__unknown_chr = unknown_chr
+
+    @property
+    def unicode_normalization(self) -> Literal["Dense", "Composite", None]:
+        return self.__unicode_normalization
+
+    @abstractmethod
+    def __call__(self, text: str) -> str:
+        """Convert text to the alphabet representation."""
+        pass
+
+    @property
+    @abstractmethod
+    def src_alphabet_str(self) -> str:
+        pass
+
+    @property
+    @abstractmethod
+    def dst_alphabet_str(self) -> str:
+        pass
+
+    @property
+    def unknown_chr(self) -> str:
+        return self.__unknown_chr
+
+    @abstractmethod
+    def __repr__(self):
+        raise NotImplementedError("Subclasses must implement __repr__ method.")
+
+    @property
+    def alphabet_tsv(self) -> str:
+        title = " #\tUnicode Number\tUnicode x10\tUnicode x16\tPython String"
+        lines = [title]
+        for n, c in enumerate(self.unknown_chr + self.dst_alphabet_str):
+            lines.append(f"{n}\t{unicodedata.name(c)}\t{ord(c)}\t{ord(c):x}\t{repr(c)}")
+        return "\n".join(lines)
+    
+    @property
+    def mapping_tsv(self) -> str:
+        title = "Source Character\tDestination Character"
+        lines = [title]
+        for src, dst in sorted(self.mapping_dict.items()):
+            lines.append(f"{repr(src)}\t{repr(dst)}")
+        return "\n".join(lines)
+    
+    def __ne__(self, other: Any) -> bool:
+        if not isinstance(other, AbstractLemmatizer):
+            return True
+        else:
+            return self.mapping_tsv != other.mapping_tsv
+
+    def __eq__(self, other: Any) -> bool:
+        if not isinstance(other, AbstractLemmatizer):
+            return False
+        else:
+            return self.mapping_tsv == other.mapping_tsv
+    
+    def __len__(self) -> int:
+        """Return the size of the destination alphabet."""
+        return len(self.dst_alphabet_str) + 1
+
+    def get_unigram(self, text: str) -> Tuple[np.ndarray, np.ndarray, Dict[int, str]]:
+        # adding all characters atleast once to make np.unique count zero counts
+        #src_alphabet_str, _, __np_chrord2dense, __np_dense2chrord = self.__create_mappers(self.mapping_dict, self.unknown_chr)
+        np_text = fast_str_to_numpy(self.unknown_chr + self.src_alphabet_str + text)
+        mapped_np_text = self.__np_chrord2dense[np_text]
+        values, counts = np.unique(mapped_np_text, return_counts=True)
+        counts = counts - 1  # removing the counts of the added characters
+        labels = np.array([c for c in fast_numpy_to_str(self.__np_dense2chrord[values])], dtype=np.str_)
+        return values, counts, labels
+
+    def get_cer(self, pred: str, true: str) -> float:
+        np_pred = fast_str_to_numpy(pred)
+        np_true = fast_str_to_numpy(true)
+        mapped_np_pred = self.__npint2int[np_pred]
+        mapped_np_true = self.__npint2int[np_true]
+        return np.mean(mapped_np_pred != mapped_np_true)
+
+    def get_encoding_information_loss(self, text: str) -> float:
+        np_text = fast_str_to_numpy(text)
+        mapped_np_text = self.__npint2int[np_text]
+        return np.mean(np_text != mapped_np_text)
+
+    # def get_unigram(self, text: str) -> Tuple[np.ndarray, np.ndarray, Dict[int, str]]:
+    #     # adding all characters atleast once to make np.unique count zero counts
+    #     src_alphabet_str, _, __np_chrord2dense, __np_dense2chrord = AbstractLemmatizer.__create_mappers(self.mapping_dict, self.unknown_chr)
+    #     np_text = fast_str_to_numpy(self.unknown_chr + src_alphabet_str + text)
+    #     mapped_np_text = __np_chrord2dense[np_text]
+    #     values, counts = np.unique(mapped_np_text, return_counts=True)
+    #     counts = counts - 1  # removing the counts of the added characters
+    #     labels = np.array([c for c in fast_numpy_to_str(__np_dense2chrord[values])], dtype=np.str_)
+    #     labels[0] = self.unknown_chr  # Ensure the unknown character is included
+    #     print("\n\nLABELS:\n", labels, "\n\n\n")
+    #     return values, counts, labels
+
+
+class GenericLemmatizer(AbstractLemmatizer):
+    @classmethod
+    def from_alphabet_mapping(cls, src_alphabet_str: str, dst_alphabet_str: Optional[str] = None,
+                              unknown_chr: str = "�", override_map: Optional[Dict[str, str]] = None,
+                              min_similarity: float = .25, verbose: int = 0) -> 'GenericLemmatizer':
+        if dst_alphabet_str is None:
+            mapping_dict = {c:c for c in src_alphabet_str}
+            if unknown_chr not in mapping_dict:
+                mapping_dict[unknown_chr] = unknown_chr
+            return cls(mapping_dict=mapping_dict, unknown_chr=unknown_chr)
+        mapping_dict = defaultdict(lambda: unknown_chr)  # Default to unknown character
+        if override_map is not None:
+            mapping_dict.update(override_map)  # Map destination characters to themselves
+            remain_sources = []
+            for c in src_alphabet_str:
+                if c not in override_map:
+                    remain_sources.append(c)
+        else:
+            remain_sources = list(src_alphabet_str)
+        if unknown_chr not in dst_alphabet_str:
+            dst_alphabet_str = unknown_chr + dst_alphabet_str
+        s_map = np.zeros((len(remain_sources), len(dst_alphabet_str)))
+        unknown_chr_dst_idx = dst_alphabet_str.index(unknown_chr)
+        for src_n, src_c in enumerate(remain_sources):
+            for dst_n, dst_c in enumerate(dst_alphabet_str):
+                s_map[src_n, dst_n] = char_similarity(src_c, dst_c)
+
+        too_small = s_map < min_similarity
+        s_map[too_small] = 0  # Apply minimum similarity threshold
+        s_map[:, unknown_chr_dst_idx]+= .00001  # Set similarity to unknown character to 0
+
+        if verbose:
+            print(f"Similarity matrix:\n{s_map}\n", file=sys.stderr)
+
+        dst_symbol_array = np.array(list(dst_alphabet_str), dtype=np.str_)
+        src_symbol_array = np.array(list(remain_sources), dtype=np.str_)
+        best_dst_idx = np.argmax(s_map, axis=1)
+
+        if verbose:
+            print(f"Best destination indices: {best_dst_idx}\n", file=sys.stderr)
+            print(f"Source symbols: {src_symbol_array}\n", file=sys.stderr)
+            print(f"Destination symbols: {dst_symbol_array}\n", file=sys.stderr)
+            if verbose > 1:
+                for n in range(s_map.shape[0]):
+                    print(f"Source: {repr(src_symbol_array[n])} -> Best Destination: {repr(dst_symbol_array[best_dst_idx[n]])} with similarity {s_map[n, best_dst_idx[n]]:.4f}", file=sys.stderr)
+        mapping_dict.update({src_symbol_array[n]: dst_symbol_array[best_dst_idx[n]] for n in range(len(remain_sources))})
+        mapping_dict = {k: v for k, v in sorted(mapping_dict.items())}
+        if verbose:
+            print(f"Mapping dictionary: {mapping_dict}\n", file=sys.stderr)
+        if unknown_chr not in mapping_dict:
+            mapping_dict[unknown_chr] = unknown_chr
+        else:
+            assert mapping_dict[unknown_chr] == unknown_chr, f"unknown_chr must map to itself in the mapping_dict. Found {repr(mapping_dict[unknown_chr])} instead of {repr(unknown_chr)}."
+        mapping_dict = {str(k): str(v) for k, v in sorted(mapping_dict.items())}
+        return cls(mapping_dict=mapping_dict, unknown_chr=unknown_chr)
+
+    def copy_removing_unused_inputs(self, txt: str) -> Any:
+        txt = self.normalize_unicode(txt)
+        txt = self.fast_alphabet_extraction(txt)
+        mapping_dict = self.mapping_dict.copy()
+        reduced_mapping_dict = {}
+        for k in txt:
+            if k in mapping_dict:
+                reduced_mapping_dict[k] = mapping_dict[k]
+        return self.__class__(mapping_dict=reduced_mapping_dict, unknown_chr=self.unknown_chr, unicode_normalization=self.unicode_normalization)
+
+
+    def __init__(self, mapping_dict = {}, unknown_chr: str = "�", unicode_normalization: Literal["Dense", "Composed", None] = "Dense"):
+        super().__init__(unicode_normalization=unicode_normalization, unknown_chr=unknown_chr)
+        #print(f"Creating GenericLemmatizer with mapping_dict={repr(mapping_dict)} and unknown_chr={repr(unknown_chr)}")
+        self.mapping_dict = mapping_dict.copy()
+    
+    def len(self) -> int:
+        """Return the size of the destination alphabet."""
+        return len(self.dst_alphabet_str) + 1
+
+    def __call__(self, text: str) -> str:
+        text = self.normalize_unicode(text)  #  This might be the bottleneck in a fast implementation.
+        return ''.join(self.mapping_dict.get(c, self.unknown_chr) for c in text)
+
+    @property
+    def src_alphabet_str(self) -> str:
+        return ''.join(sorted(self.mapping_dict.keys()))
+
+    @property
+    def dst_alphabet_str(self) -> str:
+        res = ''.join(sorted(set(self.mapping_dict.values())))
+        if self.unknown_chr not in res:
+            res += self.unknown_chr
+        return res
+
+    def __repr__(self):
+        return f"GenericLemmatizer(mapping_dict={repr(self.mapping_dict)}, unknown_chr={repr(self.unknown_chr)})"

pylelemmatize/all_charsets.py

@@ -0,0 +1,282 @@
+from typing import Dict, List
+
+
+from .charset_iso import get_encoding_dicts as get_iso_encoding_dicts
+from .charset_mes import get_encoding_dicts as get_mes_encoding_dicts
+from .charset_ascii import get_encoding_dicts as get_ascii_encoding_dicts
+from .charset_mufi import get_encoding_dicts as get_mufi_encoding_dicts
+
+
+
+from .abstract_mapper import  fast_cer, GenericLemmatizer
+from .fast_mapper import LemmatizerBMP
+
+
+class Charsets:
+    """A class that provides access to various character sets as unicode strings.
+
+    This class is a singleton and is instantiated as `charset` in the `pylelemmatize` package.
+
+    Usage::
+        from pylelemmatize import Charsets
+        charsets = Charsets()
+        print(charsets.mes1)  # prints the MES1 character set as a string
+    """
+    def __init__(self):
+        self.lib_cache__ = {}
+        self.alphabets = ["mes1", "mes2", "mes3a", "mes3b",
+                     "iso_8859_1", "iso_8859_2", "iso_8859_3", "iso_8859_4",
+                     "iso_8859_5", "iso_8859_6", "iso_8859_7", "iso_8859_8",
+                     "iso_8859_9", "iso_8859_10", "iso_8859_11", "iso_8859_12", "iso_8859_13",
+                     "iso_8859_14", "iso_8859_15", "iso_8859_16",
+                     "ascii_lowercase", "ascii_uppercase", "ascii_letters", "digits",
+                     "hexdigits", "octdigits", "punctuation",
+                     "bmp_mufi", "bmp_pua_mufi", "nonbmp_mufi",
+                     "nonbmp_pua_mufi"]
+
+    @property
+    def mes_charsets(self):
+        if self.lib_cache__.get("mes") is None:
+            self.lib_cache__["mes"] = get_mes_encoding_dicts()
+        return self.lib_cache__["mes"]
+    @property
+    def mes1(self):
+        return self.mes_charsets["mes1"]
+    @property
+    def mes2(self):
+        return self.mes_charsets["mes2"]
+    @property
+    def mes3a(self):
+        return self.mes_charsets["mes3a"]
+    @property
+    def mes3b(self):
+        return self.mes_charsets["mes3b"]
+
+    @property
+    def iso_charsets(self):
+        if self.lib_cache__.get("iso") is None:
+            self.lib_cache__["iso"] = get_iso_encoding_dicts()
+        return self.lib_cache__["iso"]
+    @property
+    def iso_8859_2(self):
+        return self.iso_charsets["iso8859_2"]
+    @property
+    def iso_8859_3(self):
+        return self.iso_charsets["iso8859_3"]
+    @property
+    def iso_8859_4(self):
+        return self.iso_charsets["iso8859_4"]
+    @property
+    def iso_8859_5(self):
+        return self.iso_charsets["iso8859_5"]
+    @property
+    def iso_8859_6(self):
+        return self.iso_charsets["iso8859_6"]
+    @property
+    def iso_8859_7(self):
+        return self.iso_charsets["iso8859_7"]
+    @property
+    def iso_8859_8(self):
+        return self.iso_charsets["iso8859_8"]
+    @property
+    def iso_8859_9(self):
+        return self.iso_charsets["iso8859_9"]
+    @property
+    def iso_8859_10(self):
+        return self.iso_charsets["iso8859_10"]
+    @property
+    def iso_8859_11(self):  
+        return self.iso_charsets["iso8859_11"]
+    @property
+    def iso_8859_12(self):
+        return self.iso_charsets["iso8859_12"]
+    @property
+    def iso_8859_13(self):
+        return self.iso_charsets["iso8859_13"]
+    @property
+    def iso_8859_14(self):
+        return self.iso_charsets["iso8859_14"]
+    @property
+    def iso_8859_15(self):
+        return self.iso_charsets["iso8859_15"]
+    @property
+    def iso_8859_16(self):
+        return self.iso_charsets["iso8859_16"]
+
+    @property
+    def ascii_charsets(self):
+        if self.lib_cache__.get("ascii") is None:
+            self.lib_cache__["ascii"] = get_ascii_encoding_dicts()
+        return self.lib_cache__["ascii"]
+    @property
+    def ascii(self):
+        return self.ascii_charsets["ascii"]
+    @property
+    def ascii_lowercase(self):
+        return self.ascii_charsets["ascii_lowercase"]
+    @property
+    def ascii_uppercase(self):
+        return self.ascii_charsets["ascii_uppercase"]
+    @property
+    def ascii_letters(self):
+        return self.ascii_charsets["ascii_letters"]
+    @property
+    def digits(self):
+        return self.ascii_charsets["digits"]
+    @property
+    def hexdigits(self):
+        return self.ascii_charsets["hexdigits"]
+    @property
+    def octdigits(self):
+        return self.ascii_charsets["octdigits"]
+    @property
+    def punctuation(self):
+        return self.ascii_charsets["punctuation"]
+
+    @property
+    def mufibmp_charsets(self):
+        if self.lib_cache__.get("mufibmp") is None:
+            self.lib_cache__["mufibmp"], self.lib_cache__["mufinonbmp"] = get_mufi_encoding_dicts()
+        return self.lib_cache__["mufibmp"]
+    @property
+    def mufinonbmp_charsets(self):
+        if self.lib_cache__.get("mufinonbmp") is None:
+            self.lib_cache__["mufibmp"], self.lib_cache__["mufinonbmp"] = get_mufi_encoding_dicts()
+        return self.lib_cache__["mufinonbmp"]
+    @property
+    def mufibmp(self):
+        return self.mufibmp_charsets["bmp_mufi"]
+    @property
+    def mufibmp_pua(self):
+        return self.mufibmp_charsets["bmp_pua_mufi"]
+    @property
+    def mufinonbmp(self):
+        return self.mufinonbmp_charsets["nonbmp_mufi"]
+    @property
+    def mufinonbmp_pua(self):
+        return self.mufinonbmp_charsets["nonbmp_pua_mufi"]
+
+    def __getitem__(self, item):
+        return getattr(self, item)
+    
+    def __iter__(self):
+        return iter(self.alphabets)
+
+    def __len__(self):
+        return len(self.alphabets)
+    
+    def __contains__(self, item):
+        return item in self.alphabets
+    
+    def keys(self):
+        return self.alphabets
+    
+    def items(self):
+        return [(k, getattr(self, k)) for k in self.alphabets]
+
+
+def main_map_test_corpus_on_alphabets():
+    import fargv, glob, time, sys, tqdm
+    from matplotlib import pyplot as plt
+    from .util import generate_corpus
+
+    def plot_covverage(corpus_str, encoding_alphabet_strings, alphabet_to_cer: Dict[str, float], alphabet_to_missing: Dict[str, str], alphabet_to_unfound: Dict[str, str], save_plot_path: str, show_plot: bool):
+        fig, ax = plt.subplots(3, 1, figsize=(15, 10))
+        alphabet_to_missing_ratio = {k: 100*len(v)/len(corpus_str) for k, v in alphabet_to_missing.items() if len(v) > 0}
+        alphabet_to_unfound_ratio = {k: len(v) for k, v in alphabet_to_unfound.items() if len(v) > 0}
+        ax[0].bar(alphabet_to_cer.keys(), [100*v for v in alphabet_to_cer.values()])
+        ax[0].set_xticklabels(alphabet_to_cer.keys(), rotation=45)
+        ax[0].set_title("CER %")
+        
+        ax[1].bar(alphabet_to_missing_ratio.keys(), alphabet_to_missing_ratio.values())
+        ax[1].set_xticklabels(alphabet_to_missing_ratio.keys(), rotation=45)
+        ax[1].set_title("Character Recall %")
+
+        ax[2].bar(alphabet_to_unfound_ratio.keys(), alphabet_to_unfound_ratio.values())
+        ax[2].set_xticklabels(alphabet_to_unfound_ratio.keys(), rotation=45)
+        ax[2].set_yscale('log')
+        ax[2].set_title("Charset uncovered chars # (waste)")
+        
+        plt.subplots_adjust(hspace=.6)
+        if save_plot_path != "":
+            plt.savefig(save_plot_path)
+        if show_plot:
+            plt.show()
+
+    t = time.time()
+    p = {
+        "alphabets": ["ascii", f"A comma separated listof encodings must be a subset of {repr(allbmp_encoding_alphabet_strings.keys())}. Or 'all'"],
+        "corpus_glob": "",
+        "corpus_files": set([]),
+        "verbose": False,
+        "strip_xml": True,
+        "all_is_xml": False,
+        "output_tsv": "",
+        "hide_plot": False,
+        "save_plot_path": ""
+    }
+    args, _ = fargv.fargv(p)
+
+    if args.alphabets == "all":
+        args.alphabets = list(allbmp_encoding_alphabet_strings.keys())
+    else:
+        args.alphabets = [a.strip() for a in args.alphabets.split(',')]
+
+    if args.corpus_glob != "":
+        glob_files = set(glob.glob(args.corpus_glob))
+    else:
+        glob_files = set([])
+    if len(args.corpus_files) > 0:
+        corpus_files = args.corpus_files
+    else:
+        corpus_files = set([])
+
+    all_corpus_strs = []
+    for file_contnets in generate_corpus(glob_files | corpus_files, verbose=args.verbose,
+                                         strip_xml=args.strip_xml, treat_all_file_as_xml=args.all_is_xml):
+        all_corpus_strs.append(file_contnets)
+    found_corpus_str = ''.join(all_corpus_strs)
+
+    if args.verbose:
+        print(f"{time.time() - t :.2f}: Loaded corpus", file=sys.stderr)
+
+    if args.verbose:
+        alphabet_names = tqdm.tqdm(args.alphabets)
+    else:
+        alphabet_names = args.alphabets
+
+    alphabet_to_cer = {}
+    alphabet_to_missing = {}
+    alphabet_to_unfound = {}
+    for alphabet_name in alphabet_names:
+        if alphabet_name not in allbmp_encoding_alphabet_strings:
+            raise ValueError(f"Alphabet {alphabet_name} not in {allbmp_encoding_alphabet_strings.keys()}")
+
+        alphabet_str = allbmp_encoding_alphabet_strings[alphabet_name]
+        if max(alphabet_str) > '\uffff':
+            #alphabet = Alphabet(alphabet_str=alphabet_str, vectorized_mapper_sz=max(256**2, 1 + ord(max(alphabet_str))))  #  This is a hack to ensure that the vectorized mapper is large enough for all characters in MUFI
+            alphabet = GenericLemmatizer.from_alphabet_mapping(alphabet_str, mapping_dict=None)
+        else:
+            #alphabet = AlphabetBMP(alphabet_str=alphabet_str)  #  This is a hack to ensure that the vectorized mapper is large enough for all characters in MUFI
+            alphabet = LemmatizerBMP.from_alphabet_mapping(alphabet_str)
+
+        mapped_corpus_str = alphabet(found_corpus_str)
+
+        if len(found_corpus_str) > 0:
+            alphabet_to_cer[alphabet_name] = fast_cer(found_corpus_str, mapped_corpus_str)
+            alphabet_to_missing[alphabet_name] = ''.join(sorted(set(found_corpus_str) - set(alphabet_str)))
+            alphabet_to_unfound[alphabet_name] = ''.join(sorted(set(alphabet_str) - set(found_corpus_str)))
+
+        else:
+            alphabet_to_cer[alphabet_name] = 0
+            alphabet_to_missing[alphabet_name] = ""
+            alphabet_to_unfound[alphabet_name] = alphabet_str
+
+    if args.verbose:
+        for k, v in alphabet_to_cer.items():
+            print(f"{k}: CER {100*v:.2f}% Missing: {len(alphabet_to_missing[k])}, Redundant: {len(alphabet_to_unfound[k])}  {repr(alphabet_to_missing[k])}", file=sys.stderr)
+        print(f"Computed all for {len(found_corpus_str)} characters in {time.time() -t :.2f}", file=sys.stderr)
+
+    if not args.hide_plot or args.save_plot_path != "":
+        found_corpus_alphabet_str = ''.join(sorted(set(found_corpus_str)))
+        plot_covverage(found_corpus_alphabet_str, allbmp_encoding_alphabet_strings, alphabet_to_cer, alphabet_to_missing, alphabet_to_unfound, args.save_plot_path, not args.hide_plot)