nomenklatura-mpt 4.1.9__py3-none-any.whl

nomenklatura/matching/compat.py

@@ -0,0 +1,66 @@
+# This module buffers out some of the fingerprints package in anticipation of
+# a future removal of the package. All of the functionality is now contained in
+# rigour, but the different functioning of both could lead to unexpected results.
+# This module is a temporary solution to allow for a smooth transition.
+import logging
+from typing import Iterable, List, Optional
+from functools import lru_cache
+from normality import squash_spaces
+from normality.constants import WS
+from rigour.names import remove_person_prefixes
+from fingerprints.cleanup import clean_name_ascii, clean_name_light
+from fingerprints.types import replace_types
+
+log = logging.getLogger(__name__)
+
+__all__ = [
+    "fingerprint_name",
+    "clean_name_ascii",
+    "clean_name_light",
+    "names_word_list",
+    "name_words",
+]
+
+
+@lru_cache(maxsize=1024)
+def fingerprint_name(original: str) -> Optional[str]:
+    """Fingerprint a legal entity name."""
+    # this needs to happen before the replacements
+    text = original.lower()
+    text = remove_person_prefixes(text)
+    # Super hard-core string scrubbing
+    cleaned = clean_name_ascii(text)
+    if cleaned is None:
+        return None
+    cleaned = replace_types(cleaned)
+    cleaned = squash_spaces(cleaned)
+    if len(cleaned) < 1:
+        return None
+    return cleaned
+
+
+def names_word_list(
+    names: Iterable[str],
+    min_length: int = 1,
+) -> List[str]:
+    """Get a list of tokens present in the given set of names."""
+    words: List[str] = []
+    for name in names:
+        normalized = fingerprint_name(name)
+        if normalized is None:
+            continue
+        for word in normalized.split(WS):
+            if len(word) >= min_length:
+                words.append(word)
+    return words
+
+
+def name_words(name: Optional[str], min_length: int = 1) -> List[str]:
+    """Get a list of tokens present in the given name."""
+    if name is None:
+        return []
+    words: List[str] = []
+    for word in name.split(WS):
+        if len(word) >= min_length:
+            words.append(word)
+    return words

nomenklatura/matching/erun/countries.py

@@ -0,0 +1,42 @@
+from followthemoney import registry, E
+
+from nomenklatura.matching.util import type_pair, has_schema
+
+
+# def obj_country(left: E, right: E) -> float:
+#     """Check if two entities share a country."""
+#     if has_schema(left, right, "LegalEntity"):
+#         return 0.0
+#     lv, rv = type_pair(left, right, registry.country)
+#     if len(lv) == 0 or len(rv) == 0:
+#         return 0.0
+#     common = len(set(lv).intersection(rv))
+#     return 1.0 if common > 0 else -1.0
+#     # if common == 0:
+#     #     return -1.0
+#     # total = len(lv) + len(rv)
+#     # return float(common) / total
+
+
+def org_obj_country_match(left: E, right: E) -> float:
+    """Check if two entities share a country."""
+    if has_schema(left, right, "LegalEntity") and not has_schema(
+        left, right, "Organization"
+    ):
+        return 0.0
+    lv, rv = type_pair(left, right, registry.country)
+    if len(lv) == 0 or len(rv) == 0:
+        return 0.0
+    common = len(set(lv).intersection(rv))
+    return 1.0 if common > 0 else -1.0
+
+
+def per_country_mismatch(left: E, right: E) -> float:
+    """Both persons are linked to different countries."""
+    if not has_schema(left, right, "Person"):
+        return 0.0
+    qv, rv = type_pair(left, right, registry.country)
+    if len(qv) == 0 or len(rv) == 0:
+        return 0.0
+    overlap = len(set(qv).intersection(rv))
+    return 1.0 if overlap == 0 else -0.2

nomenklatura/matching/erun/identifiers.py

@@ -0,0 +1,64 @@
+from typing import Set, Tuple
+from rigour.ids import get_strong_format_names
+
+from followthemoney import EntityProxy, registry
+
+
+HONORARY_STRONG = {registry.phone, registry.email, registry.checksum}
+STRONG_FORMATS = get_strong_format_names()
+
+
+def _get_strong_identifiers(entity: EntityProxy) -> Set[Tuple[str, str]]:
+    strong_ids: Set[Tuple[str, str]] = set()
+    for prop, value in entity.itervalues():
+        if not prop.matchable:
+            continue
+        if prop.format in STRONG_FORMATS:
+            strong_ids.add((prop.format, value))
+        elif prop.type in HONORARY_STRONG:
+            strong_ids.add((prop.name, value))
+    return strong_ids
+
+
+def _get_weak_identifiers(entity: EntityProxy) -> Set[str]:
+    weak_ids: Set[str] = set()
+    for prop, value in entity.itervalues():
+        if not prop.matchable or not prop.type != registry.identifier:
+            continue
+        if prop.format in STRONG_FORMATS:
+            continue
+        weak_ids.add(value)
+    return weak_ids
+
+
+def strong_identifier_match(left: EntityProxy, right: EntityProxy) -> float:
+    """Check if two entities share any strong identifiers."""
+    left_strong = _get_strong_identifiers(left)
+    right_strong = _get_strong_identifiers(right)
+    if len(left_strong) == 0 or len(right_strong) == 0:
+        return 0.0
+    if left_strong.intersection(right_strong):
+        return 1.0
+    left_nofmt = {v for _, v in left_strong}
+    right_nofmt = {v for _, v in right_strong}
+    if left_nofmt.intersection(_get_weak_identifiers(right)):
+        return 0.7
+    if right_nofmt.intersection(_get_weak_identifiers(left)):
+        return 0.7
+    left_fmts = {f for _, f in left_strong}
+    right_fmts = {f for _, f in right_strong}
+    common_fmts = left_fmts.intersection(right_fmts)
+    return -0.2 * len(common_fmts)
+
+
+def weak_identifier_match(left: EntityProxy, right: EntityProxy) -> float:
+    """Check if two entities share any weak identifiers."""
+    left_ids = _get_weak_identifiers(left)
+    right_ids = _get_weak_identifiers(right)
+    if left_ids.intersection(right_ids):
+        return 1.0
+    # left_formats = {fmt for fmt, _ in left_ids}
+    # right_formats = {fmt for fmt, _ in right_ids}
+    # if left_formats.intersection(right_formats):
+    #     return -0.5
+    return 0.0

nomenklatura/matching/erun/misc.py

@@ -0,0 +1,71 @@
+from typing import List, Optional, Set
+from followthemoney import registry, E
+
+from nomenklatura.matching.compare.util import extract_numbers
+from nomenklatura.matching.util import type_pair
+from nomenklatura.matching.util import has_schema
+
+from rigour.addresses import normalize_address, shorten_address_keywords
+
+OTHER = registry.gender.OTHER
+
+
+def _norm_address(addr: str, latinize: bool = True) -> Optional[str]:
+    norm_addr = normalize_address(addr, latinize=latinize, min_length=4)
+    if norm_addr is not None:
+        norm_addr = shorten_address_keywords(norm_addr, latinize=latinize)
+    return norm_addr
+
+
+def _norm_place(places: List[str]) -> Set[str]:
+    parts = set()
+    for place in places:
+        norm_place = _norm_address(place)
+        if norm_place is not None:
+            for part in norm_place.split(" "):
+                parts.add(part)
+    return parts
+
+
+def birth_place(query: E, result: E) -> float:
+    """Same place of birth."""
+    if not has_schema(query, result, "Person"):
+        return 0.0
+    lparts = _norm_place(query.get("birthPlace", quiet=True))
+    rparts = _norm_place(result.get("birthPlace", quiet=True))
+    overlap = len(lparts.intersection(rparts))
+    base_length = max(1.0, min(len(lparts), len(rparts)))
+    return overlap / base_length
+
+
+def address_match(query: E, result: E) -> float:
+    """Text similarity between addresses."""
+    lv, rv = type_pair(query, result, registry.address)
+    lvn = _norm_place(lv)
+    rvn = _norm_place(rv)
+    if len(lvn) == 0 or len(rvn) == 0:
+        return 0.0
+    overlap = len(lvn.intersection(rvn))
+    tokens = max(1.0, min(len(lvn), len(rvn)))
+    if overlap == 0:
+        return 0.0
+    return float(overlap) / float(tokens)
+
+
+def address_numbers(query: E, result: E) -> float:
+    """Find if names contain numbers, score if the numbers are different."""
+    lv, rv = type_pair(query, result, registry.address)
+    lvn = extract_numbers(lv)
+    rvn = extract_numbers(rv)
+    common = len(lvn.intersection(rvn))
+    disjoint = len(lvn.difference(rvn))
+    return common - disjoint
+
+
+def gender_mismatch(query: E, result: E) -> float:
+    """Both entities have a different gender associated with them."""
+    qv = {v for v in query.get("gender", quiet=True) if v != OTHER}
+    rv = {v for v in result.get("gender", quiet=True) if v != OTHER}
+    if len(qv) == 1 and len(rv) == 1 and len(qv.intersection(rv)) == 0:
+        return 1.0
+    return 0.0

nomenklatura/matching/erun/model.py

@@ -0,0 +1,110 @@
+import pickle
+import numpy as np
+from typing import List, Dict, Tuple, cast
+from functools import cache
+from sklearn.pipeline import Pipeline  # type: ignore
+from followthemoney import E
+
+from nomenklatura.matching.erun.names import name_levenshtein, family_name_match
+from nomenklatura.matching.erun.names import name_token_overlap, name_numbers
+from nomenklatura.matching.erun.names import obj_name_levenshtein
+from nomenklatura.matching.erun.misc import address_match, address_numbers
+from nomenklatura.matching.erun.misc import birth_place
+from nomenklatura.matching.erun.misc import gender_mismatch
+from nomenklatura.matching.erun.countries import (
+    org_obj_country_match,
+    per_country_mismatch,
+)
+from nomenklatura.matching.erun.identifiers import strong_identifier_match
+from nomenklatura.matching.erun.identifiers import weak_identifier_match
+from nomenklatura.matching.compare.dates import dob_matches, dob_year_matches
+from nomenklatura.matching.compare.dates import dob_year_disjoint
+from nomenklatura.matching.types import (
+    FeatureDocs,
+    FeatureDoc,
+    MatchingResult,
+    ScoringConfig,
+)
+from nomenklatura.matching.types import CompareFunction, FtResult
+from nomenklatura.matching.types import Encoded, ScoringAlgorithm
+from nomenklatura.matching.util import make_github_url
+from nomenklatura.util import DATA_PATH
+
+
+class EntityResolveRegression(ScoringAlgorithm):
+    """Entity resolution matcher. Do not use this in (regulated) screening scenarios."""
+
+    NAME = "er-unstable"
+    MODEL_PATH = DATA_PATH.joinpath(f"{NAME}.pkl")
+    FEATURES: List[CompareFunction] = [
+        name_token_overlap,
+        name_numbers,
+        name_levenshtein,
+        strong_identifier_match,
+        weak_identifier_match,
+        dob_matches,
+        dob_year_matches,
+        FtResult.unwrap(dob_year_disjoint),
+        family_name_match,
+        birth_place,
+        gender_mismatch,
+        per_country_mismatch,
+        org_obj_country_match,
+        obj_name_levenshtein,
+        address_match,
+        address_numbers,
+    ]
+
+    @classmethod
+    def save(cls, pipe: Pipeline, coefficients: Dict[str, float]) -> None:
+        """Store a classification pipeline after training."""
+        mdl = pickle.dumps({"pipe": pipe, "coefficients": coefficients})
+        with open(cls.MODEL_PATH, "wb") as fh:
+            fh.write(mdl)
+        cls.load.cache_clear()
+
+    @classmethod
+    @cache
+    def load(cls) -> Tuple[Pipeline, Dict[str, float]]:
+        """Load a pre-trained classification pipeline for ad-hoc use."""
+        with open(cls.MODEL_PATH, "rb") as fh:
+            matcher = pickle.loads(fh.read())
+        pipe = cast(Pipeline, matcher["pipe"])
+        coefficients = cast(Dict[str, float], matcher["coefficients"])
+        current = [f.__name__ for f in cls.FEATURES]
+        if list(coefficients.keys()) != current:
+            raise RuntimeError("Model was not trained on identical features!")
+        return pipe, coefficients
+
+    @classmethod
+    def get_feature_docs(cls) -> FeatureDocs:
+        """Return an explanation of the features and their coefficients."""
+        features: FeatureDocs = {}
+        _, coefficients = cls.load()
+        for func in cls.FEATURES:
+            name = func.__name__
+            features[name] = FeatureDoc(
+                description=func.__doc__,
+                coefficient=float(coefficients[name]),
+                url=make_github_url(func),
+            )
+        return features
+
+    @classmethod
+    def compare(cls, query: E, result: E, config: ScoringConfig) -> MatchingResult:
+        """Use a regression model to compare two entities."""
+        pipe, _ = cls.load()
+        encoded = cls.encode_pair(query, result)
+        npfeat = np.array([encoded])
+        pred = pipe.predict_proba(npfeat)
+        score = cast(float, pred[0][1])
+        explanations: Dict[str, FtResult] = {}
+        for feature, coeff in zip(cls.FEATURES, encoded):
+            name = feature.__name__
+            explanations[name] = FtResult(score=float(coeff), detail=None)
+        return MatchingResult.make(score=score, explanations=explanations)
+
+    @classmethod
+    def encode_pair(cls, left: E, right: E) -> Encoded:
+        """Encode the comparison between two entities as a set of feature values."""
+        return [f(left, right) for f in cls.FEATURES]

nomenklatura/matching/erun/names.py

@@ -0,0 +1,126 @@
+from functools import lru_cache
+from typing import Set
+from followthemoney import EntityProxy, registry, E
+from followthemoney.names import schema_type_tag
+from rigour.text.distance import levenshtein_similarity
+from rigour.names import Name, NameTypeTag
+from rigour.names import is_stopword
+from rigour.names import remove_org_prefixes, remove_obj_prefixes
+from rigour.names import remove_person_prefixes
+from rigour.names import replace_org_types_compare
+
+from nomenklatura.matching.erun.util import compare_levenshtein
+from nomenklatura.matching.util import max_in_sets, has_schema
+from nomenklatura.util import unroll
+
+
+@lru_cache(maxsize=512)
+def _entity_names(entity: EntityProxy) -> Set[Name]:
+    names: Set[Name] = set()
+    tag = schema_type_tag(entity.schema)
+    for string in entity.get_type_values(registry.name, matchable=True):
+        if tag in (NameTypeTag.ORG, NameTypeTag.ENT):
+            string = replace_org_types_compare(string)
+            string = remove_org_prefixes(string)
+        elif tag == NameTypeTag.PER:
+            string = remove_person_prefixes(string)
+        else:
+            string = remove_obj_prefixes(string)
+        n = Name(string, tag=tag)
+        names.add(n)
+    return names
+
+
+def name_levenshtein(left: E, right: E) -> float:
+    """Consider the edit distance (as a fraction of name length) between the two most
+    similar names linked to both entities."""
+    if not has_schema(left, right, "LegalEntity"):
+        return 0.0
+    if has_schema(left, right, "Person"):
+        left_names: Set[str] = set()
+        for name in _entity_names(left):
+            left_names.add(" ".join(sorted(part.comparable for part in name.parts)))
+            left_names.add(name.comparable)
+        right_names: Set[str] = set()
+        for name in _entity_names(right):
+            right_names.add(" ".join(sorted(part.comparable for part in name.parts)))
+            right_names.add(name.comparable)
+    else:
+        left_names = {n.comparable for n in _entity_names(left)}
+        right_names = {n.comparable for n in _entity_names(right)}
+    return max_in_sets(left_names, right_names, compare_levenshtein)
+
+
+def _entity_lastnames(entity: EntityProxy) -> Set[str]:
+    names: Set[str] = set()
+    for string in entity.get("lastName", quiet=True):
+        n = Name(string, tag=NameTypeTag.PER)
+        for part in n.parts:
+            if len(part.comparable) > 2 and not is_stopword(part.form):
+                names.add(part.comparable)
+    return names
+
+
+def family_name_match(left: E, right: E) -> float:
+    """Matching family name between the two entities."""
+    if not has_schema(left, right, "Person"):
+        return 0.0
+    lnames = _entity_lastnames(left)
+    rnames = _entity_lastnames(right)
+    if len(lnames) == 0 or len(rnames) == 0:
+        return 0.0
+    overlap = lnames.intersection(rnames)
+    return -1.0 if len(overlap) == 0 else 1.0
+
+
+def _name_tokens(entity: EntityProxy) -> Set[str]:
+    tokens: Set[str] = set()
+    for name in _entity_names(entity):
+        for part in name.parts:
+            cmp = part.comparable
+            if len(cmp) > 2 and not is_stopword(part.form):
+                tokens.add(cmp)
+    return tokens
+
+
+def name_token_overlap(left: E, right: E) -> float:
+    """Evaluate the proportion of identical words in each name."""
+    left_tokens = _name_tokens(left)
+    right_tokens = _name_tokens(right)
+    common = left_tokens.intersection(right_tokens)
+    tokens = min(len(left_tokens), len(right_tokens))
+    return float(len(common)) / float(max(2.0, tokens))
+
+
+def name_numbers(left: E, right: E) -> float:
+    """Find if names contain numbers, score if the numbers are different."""
+    left_names = [n.parts for n in _entity_names(left)]
+    right_names = [n.parts for n in _entity_names(right)]
+    left_numbers = {p.comparable for p in unroll(left_names) if p.numeric}
+    right_numbers = {p.comparable for p in unroll(right_names) if p.numeric}
+    total = len(left_numbers) + len(right_numbers)
+    if total == 0:
+        return 0.0
+    common = len(left_numbers.intersection(right_numbers))
+    if common == 0 and len(left_numbers) > 0 and len(right_numbers) > 0:
+        # If both names contain numbers, but they are different, this is a strong
+        # signal that the names are not the same.
+        return -1.0
+    return common / float(total)
+
+
+def _compare_strict_levenshtein(left: str, right: str) -> float:
+    """A stricter version of levenshtein that returns 0.0 if the names are too
+    different in length."""
+    max_edits = min(2, max(len(left), len(right)) // 4)
+    score = levenshtein_similarity(left, right, max_edits=max_edits)
+    return score**2
+
+
+def obj_name_levenshtein(left: E, right: E) -> float:
+    """Very strict name comparison on object (Vessel, RealEstate, Security) names."""
+    if has_schema(left, right, "LegalEntity"):
+        return 0.0
+    left_names = {n.comparable for n in _entity_names(left)}
+    right_names = {n.comparable for n in _entity_names(right)}
+    return max_in_sets(left_names, right_names, _compare_strict_levenshtein)

nomenklatura/matching/erun/train.py

@@ -0,0 +1,135 @@
+import logging
+import multiprocessing
+import random
+from concurrent.futures import ProcessPoolExecutor
+from pprint import pprint
+from typing import Iterable, List, Tuple
+
+import numpy as np
+from followthemoney import registry, EntityProxy
+from followthemoney.util import PathLike
+from numpy.typing import NDArray
+from sklearn import metrics  # type: ignore
+from sklearn.linear_model import LogisticRegression  # type: ignore
+from sklearn.model_selection import train_test_split  # type: ignore
+from sklearn.pipeline import make_pipeline  # type: ignore
+from sklearn.preprocessing import StandardScaler  # type: ignore
+
+from nomenklatura.judgement import Judgement
+from nomenklatura.matching.erun.model import EntityResolveRegression
+from nomenklatura.matching.pairs import JudgedPair, read_pairs
+
+log = logging.getLogger(__name__)
+
+
+def pair_convert(pair: JudgedPair) -> Tuple[List[float], int]:
+    """Encode a pair of training data into features and target."""
+    judgement = 1 if pair.judgement == Judgement.POSITIVE else 0
+    features = EntityResolveRegression.encode_pair(pair.left, pair.right)
+    return features, judgement
+
+
+def pairs_to_arrays(
+    pairs: Iterable[JudgedPair],
+) -> Tuple[NDArray[np.float32], NDArray[np.float32]]:
+    """Parallelize feature computation for training data"""
+    xrows = []
+    yrows = []
+    threads = multiprocessing.cpu_count()
+    log.info("Compute threads: %d", threads)
+    with ProcessPoolExecutor(max_workers=threads) as executor:
+        results = executor.map(pair_convert, pairs, chunksize=1000)
+        for idx, (x, y) in enumerate(results):
+            if idx > 0 and idx % 10000 == 0:
+                log.info("Computing features: %s....", idx)
+            xrows.append(x)
+            yrows.append(y)
+
+    return np.array(xrows), np.array(yrows)
+
+
+def _entity_weight(entity: EntityProxy) -> float:
+    """This weights up entities with more matchable properties, to push down the
+    value of name-only matches."""
+    weight = 0.0
+    # types = set()
+    for prop, _ in entity.itervalues():
+        if prop.matchable:
+            inc_weight = 0.2 if prop.type == registry.name else 1.0
+            weight += inc_weight
+            # types.add(prop.type)
+    # if entity.schema.is_a("LegalEntity") and types == {registry.name}:
+    #     weight = weight * 0.5
+    return weight
+
+
+def weighted_pair_sort(pairs: List[JudgedPair]) -> List[JudgedPair]:
+    for pair in pairs:
+        left_weight = _entity_weight(pair.left)
+        right_weight = _entity_weight(pair.right)
+        # pair.weight = (left_weight + right_weight) / 2.0
+        pair.weight = min(left_weight, right_weight)
+    return sorted(pairs, key=lambda p: -p.weight)
+
+
+def build_dataset(
+    pairs_file: PathLike,
+) -> Tuple[NDArray[np.float32], NDArray[np.float32]]:
+    """Load and balance a dataset from a JSON file."""
+    pairs = []
+    for pair in read_pairs(pairs_file):
+        if not pair.left.schema.matchable or not pair.right.schema.matchable:
+            continue
+        if pair.left.schema.is_a("Address") or pair.right.schema.is_a("Address"):
+            continue
+        if pair.judgement == Judgement.UNSURE:
+            pair.judgement = Judgement.NEGATIVE
+        pairs.append(pair)
+    positive = [p for p in pairs if p.judgement == Judgement.POSITIVE]
+    negative = [p for p in pairs if p.judgement == Judgement.NEGATIVE]
+    log.info(
+        "Total pairs loaded: %d (%d pos/%d neg)",
+        len(pairs),
+        len(positive),
+        len(negative),
+    )
+    min_class = min(len(positive), len(negative))
+    log.info("Downsampling to %d per class", min_class)
+    if len(positive) > min_class:
+        positive = weighted_pair_sort(positive)
+        pairs = positive[:min_class] + negative
+    else:
+        negative = weighted_pair_sort(negative)
+        pairs = positive + negative[:min_class]
+    random.shuffle(pairs)
+    log.info("Training pairs after downsampling: %d", len(pairs))
+    return pairs_to_arrays(pairs)
+
+
+def train_matcher(pairs_file: PathLike) -> None:
+    X, y = build_dataset(pairs_file)
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30)
+    # logreg = LogisticRegression(class_weight={0: 95, 1: 1})
+    # logreg = LogisticRegression(penalty="l1", solver="liblinear")
+    logreg = LogisticRegression(penalty="l2")
+    log.info("Training model...")
+    pipe = make_pipeline(StandardScaler(), logreg)
+    pipe.fit(X_train, y_train)
+    coef = logreg.coef_[0]
+    coefficients = {
+        n.__name__: c for n, c in zip(EntityResolveRegression.FEATURES, coef)
+    }
+    EntityResolveRegression.save(pipe, coefficients)
+    print("Written to: %s" % EntityResolveRegression.MODEL_PATH.as_posix())
+    print("Coefficients:")
+    pprint(coefficients)
+    y_pred = pipe.predict(X_test)
+    cnf_matrix = metrics.confusion_matrix(y_test, y_pred)
+    print("Confusion matrix:\n", cnf_matrix)
+    print("Accuracy:", metrics.accuracy_score(y_test, y_pred))
+    print("Precision:", metrics.precision_score(y_test, y_pred))
+    print("Recall:", metrics.recall_score(y_test, y_pred))
+
+    y_pred_proba = pipe.predict_proba(X_test)[::, 1]
+    auc = metrics.roc_auc_score(y_test, y_pred_proba)
+    print("Area under curve:", auc)

nomenklatura/matching/erun/util.py

@@ -0,0 +1,28 @@
+from normality import ascii_text
+from typing import Iterable, Set, Tuple
+from rigour.text.distance import levenshtein
+from rigour.names import tokenize_name
+
+
+def tokenize(texts: Iterable[str]) -> Set[str]:
+    tokens: Set[str] = set()
+    for text in texts:
+        text = text.casefold()
+        for token in tokenize_name(text):
+            ascii_token = ascii_text(token)
+            if ascii_token is not None and len(ascii_token) > 2:
+                tokens.add(ascii_token)
+    return tokens
+
+
+def tokenize_pair(
+    pair: Tuple[Iterable[str], Iterable[str]],
+) -> Tuple[Set[str], Set[str]]:
+    return tokenize(pair[0]), tokenize(pair[1])
+
+
+def compare_levenshtein(left: str, right: str) -> float:
+    distance = levenshtein(left, right)
+    base = max((1, len(left), len(right)))
+    return 1.0 - (distance / float(base))
+    # return math.sqrt(distance)