nomenklatura-mpt 4.1.9__py3-none-any.whl

nomenklatura/matching/name_based/misc.py

@@ -0,0 +1,86 @@
+from itertools import product
+from typing import Iterable, Set
+from prefixdate import Precision
+from followthemoney.proxy import E
+from followthemoney.types import registry
+from rigour.text.distance import levenshtein
+from rigour.ids import StrictFormat
+
+from nomenklatura.matching.compare.util import clean_map
+from nomenklatura.matching.types import FtResult, ScoringConfig
+from nomenklatura.matching.util import has_schema, props_pair, type_pair
+
+
+def _dates_precision(values: Iterable[str], precision: Precision) -> Set[str]:
+    dates = set()
+    for value in values:
+        if len(value) >= precision.value:
+            dates.add(value[: precision.value])
+    return dates
+
+
+def _flip_day_month(value: str) -> str:
+    # This is such a common mistake we want to accomodate it.
+    year, month, day = value.split("-", 2)
+    return f"{year}-{day}-{month}"
+
+
+def dob_day_disjoint(query: E, result: E, config: ScoringConfig) -> FtResult:
+    """The birth date of the two entities is not the same."""
+    query_dates, result_dates = props_pair(query, result, ["birthDate"])
+    if len(query_dates) == 0 or len(result_dates) == 0:
+        return FtResult(score=0.0, detail="No birth dates provided")
+    result_days = _dates_precision(result_dates, Precision.DAY)
+    query_days = _dates_precision(query_dates, Precision.DAY)
+    if len(result_days) == 0 or len(query_days) == 0:
+        return FtResult(score=0.0, detail="Birth days don't include day precision")
+    overlap = query_days.intersection(result_days)
+    if len(overlap) > 0:
+        return FtResult(score=0.0, detail=f"Birth day match: {', '.join(overlap)}")
+    query_flipped = set([_flip_day_month(d) for d in query_days])
+    overlap = query_flipped.intersection(result_days)
+    if len(overlap) > 0:
+        detail = f"Birth day flipped match: {', '.join(overlap)}"
+        return FtResult(score=0.5, detail=detail)
+    return FtResult(score=1.0, detail="Birth day mis-match")
+
+
+def dob_year_disjoint(query: E, result: E, config: ScoringConfig) -> FtResult:
+    """The birth date of the two entities is not the same."""
+    query_dates, result_dates = props_pair(query, result, ["birthDate"])
+    query_years = _dates_precision(query_dates, Precision.YEAR)
+    result_years = _dates_precision(result_dates, Precision.YEAR)
+    if len(query_years) == 0 or len(result_years) == 0:
+        return FtResult(score=0.0, detail="No birth years provided")
+    overlap = query_years.intersection(result_years)
+    if len(overlap) > 0:
+        detail = f"Birth year match: {', '.join(overlap)}"
+        return FtResult(score=0.0, detail=detail)
+    return FtResult(score=1.0, detail="Birth year mis-match")
+
+
+def orgid_disjoint(query: E, result: E, config: ScoringConfig) -> FtResult:
+    """Two companies or organizations have different tax identifiers or registration
+    numbers."""
+    if not has_schema(query, result, "Organization"):
+        return FtResult(score=0.0, detail="Neither entity is an organization")
+    query_ids_, result_ids_ = type_pair(query, result, registry.identifier)
+    query_ids = clean_map(query_ids_, StrictFormat.normalize)
+    result_ids = clean_map(result_ids_, StrictFormat.normalize)
+    if not len(query_ids) or not len(result_ids):
+        return FtResult(score=0.0, detail="Neither entity has identifiers")
+    common = query_ids.intersection(result_ids)
+    if len(common) > 0:
+        return FtResult(score=0.0, detail="Common identifiers: %s" % ", ".join(common))
+    max_ratio = 0.0
+    for query_id, result_id in product(query_ids, result_ids):
+        distance = levenshtein(query_id, result_id)
+        max_len = max(len(query_id), len(result_id))
+        ratio = 1.0 - (distance / float(max_len))
+        if ratio > 0.7:
+            max_ratio = max(max_ratio, ratio)
+    detail = "Mismatched identifiers: %s vs %s" % (
+        ", ".join(query_ids),
+        ", ".join(result_ids),
+    )
+    return FtResult(score=1 - max_ratio, detail=detail)

nomenklatura/matching/name_based/model.py

@@ -0,0 +1,59 @@
+from typing import Dict
+
+from nomenklatura.matching.types import Feature, HeuristicAlgorithm
+from nomenklatura.matching.compare.countries import country_mismatch
+from nomenklatura.matching.compare.gender import gender_mismatch
+from nomenklatura.matching.name_based.misc import orgid_disjoint
+from nomenklatura.matching.name_based.misc import dob_day_disjoint, dob_year_disjoint
+from nomenklatura.matching.name_based.names import jaro_name_parts
+from nomenklatura.matching.name_based.names import soundex_name_parts
+
+
+class NameMatcher(HeuristicAlgorithm):
+    """An algorithm that matches on entity name, using phonetic comparisons and edit
+    distance to generate potential matches. This implementation is vaguely based on
+    the behaviour proposed by the US OFAC documentation (FAQ #249)."""
+
+    # Try to re-produce results from: https://sanctionssearch.ofac.treas.gov/
+    # cf. https://ofac.treasury.gov/faqs/topic/1636
+
+    NAME = "name-based"
+    features = [
+        Feature(func=jaro_name_parts, weight=0.5),
+        Feature(func=soundex_name_parts, weight=0.5),
+    ]
+
+    @classmethod
+    def compute_score(
+        cls, scores: Dict[str, float], weights: Dict[str, float]
+    ) -> float:
+        score = 0.0
+        for feat in cls.features:
+            score += scores.get(feat.name, 0.0) * weights.get(feat.name, 0.0)
+        return score
+
+
+class NameQualifiedMatcher(HeuristicAlgorithm):
+    """Same as the name-based algorithm, but scores will be reduced if a mis-match
+    of birth dates and nationalities is found for persons, or different
+    tax/registration identifiers are included for organizations and companies."""
+
+    NAME = "name-qualified"
+    features = [
+        Feature(func=jaro_name_parts, weight=0.5),
+        Feature(func=soundex_name_parts, weight=0.5),
+        Feature(func=country_mismatch, weight=-0.1, qualifier=True),
+        Feature(func=dob_year_disjoint, weight=-0.1, qualifier=True),
+        Feature(func=dob_day_disjoint, weight=-0.15, qualifier=True),
+        Feature(func=gender_mismatch, weight=-0.1, qualifier=True),
+        Feature(func=orgid_disjoint, weight=-0.1, qualifier=True),
+    ]
+
+    @classmethod
+    def compute_score(
+        cls, scores: Dict[str, float], weights: Dict[str, float]
+    ) -> float:
+        score = 0.0
+        for feat in cls.features:
+            score += scores.get(feat.name, 0.0) * weights.get(feat.name, 0.0)
+        return score

nomenklatura/matching/name_based/names.py

@@ -0,0 +1,59 @@
+from typing import List, Optional, Tuple
+from followthemoney.proxy import E
+from followthemoney.types import registry
+from rigour.text.distance import jaro_winkler
+from rigour.text.phonetics import soundex
+
+from nomenklatura.matching.types import FtResult, ScoringConfig
+from nomenklatura.matching.util import type_pair
+from nomenklatura.matching.compat import names_word_list
+
+
+def _soundex_token(token: str) -> str:
+    if token.isalpha() and len(token) > 1:
+        out = soundex(token)
+        # doesn't handle non-ascii characters
+        if len(out):
+            return out
+    return token.upper()
+
+
+def soundex_name_parts(query: E, result: E, config: ScoringConfig) -> FtResult:
+    """Compare two sets of name parts using the phonetic matching."""
+    query_names_, result_names_ = type_pair(query, result, registry.name)
+    query_soundex = set([_soundex_token(p) for p in names_word_list(query_names_)])
+    result_soundex = set([_soundex_token(p) for p in names_word_list(result_names_)])
+    overlap = query_soundex.intersection(result_soundex)
+    if len(overlap) == 0:
+        return FtResult(score=0.0, detail=None)
+    min_len = min(len(query_soundex), len(result_soundex))
+    score = len(overlap) / float(max(1.0, min_len))
+    detail = f"Matched {len(overlap)} tokens: {', '.join(overlap)}"
+    return FtResult(score=score, detail=detail)
+
+
+def jaro_name_parts(query: E, result: E, config: ScoringConfig) -> FtResult:
+    """Compare two sets of name parts using the Jaro-Winkler string similarity
+    algorithm."""
+    query_names_, result_names_ = type_pair(query, result, registry.name)
+    result_parts = set(names_word_list(result_names_))
+    similiarities: List[float] = []
+    tokens: List[Tuple[str, str]] = []
+    for part in set(names_word_list(query_names_)):
+        best = 0.0
+        best_token: Optional[str] = None
+
+        for other in result_parts:
+            part_similarity = jaro_winkler(part, other)
+            if part_similarity > 0.5 and part_similarity > best:
+                best = part_similarity
+                best_token = other
+
+        similiarities.append(best)
+        if best_token is not None:
+            tokens.append((part, best_token))
+    if len(similiarities) == 0:
+        return FtResult(score=0.0, detail=None)
+    score = sum(similiarities) / float(max(1.0, len(similiarities)))
+    mapping = ", ".join(f"{a} -> {b}" for a, b in tokens)
+    return FtResult(score=score, detail=f"Matched {len(tokens)} tokens: {mapping}")

nomenklatura/matching/pairs.py

@@ -0,0 +1,42 @@
+import json
+from typing import Generator, Dict, Any
+from followthemoney.proxy import EntityProxy
+from followthemoney.util import PathLike
+
+from nomenklatura.judgement import Judgement
+
+
+class JudgedPair(object):
+    """A pair of two entities which have been judged to be the same
+    (or not) by a user."""
+
+    __slots__ = ("left", "right", "weight", "judgement")
+
+    def __init__(
+        self, left: EntityProxy, right: EntityProxy, judgement: Judgement
+    ) -> None:
+        self.left = left
+        self.right = right
+        self.judgement = judgement
+        self.weight = 0.0
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "left": self.left.to_dict(),
+            "right": self.right.to_dict(),
+            "judgement": self.judgement.value,
+            "weight": self.weight,
+        }
+
+
+def read_pairs(pairs_file: PathLike) -> Generator[JudgedPair, None, None]:
+    """Read judgement pairs (training data) from a JSON file."""
+    with open(pairs_file, "r") as fh:
+        while line := fh.readline():
+            data = json.loads(line)
+            left_entity = EntityProxy.from_dict(data["left"])
+            right_entity = EntityProxy.from_dict(data["right"])
+            judgement = Judgement(data["judgement"])
+            if judgement not in (Judgement.POSITIVE, Judgement.NEGATIVE):
+                continue
+            yield JudgedPair(left_entity, right_entity, judgement)

nomenklatura/matching/regression_v1/misc.py

@@ -0,0 +1,75 @@
+from followthemoney.proxy import E
+from followthemoney.types import registry
+
+from nomenklatura.matching.regression_v1.util import tokenize_pair, compare_levenshtein
+from nomenklatura.matching.compare.util import has_overlap, extract_numbers, is_disjoint
+from nomenklatura.matching.util import props_pair, type_pair
+from nomenklatura.matching.util import max_in_sets, has_schema
+from nomenklatura.matching.compat import clean_name_ascii
+
+
+def birth_place(query: E, result: E) -> float:
+    """Same place of birth."""
+    lv, rv = tokenize_pair(props_pair(query, result, ["birthPlace"]))
+    tokens = min(len(lv), len(rv))
+    return float(len(lv.intersection(rv))) / float(max(2.0, tokens))
+
+
+def address_match(query: E, result: E) -> float:
+    """Text similarity between addresses."""
+    lv, rv = type_pair(query, result, registry.address)
+    lvn = [clean_name_ascii(v) for v in lv]
+    rvn = [clean_name_ascii(v) for v in rv]
+    return max_in_sets(lvn, rvn, compare_levenshtein)
+
+
+def address_numbers(query: E, result: E) -> float:
+    """Find if names contain numbers, score if the numbers are different."""
+    lv, rv = type_pair(query, result, registry.address)
+    lvn = extract_numbers(lv)
+    rvn = extract_numbers(rv)
+    common = len(lvn.intersection(rvn))
+    disjoint = len(lvn.difference(rvn))
+    return common - disjoint
+
+
+def phone_match(query: E, result: E) -> float:
+    """Matching phone numbers between the two entities."""
+    lv, rv = type_pair(query, result, registry.phone)
+    return 1.0 if has_overlap(lv, rv) else 0.0
+
+
+def email_match(query: E, result: E) -> float:
+    """Matching email addresses between the two entities."""
+    lv, rv = type_pair(query, result, registry.email)
+    return 1.0 if has_overlap(lv, rv) else 0.0
+
+
+def identifier_match(query: E, result: E) -> float:
+    """Matching identifiers (e.g. passports, national ID cards, registration or
+    tax numbers) between the two entities."""
+    if has_schema(query, result, "Organization"):
+        return 0.0
+    lv, rv = type_pair(query, result, registry.identifier)
+    return 1.0 if has_overlap(lv, rv) else 0.0
+
+
+def org_identifier_match(query: E, result: E) -> float:
+    """Matching identifiers (e.g. registration or tax numbers) between two
+    organizations or companies."""
+    if not has_schema(query, result, "Organization"):
+        return 0.0
+    lv, rv = type_pair(query, result, registry.identifier)
+    return 1.0 if has_overlap(lv, rv) else 0.0
+
+
+def gender_mismatch(query: E, result: E) -> float:
+    """Both entities have a different gender associated with them."""
+    qv, rv = props_pair(query, result, ["gender"])
+    return 1.0 if is_disjoint(qv, rv) else 0.0
+
+
+def country_mismatch(query: E, result: E) -> float:
+    """Both entities are linked to different countries."""
+    qv, rv = type_pair(query, result, registry.country)
+    return 1.0 if is_disjoint(qv, rv) else 0.0

nomenklatura/matching/regression_v1/model.py

@@ -0,0 +1,110 @@
+import pickle
+import numpy as np
+from typing import List, Dict, Tuple, cast
+from functools import cache
+from sklearn.pipeline import Pipeline  # type: ignore
+from followthemoney.proxy import E
+
+from nomenklatura.matching.regression_v1.names import first_name_match
+from nomenklatura.matching.regression_v1.names import family_name_match
+from nomenklatura.matching.regression_v1.names import name_levenshtein, name_match
+from nomenklatura.matching.regression_v1.names import name_token_overlap, name_numbers
+from nomenklatura.matching.regression_v1.misc import phone_match, email_match
+from nomenklatura.matching.regression_v1.misc import address_match, address_numbers
+from nomenklatura.matching.regression_v1.misc import identifier_match, birth_place
+from nomenklatura.matching.regression_v1.misc import org_identifier_match
+from nomenklatura.matching.regression_v1.misc import gender_mismatch
+from nomenklatura.matching.regression_v1.misc import country_mismatch
+from nomenklatura.matching.compare.dates import dob_matches, dob_year_matches
+from nomenklatura.matching.compare.dates import dob_year_disjoint
+from nomenklatura.matching.types import (
+    FeatureDocs,
+    FeatureDoc,
+    MatchingResult,
+    ScoringConfig,
+)
+from nomenklatura.matching.types import CompareFunction, FtResult
+from nomenklatura.matching.types import Encoded, ScoringAlgorithm
+from nomenklatura.matching.util import make_github_url
+from nomenklatura.util import DATA_PATH
+
+
+class RegressionV1(ScoringAlgorithm):
+    """A simple matching algorithm based on a regression model."""
+
+    NAME = "regression-v1"
+    MODEL_PATH = DATA_PATH.joinpath(f"{NAME}.pkl")
+    FEATURES: List[CompareFunction] = [
+        name_match,
+        name_token_overlap,
+        name_numbers,
+        name_levenshtein,
+        phone_match,
+        email_match,
+        identifier_match,
+        dob_matches,
+        dob_year_matches,
+        FtResult.unwrap(dob_year_disjoint),
+        first_name_match,
+        family_name_match,
+        birth_place,
+        gender_mismatch,
+        country_mismatch,
+        org_identifier_match,
+        address_match,
+        address_numbers,
+    ]
+
+    @classmethod
+    def save(cls, pipe: Pipeline, coefficients: Dict[str, float]) -> None:
+        """Store a classification pipeline after training."""
+        mdl = pickle.dumps({"pipe": pipe, "coefficients": coefficients})
+        with open(cls.MODEL_PATH, "wb") as fh:
+            fh.write(mdl)
+        cls.load.cache_clear()
+
+    @classmethod
+    @cache
+    def load(cls) -> Tuple[Pipeline, Dict[str, float]]:
+        """Load a pre-trained classification pipeline for ad-hoc use."""
+        with open(cls.MODEL_PATH, "rb") as fh:
+            matcher = pickle.loads(fh.read())
+        pipe = cast(Pipeline, matcher["pipe"])
+        coefficients = cast(Dict[str, float], matcher["coefficients"])
+        current = [f.__name__ for f in cls.FEATURES]
+        if list(coefficients.keys()) != current:
+            raise RuntimeError("Model was not trained on identical features!")
+        return pipe, coefficients
+
+    @classmethod
+    def get_feature_docs(cls) -> FeatureDocs:
+        """Return an explanation of the features and their coefficients."""
+        features: FeatureDocs = {}
+        _, coefficients = cls.load()
+        for func in cls.FEATURES:
+            name = func.__name__
+            features[name] = FeatureDoc(
+                description=func.__doc__,
+                coefficient=float(coefficients[name]),
+                url=make_github_url(func),
+            )
+        return features
+
+    @classmethod
+    def compare(cls, query: E, result: E, config: ScoringConfig) -> MatchingResult:
+        """Use a regression model to compare two entities."""
+        pipe, _ = cls.load()
+        encoded = cls.encode_pair(query, result)
+        npfeat = np.array([encoded])
+        pred = pipe.predict_proba(npfeat)
+        score = cast(float, pred[0][1])
+        explanations: Dict[str, FtResult] = {}
+        for feature, coeff in zip(cls.FEATURES, encoded):
+            name = feature.__name__
+            explanations[name] = FtResult(score=float(coeff), detail=None)
+        return MatchingResult.make(score=score, explanations=explanations)
+
+    @classmethod
+    def encode_pair(cls, left: E, right: E) -> Encoded:
+        """Encode the comparison between two entities as a set of feature values."""
+        return [f(left, right) for f in cls.FEATURES]

nomenklatura/matching/regression_v1/names.py

@@ -0,0 +1,63 @@
+from typing import Iterable, Set
+from followthemoney.proxy import E
+from followthemoney.types import registry
+
+from nomenklatura.matching.regression_v1.util import tokenize_pair, compare_levenshtein
+from nomenklatura.matching.compare.util import is_disjoint, has_overlap, extract_numbers
+from nomenklatura.matching.util import props_pair, type_pair
+from nomenklatura.matching.util import max_in_sets
+from nomenklatura.matching.compat import fingerprint_name
+
+
+def normalize_names(raws: Iterable[str]) -> Set[str]:
+    names = set()
+    for raw in raws:
+        name = fingerprint_name(raw)
+        if name is not None:
+            names.add(name[:128])
+    return names
+
+
+def name_levenshtein(left: E, right: E) -> float:
+    """Consider the edit distance (as a fraction of name length) between the two most
+    similar names linked to both entities."""
+    lv, rv = type_pair(left, right, registry.name)
+    lvn, rvn = normalize_names(lv), normalize_names(rv)
+    return max_in_sets(lvn, rvn, compare_levenshtein)
+
+
+def first_name_match(left: E, right: E) -> float:
+    """Matching first/given name between the two entities."""
+    lv, rv = tokenize_pair(props_pair(left, right, ["firstName"]))
+    return 1.0 if has_overlap(lv, rv) else 0.0
+
+
+def family_name_match(left: E, right: E) -> float:
+    """Matching family name between the two entities."""
+    lv, rv = tokenize_pair(props_pair(left, right, ["lastName"]))
+    return 1.0 if has_overlap(lv, rv) else 0.0
+
+
+def name_match(left: E, right: E) -> float:
+    """Check for exact name matches between the two entities."""
+    lv, rv = type_pair(left, right, registry.name)
+    lvn, rvn = normalize_names(lv), normalize_names(rv)
+    common = [len(n) for n in lvn.intersection(rvn)]
+    max_common = max(common, default=0)
+    if max_common == 0:
+        return 0.0
+    return float(max_common)
+
+
+def name_token_overlap(left: E, right: E) -> float:
+    """Evaluate the proportion of identical words in each name."""
+    lv, rv = tokenize_pair(type_pair(left, right, registry.name))
+    common = lv.intersection(rv)
+    tokens = min(len(lv), len(rv))
+    return float(len(common)) / float(max(2.0, tokens))
+
+
+def name_numbers(left: E, right: E) -> float:
+    """Find if names contain numbers, score if the numbers are different."""
+    lv, rv = type_pair(left, right, registry.name)
+    return 1.0 if is_disjoint(extract_numbers(lv), extract_numbers(rv)) else 0.0

nomenklatura/matching/regression_v1/train.py

@@ -0,0 +1,87 @@
+import logging
+import numpy as np
+import multiprocessing
+from typing import Iterable, List, Tuple
+from pprint import pprint
+from numpy.typing import NDArray
+from sklearn.pipeline import make_pipeline  # type: ignore
+from sklearn.preprocessing import StandardScaler  # type: ignore
+from sklearn.model_selection import train_test_split  # type: ignore
+from sklearn.linear_model import LogisticRegression  # type: ignore
+from sklearn import metrics  # type: ignore
+from concurrent.futures import ThreadPoolExecutor
+from followthemoney.util import PathLike
+
+from nomenklatura.judgement import Judgement
+from nomenklatura.matching.pairs import read_pairs, JudgedPair
+from nomenklatura.matching.regression_v1.model import RegressionV1
+
+log = logging.getLogger(__name__)
+
+
+def pair_convert(pair: JudgedPair) -> Tuple[List[float], int]:
+    """Encode a pair of training data into features and target."""
+    judgement = 1 if pair.judgement == Judgement.POSITIVE else 0
+    features = RegressionV1.encode_pair(pair.left, pair.right)
+    return features, judgement
+
+
+def pairs_to_arrays(
+    pairs: Iterable[JudgedPair],
+) -> Tuple[NDArray[np.float32], NDArray[np.float32]]:
+    """Parallelize feature computation for training data"""
+    xrows = []
+    yrows = []
+    threads = multiprocessing.cpu_count()
+    log.info("Compute threads: %d", threads)
+    with ThreadPoolExecutor(max_workers=threads) as excecutor:
+        results = excecutor.map(pair_convert, pairs)
+        for idx, (x, y) in enumerate(results):
+            if idx > 0 and idx % 10000 == 0:
+                log.info("Computing features: %s....", idx)
+            xrows.append(x)
+            yrows.append(y)
+
+    return np.array(xrows), np.array(yrows)
+
+
+def train_matcher(pairs_file: PathLike) -> None:
+    pairs = []
+    for pair in read_pairs(pairs_file):
+        # HACK: support more eventually:
+        # if not pair.left.schema.is_a("LegalEntity"):
+        #     continue
+        if pair.judgement == Judgement.UNSURE:
+            pair.judgement = Judgement.NEGATIVE
+        # randomize_entity(pair.left)
+        # randomize_entity(pair.right)
+        pairs.append(pair)
+    # random.shuffle(pairs)
+    # pairs = pairs[:30000]
+    positive = len([p for p in pairs if p.judgement == Judgement.POSITIVE])
+    negative = len([p for p in pairs if p.judgement == Judgement.NEGATIVE])
+    log.info("Total pairs loaded: %d (%d pos/%d neg)", len(pairs), positive, negative)
+    X, y = pairs_to_arrays(pairs)
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30)
+    # logreg = LogisticRegression(class_weight={0: 95, 1: 1})
+    # logreg = LogisticRegression(penalty="l1", solver="liblinear")
+    logreg = LogisticRegression(penalty="l2")
+    log.info("Training model...")
+    pipe = make_pipeline(StandardScaler(), logreg)
+    pipe.fit(X_train, y_train)
+    coef = logreg.coef_[0]
+    coefficients = {n.__name__: c for n, c in zip(RegressionV1.FEATURES, coef)}
+    RegressionV1.save(pipe, coefficients)
+    print("Written to: %s" % RegressionV1.MODEL_PATH.as_posix())
+    print("Coefficients:")
+    pprint(coefficients)
+    y_pred = pipe.predict(X_test)
+    cnf_matrix = metrics.confusion_matrix(y_test, y_pred)
+    print("Confusion matrix:\n", cnf_matrix)
+    print("Accuracy:", metrics.accuracy_score(y_test, y_pred))
+    print("Precision:", metrics.precision_score(y_test, y_pred))
+    print("Recall:", metrics.recall_score(y_test, y_pred))
+
+    y_pred_proba = pipe.predict_proba(X_test)[::, 1]
+    auc = metrics.roc_auc_score(y_test, y_pred_proba)
+    print("Area under curve:", auc)

nomenklatura/matching/regression_v1/util.py

@@ -0,0 +1,31 @@
+from normality.constants import WS
+from typing import Iterable, Set, Tuple
+from rigour.text.distance import levenshtein
+
+from nomenklatura.matching.compat import clean_name_ascii
+
+
+def tokenize(texts: Iterable[str]) -> Set[str]:
+    tokens: Set[str] = set()
+    for text in texts:
+        cleaned = clean_name_ascii(text)
+        if cleaned is None:
+            continue
+        for token in cleaned.split(WS):
+            token = token.strip()
+            if len(token) > 2:
+                tokens.add(token)
+    return tokens
+
+
+def tokenize_pair(
+    pair: Tuple[Iterable[str], Iterable[str]],
+) -> Tuple[Set[str], Set[str]]:
+    return tokenize(pair[0]), tokenize(pair[1])
+
+
+def compare_levenshtein(left: str, right: str) -> float:
+    distance = levenshtein(left, right)
+    base = max((1, len(left), len(right)))
+    return 1.0 - (distance / float(base))
+    # return math.sqrt(distance)

nomenklatura/matching/svm_v1/__init__.py

@@ -0,0 +1,5 @@
+"""Support Vector Machine matching algorithm for nomenklatura."""
+
+from .model import SVMV1
+
+__all__ = ["SVMV1"]