nomenklatura-mpt 4.1.9__py3-none-any.whl

nomenklatura/index/tokenizer.py

@@ -0,0 +1,92 @@
+from typing import Generic, Generator, Optional, Tuple, Set
+from normality import WS, category_replace, ascii_text
+from rigour.ids import StrictFormat
+from rigour.names import tokenize_name
+from rigour.names import remove_person_prefixes
+from rigour.names.org_types import replace_org_types_display
+from followthemoney import registry, Property, DS, SE
+
+
+WORD_FIELD = "word"
+NAME_PART_FIELD = "namepart"
+SKIP_FULL = (
+    # registry.name,
+    registry.address,
+    registry.text,
+    registry.string,
+    registry.number,
+    registry.json,
+)
+TEXT_TYPES = (
+    registry.text,
+    registry.string,
+    registry.address,
+    registry.identifier,
+    registry.name,
+)
+
+
+def normalize_name(name: Optional[str]) -> Optional[str]:
+    """Normalize a name by removing prefixes and suffixes."""
+    if name is None:
+        return None
+    name = name.lower()
+    name = " ".join(tokenize_name(name))
+    name = ascii_text(name)
+    if len(name) < 2:
+        return None
+    return name
+
+
+class Tokenizer(Generic[DS, SE]):
+    def value(
+        self, prop: Property, value: str
+    ) -> Generator[Tuple[str, str], None, None]:
+        """Perform type-specific token generation for a property value."""
+        type = prop.type
+        if not prop.matchable:
+            return
+        if type in (registry.url, registry.topic, registry.entity):
+            return
+        if type not in SKIP_FULL:
+            token_value = value[:100].lower()
+            yield type.name, token_value
+        if type == registry.date:
+            if len(value) > 4:
+                yield type.name, value[:4]
+            yield type.name, value[:10]
+            return
+        if type == registry.name:
+            name_parts: Set[str] = set()
+            # this needs to happen before the replacements
+            text = remove_person_prefixes(value)
+            for token in tokenize_name(text.lower(), token_min_length=3):
+                name_parts.add(token)
+            # Super hard-core string scrubbing
+            cleaned = normalize_name(text)
+            if cleaned is not None:
+                cleaned = replace_org_types_display(cleaned, normalizer=normalize_name)
+                yield type.name, cleaned
+                for token in cleaned.split(WS):
+                    name_parts.add(token)
+            for part in name_parts:
+                if len(part) > 2 and len(part) < 30:
+                    yield NAME_PART_FIELD, part
+            return
+        if type == registry.identifier:
+            clean_id = StrictFormat.normalize(value)
+            if clean_id is not None:
+                yield type.name, clean_id
+            return
+        if type in TEXT_TYPES:
+            text = value.lower()
+            replaced = category_replace(text)
+            for word in replaced.split(WS):
+                if len(word) >= 3:
+                    yield WORD_FIELD, word
+
+    def entity(self, entity: SE) -> Generator[Tuple[str, str], None, None]:
+        # yield f"d:{entity.dataset.name}", 0.0
+        for prop, value in entity.itervalues():
+            for field, token in self.value(prop, value):
+                yield field, token

nomenklatura/judgement.py

@@ -0,0 +1,21 @@
+from enum import Enum
+
+
+class Judgement(Enum):
+    """A judgement of whether two entities are the same."""
+
+    POSITIVE = "positive"
+    NEGATIVE = "negative"
+    UNSURE = "unsure"
+    NO_JUDGEMENT = "no_judgement"
+
+    def __add__(self, other: "Judgement") -> "Judgement":
+        pair = {self, other}
+        if pair == {Judgement.POSITIVE}:
+            return Judgement.POSITIVE
+        elif pair == {Judgement.POSITIVE, Judgement.NEGATIVE}:
+            return Judgement.NEGATIVE
+        return Judgement.UNSURE
+
+    def to_dict(self) -> str:
+        return str(self.value)

nomenklatura/kv.py

@@ -0,0 +1,40 @@
+import redis
+import logging
+from typing import Union
+from functools import cache
+from redis.client import Redis
+from fakeredis import FakeStrictRedis
+
+from rigour.env import ENCODING
+from nomenklatura import settings
+
+log = logging.getLogger(__name__)
+
+
+@cache
+def get_redis() -> "Redis[bytes]":
+    """Return a Redis connection configured from the environment."""
+    if settings.TESTING or not len(settings.REDIS_URL.strip()):
+        log.info("Using in-memory key-value store...")
+        return FakeStrictRedis(decode_responses=False)
+    db = redis.from_url(settings.REDIS_URL, decode_responses=False)
+    # for kvrocks:
+    if len(db.config_get("redis-cursor-compatible")):
+        db.config_set("redis-cursor-compatible", "yes")
+    return db
+
+
+def close_redis() -> None:
+    """Close the Redis connection."""
+    get_redis().close()
+    get_redis.cache_clear()
+
+
+def b(s: str) -> bytes:
+    """Encode a string to bytes."""
+    return s.encode(ENCODING)
+
+
+def bv(s: Union[bytes, str, int, float]) -> bytes:
+    """Decode bytes to a string."""
+    return s  # type: ignore

nomenklatura/matching/__init__.py

@@ -0,0 +1,47 @@
+from typing import List, Type, Optional
+from nomenklatura.matching.regression_v1.model import RegressionV1
+from nomenklatura.matching.svm_v1.model import SVMV1
+from nomenklatura.matching.regression_v1.train import train_matcher as train_v1_matcher
+from nomenklatura.matching.svm_v1.train import train_matcher as train_svm_matcher
+from nomenklatura.matching.name_based import NameMatcher, NameQualifiedMatcher
+from nomenklatura.matching.erun.model import EntityResolveRegression
+from nomenklatura.matching.erun.train import train_matcher as train_erun_matcher
+from nomenklatura.matching.logic_v1.model import LogicV1
+from nomenklatura.matching.logic_v2.model import LogicV2
+from nomenklatura.matching.types import ScoringAlgorithm, ScoringConfig
+
+ALGORITHMS: List[Type[ScoringAlgorithm]] = [
+    LogicV1,
+    LogicV2,
+    NameMatcher,
+    NameQualifiedMatcher,
+    RegressionV1,
+    EntityResolveRegression,
+    SVMV1
+]
+
+DefaultAlgorithm = RegressionV1
+
+
+def get_algorithm(name: str) -> Optional[Type[ScoringAlgorithm]]:
+    """Return the scoring algorithm class with the given name."""
+    for algorithm in ALGORITHMS:
+        if algorithm.NAME == name:
+            return algorithm
+    return None
+
+
+__all__ = [
+    "RegressionV1",
+    "EntityResolveRegression",
+    "train_v1_matcher",
+    "train_svm_matcher",
+    "train_erun_matcher",
+    "DefaultAlgorithm",
+    "ScoringAlgorithm",
+    "NameMatcher",
+    "NameQualifiedMatcher",
+    "ScoringConfig",
+    "LogicV1",
+    "LogicV2",
+]

nomenklatura/matching/bench.py

@@ -0,0 +1,32 @@
+import logging
+import datetime
+from timeit import timeit
+from itertools import cycle
+from followthemoney.util import PathLike
+
+from nomenklatura.matching import get_algorithm
+from nomenklatura.matching.pairs import read_pairs
+from nomenklatura.matching.types import ScoringConfig
+
+
+log = logging.getLogger(__name__)
+
+
+def bench_matcher(name: str, pairs_file: PathLike, number: int) -> None:
+    config = ScoringConfig.defaults()
+    log.info("Loading pairs from %s", pairs_file)
+    pairs = list(read_pairs(pairs_file))
+    log.info("Read %d pairs", len(pairs))
+    matcher = get_algorithm(name)
+    if matcher is None:
+        raise ValueError("No matcher named %s", name)
+    log.info("Loaded %s", matcher.NAME)
+    infinite_pairs = cycle(pairs)
+
+    def compare_one_pair() -> None:
+        pair = next(infinite_pairs)
+        matcher.compare(pair.left, pair.right, config)
+
+    log.info("Running benchmark for %d iterations", number)
+    seconds = timeit(compare_one_pair, number=number)
+    log.info("Total time %s", datetime.timedelta(seconds=seconds))

nomenklatura/matching/compare/addresses.py

@@ -0,0 +1,71 @@
+from functools import lru_cache
+from typing import List, Set
+from followthemoney.proxy import E
+from followthemoney.types import registry
+from itertools import product
+from rigour.text import levenshtein_similarity
+from rigour.addresses import normalize_address, remove_address_keywords
+
+from nomenklatura.matching.types import FtResult, ScoringConfig
+from nomenklatura.matching.util import has_schema
+
+
+@lru_cache(maxsize=128)
+def _normalize_address(addr: str) -> Set[str]:
+    """Normalize an address string into tokens."""
+    norm = normalize_address(addr, latinize=True)
+    if norm is None:
+        return set()
+    norm = remove_address_keywords(norm, latinize=True)
+    if norm is None:
+        return set()
+    return set([n for n in norm.split() if len(n) > 0])
+
+
+def _address_match(query_addrs: List[str], result_addrs: List[str]) -> FtResult:
+    """Text similarity between addresses."""
+    if len(query_addrs) == 0 or len(result_addrs) == 0:
+        return FtResult(score=0.0, detail="No addresses provided")
+    max_result = FtResult(score=0.0, detail=None)
+    query_norms = [_normalize_address(addr) for addr in query_addrs]
+    result_norms = [_normalize_address(addr) for addr in result_addrs]
+    for query_tokens, result_tokens in product(query_norms, result_norms):
+        if len(query_tokens) == 0 or len(result_tokens) == 0:
+            continue
+        # pick out tokens that are in both sets and treat those as safe gains
+        overlap = query_tokens.intersection(result_tokens)
+        if len(overlap) == len(query_tokens) or len(overlap) == len(result_tokens):
+            detail = f"Address matches subset: {' '.join(overlap)}"
+            return FtResult(score=1.0, detail=detail)
+
+        # sort the address tokens alphabetically to address different orderings
+        query_rem = sorted([t for t in query_tokens if t not in overlap])
+        query_fuzzy = " ".join(query_rem)
+        result_rem = sorted([t for t in result_tokens if t not in overlap])
+        result_fuzzy = " ".join(result_rem)
+        fuzzy_len = max(len(query_fuzzy), len(result_fuzzy))
+        score = levenshtein_similarity(query_fuzzy, result_fuzzy, max_edits=fuzzy_len)
+
+        # combine the scores from overlap and levenshtein
+        rem_len = max(len(query_rem), len(result_rem))
+        score = (len(overlap) + (rem_len * (score))) / (rem_len + len(overlap))
+        if score > max_result.score:
+            detail = f"Matched addresses: {query_fuzzy} <-> {result_fuzzy}"
+            max_result = FtResult(score=score, detail=detail)
+    return max_result
+
+
+def address_entity_match(query: E, result: E, config: ScoringConfig) -> FtResult:
+    """Two address entities relate to similar addresses."""
+    if not has_schema(query, result, "Address"):
+        return FtResult(score=0.0, detail=None)
+    return _address_match(query.get("full"), result.get("full"))
+
+
+def address_prop_match(query: E, result: E, config: ScoringConfig) -> FtResult:
+    """Two entities have similar stated addresses."""
+    if has_schema(query, result, "Address"):
+        return FtResult(score=0.0, detail=None)
+    query_addrs = query.get_type_values(registry.address, matchable=True)
+    result_addrs = result.get_type_values(registry.address, matchable=True)
+    return _address_match(query_addrs, result_addrs)

nomenklatura/matching/compare/countries.py

@@ -0,0 +1,15 @@
+from followthemoney.proxy import E
+from followthemoney.types import registry
+
+from nomenklatura.matching.types import FtResult, ScoringConfig
+from nomenklatura.matching.util import type_pair
+
+
+def country_mismatch(query: E, result: E, config: ScoringConfig) -> FtResult:
+    """Both entities are linked to different countries."""
+    qv, rv = type_pair(query, result, registry.country)
+    if len(qv) > 0 and len(rv) > 0:
+        if len(set(qv).intersection(rv)) == 0:
+            detail = f"Different countries: {qv} / {rv}"
+            return FtResult(score=1.0, detail=detail)
+    return FtResult(score=0.0, detail=None)

nomenklatura/matching/compare/dates.py

@@ -0,0 +1,83 @@
+from typing import Iterable, Set
+from prefixdate import Precision
+from followthemoney.proxy import E
+
+from nomenklatura.matching.types import FtResult, ScoringConfig
+from nomenklatura.matching.compare.util import has_overlap
+from nomenklatura.matching.util import props_pair
+
+
+def _dates_precision(values: Iterable[str], precision: Precision) -> Set[str]:
+    dates = set()
+    for value in values:
+        if len(value) >= precision.value:
+            dates.add(value[: precision.value])
+    return dates
+
+
+def _flip_day_month(value: str) -> str:
+    # This is such a common mistake we want to accomodate it.
+    year, month, day = value.split("-", 2)
+    return f"{year}-{day}-{month}"
+
+
+def dob_matches(query: E, result: E) -> float:
+    """The birth date of the two entities is the same."""
+    query_dates, result_dates = props_pair(query, result, ["birthDate"])
+    if len(query_dates) == 0 or len(result_dates) == 0:
+        return 0.0
+    result_days = _dates_precision(result_dates, Precision.DAY)
+    query_days = _dates_precision(query_dates, Precision.DAY)
+    if has_overlap(query_days, result_days):
+        return 1.0
+    query_flipped = [_flip_day_month(d) for d in query_days]
+    if has_overlap(query_flipped, result_days):
+        return 0.5
+    return 0.0
+
+
+def dob_year_matches(query: E, result: E) -> float:
+    """The birth date of the two entities is the same."""
+    query_dates, result_dates = props_pair(query, result, ["birthDate"])
+    query_years = _dates_precision(query_dates, Precision.YEAR)
+    result_years = _dates_precision(result_dates, Precision.YEAR)
+    if has_overlap(query_years, result_years):
+        return 1.0
+    return 0.0
+
+
+def dob_day_disjoint(query: E, result: E, config: ScoringConfig) -> FtResult:
+    """The birth date of the two entities is not the same."""
+    query_dates, result_dates = props_pair(query, result, ["birthDate"])
+    if len(query_dates) == 0 or len(result_dates) == 0:
+        return FtResult(score=0.0, detail="No birth dates provided")
+    result_days = _dates_precision(result_dates, Precision.DAY)
+    query_days = _dates_precision(query_dates, Precision.DAY)
+    if len(result_days) == 0 or len(query_days) == 0:
+        return FtResult(score=0.0, detail="No birth days provided")
+    if has_overlap(query_days, result_days):
+        match = ", ".join(query_days.intersection(result_days))
+        detail = f"Birth day match: {match}"
+        return FtResult(score=0.0, detail=detail)
+    query_flipped = [_flip_day_month(d) for d in query_days]
+    if has_overlap(query_flipped, result_days):
+        match = ", ".join(result_days.intersection(query_flipped))
+        detail = f"Birth day mis-match (flipped): {match}"
+        return FtResult(score=0.5, detail=detail)
+    detail = f"Birth day mis-match: {', '.join(query_days)} vs {', '.join(result_days)}"
+    return FtResult(score=1.0, detail=detail)
+
+
+def dob_year_disjoint(query: E, result: E, config: ScoringConfig) -> FtResult:
+    """The birth date of the two entities is not the same."""
+    query_dates, result_dates = props_pair(query, result, ["birthDate"])
+    query_years = _dates_precision(query_dates, Precision.YEAR)
+    result_years = _dates_precision(result_dates, Precision.YEAR)
+    if len(query_years) == 0 or len(result_years) == 0:
+        return FtResult(score=0.0, detail="No birth years provided")
+    common = query_years.intersection(result_years)
+    if len(common) > 0:
+        detail = f"Birth year match: {', '.join(common)}"
+        return FtResult(score=0.0, detail=detail)
+    detail = f"Birth years: {', '.join(query_years)} vs {', '.join(result_years)}"
+    return FtResult(score=1.0, detail=detail)

nomenklatura/matching/compare/gender.py

@@ -0,0 +1,15 @@
+from followthemoney.proxy import E
+
+from nomenklatura.matching.types import FtResult, ScoringConfig
+from nomenklatura.matching.util import props_pair
+from nomenklatura.matching.util import FNUL
+
+
+def gender_mismatch(query: E, result: E, config: ScoringConfig) -> FtResult:
+    """Both entities have a different gender associated with them."""
+    qv, rv = props_pair(query, result, ["gender"])
+    if len(qv) > 0 and len(rv) > 0:
+        if len(set(qv).intersection(rv)) == 0:
+            detail = f"Different genders: {qv} / {rv}"
+            return FtResult(score=1.0, detail=detail)
+    return FtResult(score=FNUL, detail=None)

nomenklatura/matching/compare/identifiers.py

@@ -0,0 +1,30 @@
+from rigour.ids import StrictFormat
+from followthemoney import E, registry
+
+from nomenklatura.matching.types import FtResult, ScoringConfig
+from nomenklatura.matching.util import type_pair, props_pair, has_schema
+from nomenklatura.matching.compare.util import clean_map
+from nomenklatura.matching.util import FNUL
+
+
+def crypto_wallet_address(query: E, result: E, config: ScoringConfig) -> FtResult:
+    """Two cryptocurrency wallets have the same public key."""
+    if not has_schema(query, result, "CryptoWallet"):
+        return FtResult(score=FNUL, detail=None)
+    lv, rv = props_pair(query, result, ["publicKey"])
+    for key in lv.intersection(rv):
+        if len(key) > 10:
+            return FtResult(score=1.0, detail="Matched address: %s" % key)
+    return FtResult(score=FNUL, detail=None)
+
+
+def identifier_match(query: E, result: E, config: ScoringConfig) -> FtResult:
+    """Two entities have the same tax or registration identifier."""
+    query_ids_, result_ids_ = type_pair(query, result, registry.identifier)
+    query_ids = clean_map(query_ids_, StrictFormat.normalize)
+    result_ids = clean_map(result_ids_, StrictFormat.normalize)
+    common = query_ids.intersection(result_ids)
+    if len(common) > 0:
+        detail = "Matched identifiers: %s" % ", ".join(common)
+        return FtResult(score=1.0, detail=detail)
+    return FtResult(score=FNUL, detail=None)

nomenklatura/matching/compare/names.py

@@ -0,0 +1,157 @@
+from typing import List, Dict, Tuple
+from itertools import product
+from followthemoney.proxy import E
+from followthemoney.types import registry
+from rigour.text.distance import levenshtein_similarity
+from rigour.text.distance import jaro_winkler, is_levenshtein_plausible
+from nomenklatura.matching.types import FtResult, ScoringConfig
+from nomenklatura.matching.util import type_pair, props_pair, has_schema
+from nomenklatura.matching.compare.util import is_disjoint, clean_map
+from nomenklatura.matching.compat import clean_name_ascii, clean_name_light
+from nomenklatura.matching.compat import fingerprint_name, name_words, names_word_list
+from nomenklatura.matching.util import FNUL
+
+
+def _name_parts(name: str) -> List[str]:
+    return name_words(clean_name_ascii(name))
+
+
+def _align_name_parts(query: List[str], result: List[str]) -> float:
+    if len(query) == 0 or len(result) == 0:
+        return 0.0
+
+    scores: Dict[Tuple[str, str], float] = {}
+    # compute all pairwise scores for name parts:
+    for qn, rn in product(set(query), set(result)):
+        score = jaro_winkler(qn, rn)
+        if score > 0.0 and is_levenshtein_plausible(qn, rn):
+            scores[(qn, rn)] = score
+    pairs: List[Tuple[str, str]] = []
+    # original length of query:
+    length = len(query)
+    total_score = 1.0
+    # find the best pairing for each name part by score:
+    for (qn, rn), score in sorted(scores.items(), key=lambda i: i[1], reverse=True):
+        # one name part can only be used once, but can show up multiple times:
+        while qn in query and rn in result:
+            query.remove(qn)
+            result.remove(rn)
+            total_score = total_score * score
+            pairs.append((qn, rn))
+    # assume there should be at least a candidate for each query name part:
+    if len(pairs) < length:
+        return 0.0
+    # weakest evidence first to bias jaro-winkler for lower scores on imperfect matches:
+    aligned = pairs[::-1]
+    query_aligned = "".join(p[0] for p in aligned)
+    result_aligned = "".join(p[1] for p in aligned)
+    if not is_levenshtein_plausible(query_aligned, result_aligned):
+        return 0.0
+    # return an amped-up jaro-winkler score for the aligned name parts:
+    return total_score
+    # return jaro_winkler(query_aligned, result_aligned)
+
+
+def person_name_jaro_winkler(query: E, result: E) -> float:
+    """Compare two persons' names using the Jaro-Winkler string similarity algorithm."""
+    if not has_schema(query, result, "Person"):
+        return FNUL
+    query_names_, result_names_ = type_pair(query, result, registry.name)
+    query_names = [_name_parts(n) for n in query_names_]
+    result_names = [_name_parts(n) for n in result_names_]
+    score = 0.0
+    for qn, rn in product(query_names, result_names):
+        qns = "".join(qn)
+        rns = "".join(rn)
+        if is_levenshtein_plausible(qns, rns):
+            score = max(score, jaro_winkler(qns, rns) ** len(qns))
+        score = max(score, _align_name_parts(list(qn), list(rn)))
+    return score
+
+
+def name_fingerprint_levenshtein(query: E, result: E) -> float:
+    """Two non-person entities have similar fingerprinted names. This includes
+    simplifying entity type names (e.g. "Limited" -> "Ltd") and uses the
+    Damerau-Levensthein string distance algorithm."""
+    if has_schema(query, result, "Person"):
+        return FNUL
+    query_names, result_names = type_pair(query, result, registry.name)
+    max_score = FNUL
+    for qn, rn in product(query_names, result_names):
+        score = levenshtein_similarity(qn, rn)
+        max_score = max(max_score, score)
+        qfp = fingerprint_name(qn)
+        rfp = fingerprint_name(rn)
+        if qfp is None or rfp is None:
+            continue
+        score = levenshtein_similarity(qfp.replace(" ", ""), rfp.replace(" ", ""))
+        max_score = max(max_score, score)
+        qtokens = name_words(qfp, min_length=2)
+        rtokens = name_words(rfp, min_length=2)
+        for part in name_words(clean_name_ascii(rfp), min_length=2):
+            if part not in rtokens:
+                rtokens.append(part)
+
+        scores: Dict[Tuple[str, str], float] = {}
+        # compute all pairwise scores for name parts:
+        for q, r in product(set(qtokens), set(rtokens)):
+            scores[(q, r)] = levenshtein_similarity(
+                q, r, max_edits=None, max_percent=1.0
+            )
+        aligned: List[Tuple[str, str, float]] = []
+        # find the best pairing for each name part by score:
+        for (q, r), score in sorted(
+            scores.items(), key=lambda i: (i[1], i[0]), reverse=True
+        ):
+            # one name part can only be used once, but can show up multiple times:
+            while q in qtokens and r in rtokens:
+                qtokens.remove(q)
+                rtokens.remove(r)
+                aligned.append((q, r, score))
+        # assume there should be at least a candidate for each query name part:
+        if len(qtokens):
+            continue
+        qaligned = "".join(p[0] for p in aligned)
+        raligned = "".join(p[1] for p in aligned)
+        score = levenshtein_similarity(qaligned, raligned)
+        max_score = max(max_score, score)
+    return max_score
+
+
+def name_literal_match(query: E, result: E, config: ScoringConfig) -> FtResult:
+    """Two entities have the same name, without normalization applied to the name."""
+    query_names, result_names = type_pair(query, result, registry.name)
+    qnames = clean_map(query_names, clean_name_light)
+    rnames = clean_map(result_names, clean_name_light)
+    overlap = qnames.intersection(rnames)
+    if len(overlap) == 0:
+        return FtResult(score=FNUL, detail=None)
+    detail = f"Identical names: {', '.join(overlap)}"
+    return FtResult(score=1.0, detail=detail)
+
+
+def last_name_mismatch(query: E, result: E) -> float:
+    """The two persons have different last names."""
+    qv, rv = props_pair(query, result, ["lastName"])
+    qvt = names_word_list(qv, min_length=2)
+    rvt = names_word_list(rv, min_length=2)
+    # TODO: levenshtein
+    # for (qn, rn) in product(qvt, rvt):
+    #     similarity = levenshtein_similarity(qn, rn)
+    return 1.0 if is_disjoint(qvt, rvt) else FNUL
+
+
+def weak_alias_match(query: E, result: E, config: ScoringConfig) -> FtResult:
+    """The query name is exactly the same as a result's weak alias."""
+    # NOTE: This is unbalanced, i.e. it treats 'query' and 'result' differently.
+    # cf. https://ofac.treasury.gov/faqs/topic/1646
+    query_names = query.get_type_values(registry.name)
+    query_names.extend(query.get("weakAlias", quiet=True))
+    result_names = result.get("weakAlias", quiet=True)
+    qnames = clean_map(query_names, clean_name_light)
+    rnames = clean_map(result_names, clean_name_light)
+    overlap = qnames.intersection(rnames)
+    if len(overlap) == 0:
+        return FtResult(score=FNUL, detail=None)
+    detail = f"Matched weak alias: {', '.join(overlap)}"
+    return FtResult(score=1.0, detail=detail)

nomenklatura/matching/compare/util.py

@@ -0,0 +1,51 @@
+import re
+from typing import List, Set, Union, Iterable, Callable, Optional
+
+CleanFunc = Optional[Callable[[str], Optional[str]]]
+FIND_NUM = re.compile(r"\d{1,}")
+
+
+def is_disjoint(
+    left: Union[Set[str], List[str]],
+    right: Union[Set[str], List[str]],
+) -> bool:
+    """Returns true if both sequences are non-empty but have no common values."""
+    if len(left) and len(right):
+        if set(left).isdisjoint(right):
+            return True
+    return False
+
+
+def has_overlap(
+    left: Union[Set[str], List[str]],
+    right: Union[Set[str], List[str]],
+) -> bool:
+    """Returns true if both sequences are non-empty and have common values."""
+    if not set(left).isdisjoint(right):
+        return True
+    return False
+
+
+def clean_map(
+    texts: Iterable[Optional[str]],
+    clean: CleanFunc = None,
+) -> Set[str]:
+    """Apply a cleaning function to a set of strings and only return non-empty ones."""
+    out: Set[str] = set()
+    for text in texts:
+        if text is None or len(text) == 0:
+            continue
+        if clean is not None:
+            text = clean(text)
+            if text is None or len(text) == 0:
+                continue
+        out.add(text)
+    return out
+
+
+def extract_numbers(values: List[str]) -> Set[str]:
+    """Extract all numbers from a list of strings."""
+    numbers: Set[str] = set()
+    for value in values:
+        numbers.update(FIND_NUM.findall(value))
+    return numbers