nomenklatura-mpt 4.1.9__py3-none-any.whl

nomenklatura/matching/logic_v1/identifiers.py

@@ -0,0 +1,104 @@
+from itertools import product
+from rigour.ids import LEI, ISIN, INN, OGRN, IMO, BIC
+from rigour.ids import StrictFormat
+from rigour.text.distance import levenshtein
+from followthemoney import E, registry
+
+from nomenklatura.matching.types import FtResult, ScoringConfig
+from nomenklatura.matching.util import has_schema, type_pair
+from nomenklatura.matching.compare.util import clean_map, CleanFunc
+
+
+def _id_prop_match(
+    query: E,
+    result: E,
+    prop_name: str,
+    clean: CleanFunc = None,
+) -> bool:
+    """Check if a specific property identifier is shared by two entities."""
+    prop = query.schema.get(prop_name)
+    if prop is None:
+        return False
+    lv = clean_map(query.get(prop), clean=clean)
+    if not len(lv):
+        return False
+    rv_ = result.get_type_values(prop.type, matchable=True)
+    rv = clean_map(rv_, clean=clean)
+    common = lv.intersection(rv)
+    return len(common) > 0
+
+
+def _bidi_id_prop_match(
+    query: E,
+    result: E,
+    prop_name: str,
+    clean: CleanFunc = None,
+) -> FtResult:
+    """Check if a specific property identifier is shared by two entities."""
+    if _id_prop_match(query, result, prop_name, clean=clean):
+        return FtResult(score=1.0, detail="Property match: %r" % prop_name)
+    if _id_prop_match(result, query, prop_name, clean=clean):
+        return FtResult(score=1.0, detail="Property match: %r" % prop_name)
+    return FtResult(score=0.0, detail="No match: %r" % prop_name)
+
+
+def lei_code_match(query: E, result: E, config: ScoringConfig) -> FtResult:
+    """Two entities have the same Legal Entity Identifier."""
+    return _bidi_id_prop_match(query, result, "leiCode", LEI.normalize)
+
+
+def bic_code_match(query: E, result: E, config: ScoringConfig) -> FtResult:
+    """Two entities have the same SWIFT BIC."""
+    return _bidi_id_prop_match(query, result, "swiftBic", BIC.normalize)
+
+
+def ogrn_code_match(query: E, result: E, config: ScoringConfig) -> FtResult:
+    """Two entities have the same Russian company registration (OGRN) code."""
+    return _bidi_id_prop_match(query, result, "ogrnCode", OGRN.normalize)
+
+
+def inn_code_match(query: E, result: E, config: ScoringConfig) -> FtResult:
+    """Two entities have the same Russian tax identifier (INN)."""
+    return _bidi_id_prop_match(query, result, "innCode", INN.normalize)
+
+
+def isin_security_match(query: E, result: E, config: ScoringConfig) -> FtResult:
+    """Two securities have the same ISIN."""
+    if not has_schema(query, result, "Security"):
+        return FtResult(score=0.0, detail="None of the entities is a security")
+    return _bidi_id_prop_match(query, result, "isin", ISIN.normalize)
+
+
+def vessel_imo_mmsi_match(query: E, result: E, config: ScoringConfig) -> FtResult:
+    """Two vessels have the same IMO or MMSI identifier."""
+    imo_res = _bidi_id_prop_match(query, result, "imoNumber", IMO.normalize)
+    if imo_res.score > 0.0:
+        return imo_res
+    return _bidi_id_prop_match(query, result, "mmsi")
+
+
+def orgid_disjoint(query: E, result: E, config: ScoringConfig) -> FtResult:
+    """Two companies or organizations have different tax identifiers or registration
+    numbers."""
+    if not has_schema(query, result, "Organization"):
+        return FtResult(score=0.0, detail=None)
+    query_ids_, result_ids_ = type_pair(query, result, registry.identifier)
+    query_ids = clean_map(query_ids_, StrictFormat.normalize)
+    result_ids = clean_map(result_ids_, StrictFormat.normalize)
+    if not len(query_ids) or not len(result_ids):
+        return FtResult(score=0.0, detail=None)
+    common = query_ids.intersection(result_ids)
+    if len(common) > 0:
+        return FtResult(score=0.0, detail=None)
+    max_ratio = 0.0
+    for query_id, result_id in product(query_ids, result_ids):
+        distance = levenshtein(query_id, result_id)
+        max_len = max(len(query_id), len(result_id))
+        ratio = 1.0 - (distance / float(max_len))
+        if ratio > 0.7:
+            max_ratio = max(max_ratio, ratio)
+    detail = "Mismatched identifiers: %s vs %s" % (
+        ", ".join(query_ids),
+        ", ".join(result_ids),
+    )
+    return FtResult(score=1 - max_ratio, detail=detail)

nomenklatura/matching/logic_v1/model.py

@@ -0,0 +1,76 @@
+from typing import Dict, List
+
+from nomenklatura.matching.types import Feature, HeuristicAlgorithm, FtResult
+from nomenklatura.matching.compare.countries import country_mismatch
+from nomenklatura.matching.compare.gender import gender_mismatch
+from nomenklatura.matching.compare.identifiers import crypto_wallet_address
+from nomenklatura.matching.compare.identifiers import identifier_match
+from nomenklatura.matching.compare.dates import dob_day_disjoint, dob_year_disjoint
+from nomenklatura.matching.compare.names import person_name_jaro_winkler
+from nomenklatura.matching.compare.names import last_name_mismatch, name_literal_match
+from nomenklatura.matching.compare.names import name_fingerprint_levenshtein
+from nomenklatura.matching.compare.names import weak_alias_match
+from nomenklatura.matching.compare.addresses import address_entity_match
+from nomenklatura.matching.logic_v1.phonetic import person_name_phonetic_match
+from nomenklatura.matching.logic_v1.phonetic import name_soundex_match
+from nomenklatura.matching.logic_v1.phonetic import name_metaphone_match
+from nomenklatura.matching.logic_v1.identifiers import bic_code_match
+from nomenklatura.matching.logic_v1.identifiers import inn_code_match, ogrn_code_match
+from nomenklatura.matching.logic_v1.identifiers import isin_security_match
+from nomenklatura.matching.logic_v1.identifiers import lei_code_match
+from nomenklatura.matching.logic_v1.identifiers import vessel_imo_mmsi_match
+from nomenklatura.matching.logic_v1.identifiers import orgid_disjoint
+from nomenklatura.matching.logic_v1.multi import numbers_mismatch
+from nomenklatura.matching.util import FNUL
+
+
+class LogicV1(HeuristicAlgorithm):
+    """A rule-based matching system that generates a set of basic scores via
+    name and identifier-based matching, and then qualifies that score using
+    supporting or contradicting features of the two entities."""
+
+    NAME = "logic-v1"
+    features = [
+        Feature(func=name_literal_match, weight=1.0),
+        Feature(func=FtResult.wrap(person_name_jaro_winkler), weight=0.8),
+        Feature(func=FtResult.wrap(person_name_phonetic_match), weight=0.9),
+        Feature(func=FtResult.wrap(name_fingerprint_levenshtein), weight=0.9),
+        # These are there so they can be enabled using custom weights:
+        Feature(func=FtResult.wrap(name_metaphone_match), weight=FNUL),
+        Feature(func=FtResult.wrap(name_soundex_match), weight=FNUL),
+        Feature(func=address_entity_match, weight=0.98),
+        Feature(func=crypto_wallet_address, weight=0.98),
+        Feature(func=isin_security_match, weight=0.98),
+        Feature(func=lei_code_match, weight=0.95),
+        Feature(func=ogrn_code_match, weight=0.95),
+        Feature(func=vessel_imo_mmsi_match, weight=0.95),
+        Feature(func=inn_code_match, weight=0.95),
+        Feature(func=bic_code_match, weight=0.95),
+        Feature(func=identifier_match, weight=0.85),
+        Feature(func=weak_alias_match, weight=0.8),
+        Feature(func=country_mismatch, weight=-0.2, qualifier=True),
+        Feature(func=FtResult.wrap(last_name_mismatch), weight=-0.2, qualifier=True),
+        Feature(func=dob_year_disjoint, weight=-0.15, qualifier=True),
+        Feature(func=dob_day_disjoint, weight=-0.2, qualifier=True),
+        Feature(func=gender_mismatch, weight=-0.2, qualifier=True),
+        Feature(func=orgid_disjoint, weight=-0.2, qualifier=True),
+        Feature(func=numbers_mismatch, weight=-0.1, qualifier=True),
+    ]
+
+    @classmethod
+    def compute_score(
+        cls, scores: Dict[str, float], weights: Dict[str, float]
+    ) -> float:
+        mains: List[float] = []
+        for feat in cls.features:
+            if feat.qualifier:
+                continue
+            weight = scores.get(feat.name, FNUL) * weights.get(feat.name, FNUL)
+            mains.append(weight)
+        score = max(mains)
+        for feat in cls.features:
+            if not feat.qualifier:
+                continue
+            weight = scores.get(feat.name, FNUL) * weights.get(feat.name, FNUL)
+            score += weight
+        return score

nomenklatura/matching/logic_v1/multi.py

@@ -0,0 +1,21 @@
+from followthemoney.proxy import E
+from followthemoney.types import registry
+
+from nomenklatura.matching.types import FtResult, ScoringConfig
+from nomenklatura.matching.compare.util import extract_numbers
+from nomenklatura.matching.util import type_pair, has_schema
+
+
+def numbers_mismatch(query: E, result: E, config: ScoringConfig) -> FtResult:
+    """Find numbers in names and addresses and penalise different numbers."""
+    if has_schema(query, result, "Address"):
+        qv, rv = type_pair(query, result, registry.address)
+    else:
+        qv, rv = type_pair(query, result, registry.name)
+    qvn = extract_numbers(qv)
+    rvn = extract_numbers(rv)
+    base = min(len(qvn), len(rvn))
+    mismatch = len(qvn.difference(rvn))
+    # print("numbers_mismatch", mismatch, base, qvn, rvn)
+    score = float(mismatch) / float(max(1, base))
+    return FtResult(score=score, detail="Mismatching numbers: %s" % mismatch)

nomenklatura/matching/logic_v1/phonetic.py

@@ -0,0 +1,142 @@
+from functools import cached_property
+from typing import List, Optional
+from itertools import product
+from normality import ascii_text
+from followthemoney.proxy import E
+from followthemoney.types import registry
+from rigour.text.scripts import can_latinize
+from rigour.text.distance import is_levenshtein_plausible
+from rigour.text.phonetics import metaphone, soundex
+from rigour.names import tokenize_name
+from rigour.util import list_intersection
+
+from nomenklatura.matching.util import type_pair, has_schema
+from nomenklatura.matching.compat import fingerprint_name, name_words
+
+
+class NameTokenPhonetic:
+    def __init__(self, token: str):
+        self.token = token
+        self.ascii = ascii_text(token) if can_latinize(token) else None
+
+    @cached_property
+    def metaphone(self) -> Optional[str]:
+        if self.ascii is not None:
+            phoneme = metaphone(self.ascii)
+            if len(phoneme) >= 3:
+                return phoneme
+        return None
+
+    # def __repr__(self) -> str:
+    #     return f"<NameTokenPhonetic {self.token!r}, {self.ascii!r}, {self.metaphone!r}>"
+
+    @classmethod
+    def from_name(cls, name: str) -> List["NameTokenPhonetic"]:
+        tokens = tokenize_name(name.lower(), token_min_length=2)
+        return [cls(token) for token in tokens]
+
+
+def metaphone_token(token: str) -> str:
+    if token.isalpha() and len(token) > 1:
+        out = metaphone(token)
+        # doesn't handle non-ascii characters
+        if len(out) >= 3:
+            return out
+    return token.upper()
+
+
+def soundex_token(token: str) -> str:
+    if token.isalpha() and len(token) > 1:
+        out = soundex(token)
+        # doesn't handle non-ascii characters
+        if len(out):
+            return out
+    return token.upper()
+
+
+def compare_parts_phonetic(left: NameTokenPhonetic, right: NameTokenPhonetic) -> bool:
+    if left.metaphone is None or right.metaphone is None:
+        return left.ascii == right.ascii
+    if (
+        left.metaphone == right.metaphone
+        and left.ascii is not None
+        and right.ascii is not None
+    ):
+        # Secondary check for Levenshtein distance:
+        if is_levenshtein_plausible(left.ascii, right.ascii):
+            return True
+    return False
+
+
+def _clean_phonetic_entity(original: str) -> Optional[str]:
+    """Normalize a legal entity name without transliteration."""
+    if not can_latinize(original):
+        return None
+    return fingerprint_name(original)
+
+
+def _token_names_compare(
+    query_names: List[List[str]], result_names: List[List[str]]
+) -> float:
+    score = 0.0
+    for q, r in product(query_names, result_names):
+        # length = max(2.0, (len(q) + len(r)) / 2.0)
+        length = max(2.0, len(q))
+        combo = len(list_intersection(q, r)) / float(length)
+        score = max(score, combo)
+    return score
+
+
+def person_name_phonetic_match(query: E, result: E) -> float:
+    """Two persons have similar names, using a phonetic algorithm."""
+    if not has_schema(query, result, "Person"):
+        return 0.0
+    query_names_, result_names_ = type_pair(query, result, registry.name)
+    query_parts = [NameTokenPhonetic.from_name(n) for n in query_names_]
+    result_parts = [NameTokenPhonetic.from_name(n) for n in result_names_]
+    score = 0.0
+    for q, r in product(query_parts, result_parts):
+        if len(q) == 0:
+            continue
+        matches = list(r)
+        matched = 0
+        for part in q:
+            for other in matches:
+                if compare_parts_phonetic(part, other):
+                    matches.remove(other)
+                    matched += 1
+                    break
+        score = max(score, matched / float(len(q)))
+    return score
+
+
+def _metaphone_tokens(token: str) -> List[str]:
+    words: List[str] = []
+    for word in name_words(_clean_phonetic_entity(token), min_length=2):
+        words.append(metaphone_token(word))
+    return words
+
+
+def name_metaphone_match(query: E, result: E) -> float:
+    """Two entities (person and non-person) have similar names, using the metaphone
+    algorithm."""
+    query_names_, result_names_ = type_pair(query, result, registry.name)
+    query_names = [_metaphone_tokens(n) for n in query_names_]
+    result_names = [_metaphone_tokens(n) for n in result_names_]
+    return _token_names_compare(query_names, result_names)
+
+
+def _soundex_tokens(token: str) -> List[str]:
+    words: List[str] = []
+    for word in name_words(_clean_phonetic_entity(token), min_length=2):
+        words.append(soundex_token(word))
+    return words
+
+
+def name_soundex_match(query: E, result: E) -> float:
+    """Two entities (person and non-person) have similar names, using the soundex
+    algorithm."""
+    query_names_, result_names_ = type_pair(query, result, registry.name)
+    query_names = [_soundex_tokens(n) for n in query_names_]
+    result_names = [_soundex_tokens(n) for n in result_names_]
+    return _token_names_compare(query_names, result_names)

nomenklatura/matching/logic_v2/identifiers.py

@@ -0,0 +1,124 @@
+from typing import Set, Type
+
+from rigour.ids import get_identifier_format, IdentifierFormat
+from followthemoney import model
+from followthemoney.property import Property
+from followthemoney.types import registry
+from followthemoney.proxy import EntityProxy
+
+from nomenklatura.matching.types import FtResult, ScoringConfig
+
+
+def _format_normalize(
+    format: Type[IdentifierFormat], entity: EntityProxy, prop: Property
+) -> Set[str]:
+    values: Set[str] = set()
+    for value in entity.get(prop, quiet=True):
+        norm_value = format.normalize(value)
+        if norm_value is not None:
+            values.add(norm_value)
+    return values
+
+
+def _identifier_format_match(
+    format_name: str, query: EntityProxy, result: EntityProxy
+) -> FtResult:
+    """Check if the identifier format is the same for two entities."""
+    schema = model.common_schema(query.schema, result.schema)
+    format = get_identifier_format(format_name)
+    query_identifiers: Set[str] = set()
+    query_format: Set[str] = set()
+    result_identifiers: Set[str] = set()
+    result_format: Set[str] = set()
+    for prop in schema.properties.values():
+        if prop.type != registry.identifier or not prop.matchable:
+            continue
+        if prop.format is not None and get_identifier_format(prop.format) != format:
+            continue
+        query_values = _format_normalize(format, query, prop)
+        query_identifiers.update(query_values)
+        result_values = _format_normalize(format, result, prop)
+        result_identifiers.update(result_values)
+        if prop.format is not None and get_identifier_format(prop.format) == format:
+            query_format.update(query_values)
+            result_format.update(result_values)
+    left_common = query_format.intersection(result_identifiers)
+    if len(left_common) > 0:
+        detail = f"Matched {format.TITLE}: {', '.join(left_common)}"
+        return FtResult(score=1.0, detail=detail)
+    right_common = result_format.intersection(query_identifiers)
+    if len(right_common) > 0:
+        detail = f"Matched {format.TITLE}: {', '.join(right_common)}"
+        return FtResult(score=1.0, detail=detail)
+    if format.STRONG:
+        non_common = query_identifiers.intersection(result_identifiers)
+        if len(non_common) > 0:
+            detail = f"Out-of-format match: {', '.join(non_common)}"
+            return FtResult(score=0.8, detail=detail)
+    return FtResult(score=0.0, detail=f"No {format.TITLE} match")
+
+
+def lei_code_match(
+    query: EntityProxy, result: EntityProxy, config: ScoringConfig
+) -> FtResult:
+    """Two entities have the same Legal Entity Identifier."""
+    return _identifier_format_match("lei", query, result)
+
+
+def bic_code_match(
+    query: EntityProxy, result: EntityProxy, config: ScoringConfig
+) -> FtResult:
+    """Two entities have the same SWIFT BIC."""
+    return _identifier_format_match("bic", query, result)
+
+
+def ogrn_code_match(
+    query: EntityProxy, result: EntityProxy, config: ScoringConfig
+) -> FtResult:
+    """Two entities have the same Russian company registration (OGRN) code."""
+    return _identifier_format_match("ogrn", query, result)
+
+
+def inn_code_match(
+    query: EntityProxy, result: EntityProxy, config: ScoringConfig
+) -> FtResult:
+    """Two entities have the same Russian tax identifier (INN)."""
+    return _identifier_format_match("inn", query, result)
+
+
+def uei_code_match(
+    query: EntityProxy, result: EntityProxy, config: ScoringConfig
+) -> FtResult:
+    """Two entities have the same US Unique Entity ID (UEI)."""
+    return _identifier_format_match("uei", query, result)
+
+
+def npi_code_match(
+    query: EntityProxy, result: EntityProxy, config: ScoringConfig
+) -> FtResult:
+    """Two entities have the same US National Provider Identifier (NPI)."""
+    return _identifier_format_match("npi", query, result)
+
+
+def isin_security_match(
+    query: EntityProxy, result: EntityProxy, config: ScoringConfig
+) -> FtResult:
+    """Two securities have the same ISIN."""
+    # if not has_schema(query, result, "Security"):
+    #     return 0.0
+    return _identifier_format_match("isin", query, result)
+
+
+def vessel_imo_mmsi_match(
+    query: EntityProxy, result: EntityProxy, config: ScoringConfig
+) -> FtResult:
+    """Two vessels have the same IMO or MMSI identifier."""
+    imo_res = _identifier_format_match("imo", query, result)
+    if imo_res.score > 0.0:
+        return imo_res
+    query_mmsis = query.get("mmsi", quiet=True)
+    result_mmsis = result.get("mmsi", quiet=True)
+    score = registry.identifier.compare_sets(query_mmsis, result_mmsis)
+    if score > 0.0:
+        return FtResult(score=score, detail="MMSI match")
+    return FtResult(score=score, detail="No IMO or MMSI match")

nomenklatura/matching/logic_v2/model.py

@@ -0,0 +1,98 @@
+from typing import Dict, List
+
+from nomenklatura.matching.types import Feature, HeuristicAlgorithm
+from nomenklatura.matching.types import ConfigVar, ConfigVarType
+from nomenklatura.matching.compare.countries import country_mismatch
+from nomenklatura.matching.compare.gender import gender_mismatch
+from nomenklatura.matching.compare.identifiers import crypto_wallet_address
+from nomenklatura.matching.compare.identifiers import identifier_match
+from nomenklatura.matching.compare.dates import dob_day_disjoint, dob_year_disjoint
+from nomenklatura.matching.compare.names import weak_alias_match
+from nomenklatura.matching.compare.addresses import address_entity_match
+from nomenklatura.matching.compare.addresses import address_prop_match
+from nomenklatura.matching.logic_v2.names.match import name_match
+from nomenklatura.matching.logic_v2.identifiers import bic_code_match
+from nomenklatura.matching.logic_v2.identifiers import inn_code_match, ogrn_code_match
+from nomenklatura.matching.logic_v2.identifiers import isin_security_match
+from nomenklatura.matching.logic_v2.identifiers import lei_code_match
+from nomenklatura.matching.logic_v2.identifiers import vessel_imo_mmsi_match
+from nomenklatura.matching.logic_v2.identifiers import uei_code_match
+from nomenklatura.matching.logic_v2.identifiers import npi_code_match
+from nomenklatura.matching.util import FNUL
+
+
+class LogicV2(HeuristicAlgorithm):
+    """A rule-based matching system that generates a set of basic scores via
+    name and identifier-based matching, and then qualifies that score using
+    supporting or contradicting features of the two entities. Version 2 uses
+    a different set of features and consolidates name matching into a single
+    feature, which uses a versatile and complex name matching algorithm."""
+
+    NAME = "logic-v2"
+    features = [
+        Feature(func=name_match, weight=1.0),
+        Feature(func=address_entity_match, weight=0.98),
+        Feature(func=crypto_wallet_address, weight=0.98),
+        Feature(func=isin_security_match, weight=0.98),
+        Feature(func=lei_code_match, weight=0.95),
+        Feature(func=ogrn_code_match, weight=0.95),
+        Feature(func=vessel_imo_mmsi_match, weight=0.95),
+        Feature(func=inn_code_match, weight=0.95),
+        Feature(func=bic_code_match, weight=0.95),
+        Feature(func=uei_code_match, weight=0.95),
+        Feature(func=npi_code_match, weight=0.95),
+        Feature(func=identifier_match, weight=0.85),
+        Feature(func=weak_alias_match, weight=0.8),
+        Feature(func=address_prop_match, weight=0.2, qualifier=True),
+        Feature(func=country_mismatch, weight=-0.2, qualifier=True),
+        Feature(func=dob_year_disjoint, weight=-0.15, qualifier=True),
+        Feature(func=dob_day_disjoint, weight=-0.25, qualifier=True),
+        Feature(func=gender_mismatch, weight=-0.2, qualifier=True),
+    ]
+    CONFIG = {
+        "nm_number_mismatch": ConfigVar(
+            type=ConfigVarType.FLOAT,
+            description="Penalty for mismatching numbers in object or company names.",
+            default=0.3,
+        ),
+        "nm_extra_query_name": ConfigVar(
+            type=ConfigVarType.FLOAT,
+            description="Weight for name parts in the query not matched to the result.",
+            default=0.8,
+        ),
+        "nm_extra_result_name": ConfigVar(
+            type=ConfigVarType.FLOAT,
+            description="Weight for name parts in the result not matched to the query.",
+            default=0.2,
+        ),
+        "nm_family_name_weight": ConfigVar(
+            type=ConfigVarType.FLOAT,
+            description="Extra weight multiplier for family name in person matches (John Smith vs. John Gruber is clearly distinct).",
+            default=1.3,
+        ),
+        "nm_fuzzy_cutoff_factor": ConfigVar(
+            type=ConfigVarType.FLOAT,
+            description="Extra factor for when a fuzzy match is triggered in name matching. "
+            "Below a certain threshold, a fuzzy match is considered as a non-match (score = 0.0). "
+            "Adjusting this multiplier will raise this threshold, making a fuzzy match trigger more leniently.",
+            default=1.0,
+        ),
+    }
+
+    @classmethod
+    def compute_score(
+        cls, scores: Dict[str, float], weights: Dict[str, float]
+    ) -> float:
+        mains: List[float] = []
+        for feat in cls.features:
+            if feat.qualifier:
+                continue
+            weight = scores.get(feat.name, FNUL) * weights.get(feat.name, FNUL)
+            mains.append(weight)
+        score = max(mains)
+        for feat in cls.features:
+            if not feat.qualifier:
+                continue
+            weight = scores.get(feat.name, FNUL) * weights.get(feat.name, FNUL)
+            score += weight
+        return score

nomenklatura/matching/logic_v2/names/__init__.py

@@ -0,0 +1,3 @@
+from nomenklatura.matching.logic_v2.names.match import name_match
+
+__all__ = ["name_match"]

nomenklatura/matching/logic_v2/names/analysis.py

@@ -0,0 +1,51 @@
+from typing import Set
+from rigour.names import NameTypeTag, Name
+from rigour.names import replace_org_types_compare, prenormalize_name
+from rigour.names import remove_person_prefixes, remove_org_prefixes
+from rigour.names import tag_org_name, tag_person_name, normalize_name
+from followthemoney import registry, EntityProxy
+from followthemoney.names import PROP_PART_TAGS
+
+
+def entity_names(
+    type_tag: NameTypeTag, entity: EntityProxy, is_query: bool = False
+) -> Set[Name]:
+    """This will transform the entity into a set of names with tags applied. The idea
+    is to tag the names with the type of entity they are, e.g. person, organization,
+    etc. and to tag the parts of the name with their type, e.g. first name, last name,
+    etc. Some extra heuristics and de-duplication are applied to reduce the number of
+    comparisons needed to find the best match.
+    """
+    seen: Set[str] = set()
+    names: Set[Name] = set()
+    for name in entity.get_type_values(registry.name, matchable=True):
+        # Remove prefix like "Mr.", "Ms.", "Dr." from the name:
+        if type_tag == NameTypeTag.PER:
+            name = remove_person_prefixes(name)
+
+        form = prenormalize_name(name)
+        if type_tag in (NameTypeTag.ORG, NameTypeTag.ENT):
+            # Replace organization types with their canonical form, e.g. "Limited Liability Company" -> "LLC"
+            form = replace_org_types_compare(form, normalizer=prenormalize_name)
+            # Remove organization prefixes like "The" (actually that's it right now)
+            form = remove_org_prefixes(form)
+
+        if form in seen:
+            continue
+        seen.add(form)
+        sname = Name(name, form=form, tag=type_tag)
+        # tag name parts from properties:
+        for prop, tag in PROP_PART_TAGS:
+            for value in entity.get(prop, quiet=True):
+                sname.tag_text(prenormalize_name(value), tag)
+
+        # tag organization types and symbols:
+        if type_tag in (NameTypeTag.ORG, NameTypeTag.ENT):
+            tag_org_name(sname, normalize_name)
+
+        if type_tag == NameTypeTag.PER:
+            tag_person_name(sname, normalize_name, any_initials=is_query)
+
+        # TODO: should we tag phonetic tokens here?
+        names.add(sname)
+    return names