nomenklatura-mpt 4.1.9__py3-none-any.whl

nomenklatura/matching/logic_v2/names/distance.py

@@ -0,0 +1,181 @@
+import math
+from functools import lru_cache
+from collections import defaultdict
+from itertools import zip_longest
+from typing import Dict, List, Optional, Tuple
+from rapidfuzz.distance import Levenshtein, Opcodes
+from rigour.names import NamePart, is_stopword
+from rigour.text.distance import levenshtein
+
+from nomenklatura.matching.logic_v2.names.util import Match
+from nomenklatura.matching.types import ScoringConfig
+from nomenklatura.util import unroll
+
+SEP = " "
+SIMILAR_PAIRS = [
+    ("0", "o"),
+    ("1", "i"),
+    ("g", "9"),
+    ("q", "9"),
+    ("b", "6"),
+    ("5", "s"),
+    ("e", "i"),
+    ("1", "l"),
+    ("o", "u"),
+    ("i", "j"),
+    ("i", "y"),
+    ("c", "k"),
+    ("n", "h"),
+]
+SIMILAR_PAIRS = SIMILAR_PAIRS + [(b, a) for a, b in SIMILAR_PAIRS]
+
+
+@lru_cache(maxsize=512)
+def strict_levenshtein(left: str, right: str, max_rate: int = 4) -> float:
+    """Calculate the string distance between two strings."""
+    if left == right:
+        return 1.0
+    max_len = max(len(left), len(right))
+    max_edits = max_len // max_rate
+    if max_edits < 1:  # We already checked for equality
+        return 0.0
+    distance = levenshtein(left, right, max_edits=max_len)
+    if distance > max_edits:
+        return 0.0
+    return (1 - (distance / max_len)) ** max_edits
+
+
+def _edit_cost(op: str, qc: Optional[str], rc: Optional[str]) -> float:
+    """Calculate the cost of a pair of characters."""
+    if op == "equal":
+        return 0.0
+    if qc == SEP and rc is None:
+        return 0.2
+    if rc == SEP and qc is None:
+        return 0.2
+    if (qc, rc) in SIMILAR_PAIRS:
+        return 0.7
+    if qc is not None and qc.isdigit():
+        return 1.5
+    if rc is not None and rc.isdigit():
+        return 1.5
+    return 1.0
+
+
+def _costs_similarity(costs: List[float], max_cost_bias: float = 1.0) -> float:
+    """Calculate a similarity score based on a list of costs."""
+    if len(costs) == 0:
+        return 0.0
+    # max_cost defines how many edits we allow for a given length.
+    # We use a log here because for very long names, we don't want an anything goes
+    # policy for very long name strings (~hundreds of characters).
+    # The log-base is a bit of a magic number. We adjusted it so that for
+    # len 8 it allows ~2 edits. That seems reasonable, but is also entirely arbitrary.
+    # We use log(x-2) to disable fuzzy-matching completely for very short
+    # names (often Chinese names in practice).
+    max_cost = math.log(max(len(costs) - 2, 1), 2.35) * max_cost_bias
+    total_cost = sum(costs)
+    if total_cost == 0:
+        return 1.0
+    if total_cost > max_cost:
+        return 0.0
+    # Normalize the score to be between 0 and 1
+    return 1 - (total_cost / len(costs))
+
+
+@lru_cache(maxsize=512)
+def _opcodes(qry_text: str, res_text: str) -> Opcodes:
+    """Get the opcodes for the Levenshtein distance between two strings."""
+    return Levenshtein.opcodes(qry_text, res_text)
+
+
+def weighted_edit_similarity(
+    qry_parts: List[NamePart], res_parts: List[NamePart], config: ScoringConfig
+) -> List[Match]:
+    """Calculate a weighted similarity score between two sets of name parts. This function implements custom
+    frills within the context of a simple Levenshtein distance calculation. For example:
+
+    * The result is returned as a list of Match objects, which contain a score, but also a weight.
+    * Removals of full tokens are penalized more lightly than intra-token edits.
+    * Some edits inside of words are considered more similar than others, e.g. "o" and "0".
+    """
+    if len(qry_parts) == 0 and len(res_parts) == 0:
+        return []
+    qry_text = SEP.join(p.comparable for p in qry_parts)
+    res_text = SEP.join(p.comparable for p in res_parts)
+
+    # Keep track of which name parts overlap and how many characters they share in the alignment
+    # produced by rapidfuzz Levenshtein opcodes.
+    overlaps: Dict[Tuple[NamePart, NamePart], int] = defaultdict(int)
+
+    # Keep track of the costs for each name part, so we can calculate a similarity score later.
+    costs: Dict[NamePart, List[float]] = defaultdict(list)
+
+    if len(qry_parts) and len(res_parts):
+        qry_cur = qry_parts[0]
+        res_cur = res_parts[0]
+        for op in _opcodes(qry_text, res_text):
+            qry_span = qry_text[op.src_start : op.src_end]
+            res_span = res_text[op.dest_start : op.dest_end]
+            for qc, rc in zip_longest(qry_span, res_span, fillvalue=None):
+                if op.tag == "equal":
+                    if qc not in (None, SEP) and rc not in (None, SEP):
+                        # TODO: should this also include "replace"?
+                        overlaps[(qry_cur, res_cur)] += 1
+                cost = _edit_cost(op.tag, qc, rc)
+                if qc is not None:
+                    costs[qry_cur].append(cost)
+                    if qc == SEP:
+                        next_idx = qry_parts.index(qry_cur) + 1
+                        if len(qry_parts) >= next_idx:
+                            qry_cur = qry_parts[next_idx]
+                if rc is not None:
+                    costs[res_cur].append(cost)
+                    if rc == SEP:
+                        next_idx = res_parts.index(res_cur) + 1
+                        if len(res_parts) >= next_idx:
+                            res_cur = res_parts[next_idx]
+
+    # Use the overlaps to create matches between query and result parts.
+    part_matches: Dict[NamePart, Match] = {}
+    for (qp, rp), overlap in overlaps.items():
+        min_len = min(len(qp.comparable), len(rp.comparable))
+        if overlap / min_len > 0.51:
+            match = part_matches.get(qp, part_matches.get(rp, Match()))
+            if qp not in match.qps:
+                match.qps.append(qp)
+            if rp not in match.rps:
+                match.rps.append(rp)
+            part_matches[rp] = match
+            part_matches[qp] = match
+
+    # Compute the scores where an overlap was applied
+    bias = config.get_float("nm_fuzzy_cutoff_factor")
+    matches = set(part_matches.values())
+    for match in matches:
+        # Score down stopwords:
+        if len(match.qps) == 1 and len(match.rps) == 1:
+            if is_stopword(match.qps[0].form):
+                match.weight = 0.7
+
+        qcosts = unroll(costs.get(p, [1.0]) for p in match.qps)
+        rcosts = unroll(costs.get(p, [1.0]) for p in match.rps)
+        match.score = _costs_similarity(qcosts, max_cost_bias=bias) * _costs_similarity(
+            rcosts, max_cost_bias=bias
+        )
+
+    # Non-matched query parts: this penalizes scenarios where name parts in the query are
+    # not matched to any name part in the result. Increasing this penalty will require queries
+    # to always be matched in full.
+    for qp in qry_parts:
+        if qp not in part_matches:
+            match = Match(qps=[qp])
+            matches.add(match)
+
+    # Non-matched result parts
+    for rp in res_parts:
+        if rp not in part_matches:
+            match = Match(rps=[rp])
+            matches.add(match)
+
+    return list(matches)

nomenklatura/matching/logic_v2/names/magic.py

@@ -0,0 +1,60 @@
+from typing import List
+from rigour.names import is_stopword
+from rigour.names import Name, NamePart, Symbol
+
+
+# Used when a match is two-sided (e.g. international~intl), to modify the importance of the match
+# in the context of a set of matches.
+SYM_WEIGHTS = {
+    Symbol.Category.ORG_CLASS: 0.7,
+    Symbol.Category.INITIAL: 0.5,
+    Symbol.Category.NICK: 0.8,
+    # in "A B International" and "X International", we don't want to give too much weight to the symbol
+    Symbol.Category.SYMBOL: 0.3,
+    # Vessel 1 vs. Vessel 2 are very different.
+    Symbol.Category.NUMERIC: 1.3,
+    Symbol.Category.LOCATION: 0.8,
+}
+
+# Used when a match is one-sided (e.g. "international" in the query but not the result), to modify
+# the impact of the extra name part on the score.
+# For the categories not listed here, we give a weight of 1.0 (see weight_extra_match below)
+EXTRAS_WEIGHTS = {
+    # Siemens AG vs. Siemens, sometimes the org class is omitted
+    Symbol.Category.ORG_CLASS: 0.7,
+    Symbol.Category.SYMBOL: 0.7,
+    # PE Fund 1 vs. PE Fund, often investments funds are numbered and that's quite important
+    Symbol.Category.NUMERIC: 1.3,
+    # Siemens Russia vs. Siemens: we don't care that much because in a local context,
+    # it's common to omit the suffix of the local subsidiary.
+    Symbol.Category.LOCATION: 0.8,
+}
+
+SYM_SCORES = {
+    Symbol.Category.ORG_CLASS: 0.8,
+    Symbol.Category.INITIAL: 0.9,
+    Symbol.Category.NAME: 0.9,
+    Symbol.Category.NICK: 0.6,
+    Symbol.Category.SYMBOL: 0.9,
+    Symbol.Category.NUMERIC: 0.9,
+    Symbol.Category.LOCATION: 0.9,
+}
+
+
+def weight_extra_match(parts: List[NamePart], name: Name) -> float:
+    """Apply a weight to a name part which remained unmatched in the system, as a function
+    of a user-supplied penalty, symbol weights, and some overrides."""
+    if len(parts) == 1 and is_stopword(parts[0].form):
+        return 0.5
+    sparts = hash(tuple(parts))
+    weight = 1.0
+    categories = set()
+    for span in name.spans:
+        if span.symbol.category == Symbol.Category.NUMERIC:
+            part = span.parts[0]
+            if len(span.parts) == 1 and not part.numeric and len(part.comparable) < 2:
+                continue
+        if sparts == hash(tuple(span.parts)):
+            categories.add(span.symbol.category)
+            weight = weight * EXTRAS_WEIGHTS.get(span.symbol.category, 1.0)
+    return weight

nomenklatura/matching/logic_v2/names/match.py

@@ -0,0 +1,195 @@
+from typing import Dict, List, Set
+from rigour.names import NameTypeTag, Name, NamePart, Span, Symbol
+from rigour.names import align_person_name_order, normalize_name
+from rigour.names import remove_obj_prefixes
+from followthemoney.proxy import E, EntityProxy
+from followthemoney import model
+from followthemoney.types import registry
+from followthemoney.names import schema_type_tag
+
+from nomenklatura.matching.logic_v2.names.analysis import entity_names
+from nomenklatura.matching.logic_v2.names.magic import weight_extra_match
+from nomenklatura.matching.logic_v2.names.pairing import Pairing
+from nomenklatura.matching.logic_v2.names.distance import weighted_edit_similarity
+from nomenklatura.matching.logic_v2.names.distance import strict_levenshtein
+from nomenklatura.matching.logic_v2.names.util import Match, numbers_mismatch
+from nomenklatura.matching.types import FtResult, ScoringConfig
+
+
+# Step 1: Generate all Matches based on symbols
+# Step 2: Generate the most highly-scored sequences of matches
+# Step 3: Pick the best sequence
+
+
+def match_name_symbolic(query: Name, result: Name, config: ScoringConfig) -> FtResult:
+    # Stage 1: We create a set of pairings between the symbols that have been annotated as spans
+    # on both names. This will try to determine the maximum, non-overlapping set of name
+    # parts that can be explained using pre-defined symbols.
+    query_symbols: Set[Symbol] = set(span.symbol for span in query.spans)
+    pairings = [Pairing.empty()]
+    result_map: Dict[Symbol, List[Span]] = {}
+    for span in result.spans:
+        if span.symbol not in query_symbols:
+            continue
+        if span.symbol not in result_map:
+            result_map[span.symbol] = []
+        result_map[span.symbol].append(span)
+    seen: Set[int] = set()
+    for part in query.parts:
+        next_pairings: List[Pairing] = []
+        for qspan in query.spans:
+            if qspan.symbol not in result_map:
+                continue
+            if part not in qspan.parts:
+                continue
+            for rspan in result_map.get(qspan.symbol, []):
+                # This assumes that these are the only factors for weighting the
+                # resulting match:
+                key = hash((qspan.parts, rspan.parts, qspan.symbol.category))
+                if key in seen:
+                    continue
+                for pairing in pairings:
+                    if pairing.can_pair(qspan, rspan):
+                        seen.add(key)
+                        next_pairing = pairing.add(qspan, rspan)
+                        next_pairings.append(next_pairing)
+        if len(next_pairings):
+            pairings = next_pairings
+
+    # Stage 2: We compute the score for each pairing, which is a combination of the
+    # symbolic match (some types of symbols are considered less strong matches than others) and
+    # the fuzzy match of the remaining name parts. Special scoring is also applied for extra
+    # name parts that are not matched to the other name during name alignment.
+    extra_query_weight = config.get_float("nm_extra_query_name")
+    extra_result_weight = config.get_float("nm_extra_result_name")
+    family_name_weight = config.get_float("nm_family_name_weight")
+    retval = FtResult(score=0.0, detail=None)
+    for pairing in pairings:
+        matches: List[Match] = pairing.matches
+
+        # Name parts that have not been tagged with a symbol:
+        query_rem = [part for part in query.parts if part not in pairing.query_used]
+        result_rem = [part for part in result.parts if part not in pairing.result_used]
+
+        if len(query_rem) > 0 or len(result_rem) > 0:
+            if query.tag == NameTypeTag.PER:
+                query_rem, result_rem = align_person_name_order(query_rem, result_rem)
+            else:
+                query_rem = NamePart.tag_sort(query_rem)
+                result_rem = NamePart.tag_sort(result_rem)
+
+            matches.extend(weighted_edit_similarity(query_rem, result_rem, config))
+
+        # Apply additional weight and score normalisation to the generated matches based
+        # on contextual clues.
+        for match in matches:
+            # Matches with one side empty, i.e. unmatched parts
+            # unmatched result part
+            if len(match.qps) == 0:
+                bias = weight_extra_match(match.rps, result)
+                match.weight = extra_result_weight * bias
+            # unmatched query part
+            elif len(match.rps) == 0:
+                bias = weight_extra_match(match.qps, query)
+                match.weight = extra_query_weight * bias
+            # We fall through here to apply the family-name boost to unmatched parts too.
+
+            # We have types of symbol matches and where we never score 1.0, but for
+            # literal matches, we always want to score 1.0
+            if match.score < 1.0 and match.qstr == match.rstr:
+                match.score = 1.0
+            # We treat family names matches as more important (but configurable) because
+            # they're just globally less murky and changeable than given names.
+            if match.is_family_name():
+                match.weight *= family_name_weight
+
+
+        # Sum up and average all the weights to get the final score for this pairing.
+        # score = sum(weights) / len(weights) if len(weights) > 0 else 0.0
+        total_weight = sum(match.weight for match in matches)
+        total_score = sum(match.weighted_score for match in matches)
+        score = total_score / total_weight if total_weight > 0 else 0.0
+        if score > retval.score:
+            detail = " ".join(str(m) for m in matches)
+            retval = FtResult(score=score, detail=detail)
+    if retval.detail is None:
+        retval.detail = f"{query.comparable!r}≉{result.comparable!r}"
+    return retval
+
+
+def _get_object_names(entity: EntityProxy) -> Set[str]:
+    """Get the names of an object entity, such as a vessel or asset."""
+    names = entity.get_type_values(registry.name, matchable=True)
+    if not names:
+        return set()
+    normalized = [normalize_name(name) for name in names]
+    return set([n for n in normalized if n is not None])
+
+
+def match_object_names(query: E, result: E, config: ScoringConfig) -> FtResult:
+    """Match the names of two objects, such as vessels or assets."""
+    result_names = _get_object_names(result)
+    mismatch_penalty = 1 - config.get_float("nm_number_mismatch")
+    best_result = FtResult(score=0.0, detail=None)
+    for query_name in _get_object_names(query):
+        query_name = remove_obj_prefixes(query_name)
+        for result_name in result_names:
+            result_name = remove_obj_prefixes(result_name)
+            score = strict_levenshtein(query_name, result_name, max_rate=5)
+            if score == 1.0:
+                detail = f"[{result_name!r} literalMatch]"
+            else:
+                detail = f"[{query_name!r}≈{result_name!r}, fuzzyMatch: {score:.2f}]"
+            if numbers_mismatch(query_name, result_name):
+                score = score * mismatch_penalty
+                detail = "Number mismatch"
+            if score > best_result.score:
+                best_result = FtResult(score=score, detail=detail)
+    return best_result
+
+
+def name_match(query: E, result: E, config: ScoringConfig) -> FtResult:
+    """Match two entities by analyzing and comparing their names."""
+    schema = model.common_schema(query.schema, result.schema)
+    type_tag = schema_type_tag(schema)
+    best = FtResult(score=0.0, detail=None)
+    if type_tag == NameTypeTag.UNK:
+        # Name matching is not supported for entities that are not listed
+        # as a person, organization, or a thing.
+        best.detail = "Unsuited for name matching: %s" % schema.name
+        return best
+    if type_tag == NameTypeTag.OBJ:
+        return match_object_names(query, result, config)
+    query_names = entity_names(type_tag, query, is_query=True)
+    result_names = entity_names(type_tag, result)
+
+    # For literal matches, return early instead of performing all the magic. This addresses
+    # a user surprise where literal matches can score below 1.0 after name de-duplication has
+    # only left a superset name on one side.
+    query_comparable = {name.comparable: name for name in query_names}
+    result_comparable = {name.comparable: name for name in result_names}
+    common = set(query_comparable).intersection(result_comparable)
+    if len(common) > 0:
+        longest = max(common, key=len)
+        match = Match(
+            qps=query_comparable[longest].parts,
+            rps=result_comparable[longest].parts,
+            score=1.0,
+        )
+        return FtResult(score=match.score, detail=str(match))
+
+    # Remove short names that are contained in longer names.
+    # This prevents a scenario where a short version of a name ("John
+    # Smith") is matched to a query ("John K Smith"), where a longer version
+    # ("John K Smith" != "John R Smith") would have disqualified the match.
+    query_names = Name.consolidate_names(query_names)
+    result_names = Name.consolidate_names(result_names)
+
+    for query_name in query_names:
+        for result_name in result_names:
+            ftres = match_name_symbolic(query_name, result_name, config)
+            if ftres.score >= best.score:
+                best = ftres
+    if best.detail is None:
+        best.detail = "No names available for matching"
+    return best

nomenklatura/matching/logic_v2/names/pairing.py

@@ -0,0 +1,81 @@
+from typing import List, Set
+
+from rigour.names import NamePart, Symbol, Span
+
+from nomenklatura.matching.logic_v2.names.magic import SYM_SCORES, SYM_WEIGHTS
+from nomenklatura.matching.logic_v2.names.util import Match
+
+
+class Pairing:
+    __slots__ = [
+        "query_used",
+        "result_used",
+        "matches",
+        "_hash",
+    ]
+
+    def __init__(
+        self,
+        query_used: Set[NamePart],
+        result_used: Set[NamePart],
+        matches: List[Match],
+    ) -> None:
+        self.query_used = query_used
+        self.result_used = result_used
+        self.matches = matches
+
+    @classmethod
+    def empty(cls) -> "Pairing":
+        """Create a new pairing with no matches."""
+        return cls(
+            query_used=set(),
+            result_used=set(),
+            matches=[],
+        )
+
+    def can_pair(self, query_span: Span, result_span: Span) -> bool:
+        """Check if two spans can be paired."""
+        if self.query_used.intersection(query_span.parts):
+            return False
+        if self.result_used.intersection(result_span.parts):
+            return False
+
+        # If the text is actually identical, we do not need to establish
+        # a pairing, as it is already a match.
+        # revised: This doesn't work because it knocks out the stopword-like functionality of
+        # symbolic matching.
+        # if query_span.form == result_span.form:
+        #     return False
+
+        # Check if one at least of the two span parts is a name initial
+        if query_span.symbol.category == Symbol.Category.INITIAL:
+            if len(query_span.parts[0]) > 1 and len(result_span.parts[0]) > 1:
+                return False
+
+        if query_span.symbol.category in (Symbol.Category.NAME, Symbol.Category.NICK):
+            # This may not be correct for many tokens since it's expected that the bulk
+            # of filtered items will be single-part span.
+            for qp, rp in zip(query_span.parts, result_span.parts):
+                if not qp.can_match(rp):
+                    return False
+
+        return True
+
+    def add(self, query_span: Span, result_span: Span) -> "Pairing":
+        """Add a pair of spans to the pairing."""
+        symbol = query_span.symbol
+        # Some types of symbols effectively also work as soft stopwords, reducing the relevance
+        # of the match. For example, "Ltd." in an organization name is not as informative as a
+        # person's first name. That's why we're assigning a low weight, even for literal matches.
+        match = Match(
+            qps=query_span.parts,
+            rps=result_span.parts,
+            symbol=query_span.symbol,
+            score=SYM_SCORES.get(symbol.category, 1.0),
+            weight=SYM_WEIGHTS.get(symbol.category, 1.0),
+        )
+        return Pairing(
+            self.query_used.union(query_span.parts),
+            self.result_used.union(result_span.parts),
+            self.matches + [match],
+        )

nomenklatura/matching/logic_v2/names/util.py

@@ -0,0 +1,89 @@
+import re
+
+from typing import Optional, Any, Sequence
+from rigour.names import NamePart, Symbol, NamePartTag
+
+
+class Match:
+    """A Match combines query and result name parts, along with a score and weight. It is one
+    part of the matching result, which is eventually aggregated into a final score."""
+
+    __slots__ = ["qps", "rps", "symbol", "score", "weight"]
+
+    def __init__(
+        self,
+        qps: Sequence[NamePart] = [],
+        rps: Sequence[NamePart] = [],
+        symbol: Optional[Symbol] = None,
+        score: float = 0.0,
+        weight: float = 1.0,
+    ) -> None:
+        """Initialize the Match object with query and result parts."""
+        self.qps = list(qps)
+        self.rps = list(rps)
+        self.symbol: Optional[Symbol] = symbol
+        self.score = score
+        self.weight = weight
+
+    @property
+    def weighted_score(self) -> float:
+        """Calculate the weighted score."""
+        return self.score * self.weight
+
+    @property
+    def qstr(self) -> str:
+        """Get the query string representation."""
+        return " ".join([part.comparable for part in self.qps])
+
+    @property
+    def rstr(self) -> str:
+        """Get the result string representation."""
+        return " ".join([part.comparable for part in self.rps])
+
+    def is_family_name(self) -> bool:
+        """Check if the match represents a family name."""
+        for np in self.qps:
+            if np.tag == NamePartTag.FAMILY:
+                return True
+        for np in self.rps:
+            if np.tag == NamePartTag.FAMILY:
+                return True
+        return False
+
+    def __hash__(self) -> int:
+        """Hash the Match object based on query and result parts."""
+        return hash((self.symbol, tuple(self.qps), tuple(self.rps)))
+
+    def __eq__(self, other: Any) -> bool:
+        """Check equality of two Match objects based on query and result parts."""
+        return hash(self) == hash(other)
+
+    def __repr__(self) -> str:
+        """String representation of the Match object."""
+        return f"<Match({str(self)})>"
+
+    def __str__(self) -> str:
+        """String representation of the Match object for debugging."""
+        qps_str = self.qstr
+        rps_str = self.rstr
+        if self.symbol is not None:
+            explanation = f"{qps_str!r}≈{rps_str!r} symbolMatch {self.symbol}"
+        elif not len(qps_str):
+            explanation = f"{rps_str!r} extraResultPart"
+        elif not len(rps_str):
+            explanation = f"{qps_str!r} extraQueryPart"
+        elif qps_str == rps_str:
+            explanation = f"{rps_str!r} literalMatch"
+        else:
+            explanation = f"{qps_str!r}≈{rps_str!r} fuzzyMatch"
+        return f"[{explanation}: {self.score:.2f}, weight {self.weight:.2f}]"
+
+
+NUMERIC = re.compile(r"\d{1,}")
+
+
+def numbers_mismatch(query: str, result: str) -> bool:
+    """Check if the number of numerals in two names is different."""
+    query_nums = set(NUMERIC.findall(query))
+    result_nums = set(NUMERIC.findall(result))
+    return len(query_nums.difference(result_nums)) > 0

nomenklatura/matching/name_based/__init__.py

@@ -0,0 +1,4 @@
+from nomenklatura.matching.name_based.model import NameMatcher
+from nomenklatura.matching.name_based.model import NameQualifiedMatcher
+
+__all__ = ["NameMatcher", "NameQualifiedMatcher"]