nomenklatura-mpt 4.1.9__py3-none-any.whl

nomenklatura/matching/svm_v1/misc.py

@@ -0,0 +1,94 @@
+from followthemoney.proxy import E
+from followthemoney.types import registry
+
+from .util import tokenize_pair, compare_levenshtein
+from nomenklatura.matching.compare.util import has_overlap, extract_numbers, is_disjoint
+from nomenklatura.matching.util import props_pair, type_pair
+from nomenklatura.matching.util import max_in_sets, has_schema
+from nomenklatura.matching.compat import clean_name_ascii
+
+
+def birth_place(query: E, result: E) -> float:
+    """Same place of birth."""
+    lv, rv = tokenize_pair(props_pair(query, result, ["birthPlace"]))
+    tokens = min(len(lv), len(rv))
+    return float(len(lv.intersection(rv))) / float(max(2.0, tokens))
+
+
+def address_match(query: E, result: E) -> float:
+    """Text similarity between addresses."""
+    lv, rv = type_pair(query, result, registry.address)
+    lvn = [clean_name_ascii(v) for v in lv if clean_name_ascii(v) is not None]
+    rvn = [clean_name_ascii(v) for v in rv if clean_name_ascii(v) is not None]
+    return max_in_sets(lvn, rvn, compare_levenshtein)
+
+
+def address_numbers(query: E, result: E) -> float:
+    """Find if names contain numbers, score if the numbers are different."""
+    lv, rv = type_pair(query, result, registry.address)
+    lvn = extract_numbers(lv)
+    rvn = extract_numbers(rv)
+    common = len(lvn.intersection(rvn))
+    disjoint = len(lvn.difference(rvn))
+    return common - disjoint
+
+
+def phone_match(query: E, result: E) -> float:
+    """Matching phone numbers between the two entities."""
+    lv, rv = type_pair(query, result, registry.phone)
+    return 1.0 if has_overlap(lv, rv) else 0.0
+
+
+def email_match(query: E, result: E) -> float:
+    """Matching email addresses between the two entities."""
+    lv, rv = type_pair(query, result, registry.email)
+    return 1.0 if has_overlap(lv, rv) else 0.0
+
+
+def identifier_match(query: E, result: E) -> float:
+    """Matching identifiers (e.g. passports, national ID cards, registration or
+    tax numbers) between the two entities."""
+    if has_schema(query, result, "Organization"):
+        return 0.0
+    lv, rv = type_pair(query, result, registry.identifier)
+    return 1.0 if has_overlap(lv, rv) else 0.0
+
+
+def org_identifier_match(query: E, result: E) -> float:
+    """Matching identifiers (e.g. registration or tax numbers) between two
+    organizations or companies."""
+    if not has_schema(query, result, "Organization"):
+        return 0.0
+    lv, rv = type_pair(query, result, registry.identifier)
+    return 1.0 if has_overlap(lv, rv) else 0.0
+
+
+def gender_mismatch(query: E, result: E) -> float:
+    """Both entities have a different gender associated with them."""
+    qv, rv = props_pair(query, result, ["gender"])
+    return 1.0 if is_disjoint(qv, rv) else 0.0
+
+
+def country_mismatch(query: E, result: E) -> float:
+    """Both entities are linked to different countries."""
+    qv, rv = type_pair(query, result, registry.country)
+    return 1.0 if is_disjoint(qv, rv) else 0.0
+
+
+def schema_match(query: E, result: E) -> float:
+    """Both entities have the same schema."""
+    return 1.0 if query.schema == result.schema else 0.0
+
+
+def property_count_similarity(query: E, result: E) -> float:
+    """Similarity in the number of properties."""
+    query_props = len([p for p in query.itervalues()])
+    result_props = len([p for p in result.itervalues()])
+    
+    if query_props == 0 and result_props == 0:
+        return 1.0
+    
+    max_props = max(query_props, result_props)
+    min_props = min(query_props, result_props)
+    
+    return float(min_props) / float(max_props) if max_props > 0 else 0.0

nomenklatura/matching/svm_v1/model.py

@@ -0,0 +1,168 @@
+import pickle
+import numpy as np
+from typing import List, Dict, Tuple, cast, Optional
+from functools import lru_cache as cache
+from sklearn.pipeline import Pipeline  # type: ignore
+from sklearn.svm import SVC  # type: ignore
+from sklearn.preprocessing import StandardScaler  # type: ignore
+from followthemoney.proxy import E
+
+from .names import first_name_match
+from .names import family_name_match
+from .names import name_levenshtein, name_match
+from .names import name_token_overlap, name_numbers
+from .names import name_length_similarity
+from .misc import phone_match, email_match
+from .misc import address_match, address_numbers
+from .misc import identifier_match, birth_place
+from .misc import org_identifier_match
+from .misc import gender_mismatch
+from .misc import country_mismatch
+from .misc import schema_match, property_count_similarity
+from nomenklatura.matching.compare.dates import dob_matches, dob_year_matches
+from nomenklatura.matching.compare.dates import dob_year_disjoint
+from nomenklatura.matching.types import (
+    FeatureDocs,
+    FeatureDoc,
+    MatchingResult,
+    ScoringConfig,
+)
+from nomenklatura.matching.types import CompareFunction, FtResult
+from nomenklatura.matching.types import Encoded, ScoringAlgorithm
+from nomenklatura.matching.util import make_github_url
+from nomenklatura.util import DATA_PATH
+
+
+class SVMV1(ScoringAlgorithm):
+    """A Support Vector Machine-based matching algorithm with RBF kernel."""
+
+    NAME = "svm-v1"
+    MODEL_PATH = DATA_PATH.joinpath(f"{NAME}.pkl")
+    FEATURES: List[CompareFunction] = [
+        name_match,
+        name_token_overlap,
+        name_numbers,
+        name_levenshtein,
+        name_length_similarity,
+        phone_match,
+        email_match,
+        identifier_match,
+        dob_matches,
+        dob_year_matches,
+        FtResult.unwrap(dob_year_disjoint),
+        first_name_match,
+        family_name_match,
+        birth_place,
+        gender_mismatch,
+        country_mismatch,
+        org_identifier_match,
+        address_match,
+        address_numbers,
+        schema_match,
+        property_count_similarity,
+    ]
+
+    @classmethod
+    def save(cls, 
+             pipeline: Pipeline, 
+             coefficients: Dict[str, float],
+             kernel: str = "rbf",
+             support_vectors_count: Optional[int] = None) -> None:
+        """Store the SVM model after training."""
+        mdl = pickle.dumps({
+            "pipeline": pipeline,
+            "coefficients": coefficients,
+            "kernel": kernel,
+            "support_vectors_count": support_vectors_count
+        })
+        with open(cls.MODEL_PATH, "wb") as fh:
+            fh.write(mdl)
+        cls.load.cache_clear()
+
+    @classmethod
+    @cache
+    def load(cls) -> Tuple[Pipeline, Dict[str, float], str, Optional[int]]:
+        """Load a pre-trained SVM model for ad-hoc use."""
+        with open(cls.MODEL_PATH, "rb") as fh:
+            model_data = pickle.loads(fh.read())
+        
+        pipeline = cast(Pipeline, model_data["pipeline"])
+        coefficients = cast(Dict[str, float], model_data["coefficients"])
+        kernel = cast(str, model_data.get("kernel", "rbf"))
+        support_vectors_count = cast(Optional[int], model_data.get("support_vectors_count"))
+        
+        current = [f.__name__ for f in cls.FEATURES]
+        if list(coefficients.keys()) != current:
+            raise RuntimeError("Model was not trained on identical features!")
+        
+        return pipeline, coefficients, kernel, support_vectors_count
+
+    @classmethod
+    def get_feature_docs(cls) -> FeatureDocs:
+        """Return an explanation of the features and their coefficients."""
+        features: FeatureDocs = {}
+        _, coefficients, _, _ = cls.load()
+        for func in cls.FEATURES:
+            name = func.__name__
+            features[name] = FeatureDoc(
+                description=func.__doc__,
+                coefficient=float(coefficients.get(name, 0.0)),
+                url=make_github_url(func),
+            )
+        return features
+
+    @classmethod
+    def compare(cls, query: E, result: E, config: ScoringConfig) -> MatchingResult:
+        """Use an SVM model to compare two entities."""
+        pipeline, coefficients, kernel, _ = cls.load()
+        
+        encoded = cls.encode_pair(query, result)
+        npfeat = np.array([encoded])
+        
+        # Get probability predictions
+        if hasattr(pipeline.named_steps['svc'], 'predict_proba'):
+            pred_proba = pipeline.predict_proba(npfeat)
+            score = float(pred_proba[0][1])  # Positive class probability
+        else:
+            # Fallback to decision function if predict_proba is not available
+            decision = pipeline.decision_function(npfeat)
+            # Convert decision function to probability-like score
+            score = float(1.0 / (1.0 + np.exp(-decision[0])))  # Sigmoid transformation
+        
+        # Create explanations
+        explanations: Dict[str, FtResult] = {}
+        for feature, coeff in zip(cls.FEATURES, encoded):
+            name = feature.__name__
+            explanations[name] = FtResult(score=float(coeff), detail=None)
+        
+        # Add model-specific information
+        explanations["svm_kernel"] = FtResult(score=0.0, detail=kernel)
+        
+        return MatchingResult.make(score=score, explanations=explanations)
+
+    @classmethod
+    def encode_pair(cls, left: E, right: E) -> Encoded:
+        """Encode the comparison between two entities as a set of feature values."""
+        return [f(left, right) for f in cls.FEATURES]
+
+    @classmethod
+    def get_decision_function(cls, features: np.ndarray) -> np.ndarray:
+        """Get the SVM decision function values."""
+        pipeline, _, _, _ = cls.load()
+        return pipeline.decision_function(features)
+
+    @classmethod
+    def get_support_vectors_info(cls) -> Dict[str, any]:
+        """Get information about the support vectors."""
+        pipeline, _, kernel, sv_count = cls.load()
+        svm = pipeline.named_steps['svc']
+        
+        info = {
+            "kernel": kernel,
+            "support_vectors_count": sv_count or len(svm.support_vectors_) if hasattr(svm, 'support_vectors_') else None,
+            "n_support": svm.n_support_.tolist() if hasattr(svm, 'n_support_') else None,
+            "gamma": svm.gamma if hasattr(svm, 'gamma') else None,
+            "C": svm.C if hasattr(svm, 'C') else None,
+        }
+        
+        return info

nomenklatura/matching/svm_v1/names.py

@@ -0,0 +1,81 @@
+from typing import Iterable, Set
+from followthemoney.proxy import E
+from followthemoney.types import registry
+
+from .util import tokenize_pair, compare_levenshtein
+from nomenklatura.matching.compare.util import is_disjoint, has_overlap, extract_numbers
+from nomenklatura.matching.util import props_pair, type_pair
+from nomenklatura.matching.util import max_in_sets
+from nomenklatura.matching.compare.names import fingerprint_name
+
+
+def normalize_names(raws: Iterable[str]) -> Set[str]:
+    names = set()
+    for raw in raws:
+        name = fingerprint_name(raw)
+        if name is not None:
+            names.add(name[:128])
+    return names
+
+
+def name_levenshtein(left: E, right: E) -> float:
+    """Consider the edit distance (as a fraction of name length) between the two most
+    similar names linked to both entities."""
+    lv, rv = type_pair(left, right, registry.name)
+    lvn, rvn = normalize_names(lv), normalize_names(rv)
+    return max_in_sets(lvn, rvn, compare_levenshtein)
+
+
+def first_name_match(left: E, right: E) -> float:
+    """Matching first/given name between the two entities."""
+    lv, rv = tokenize_pair(props_pair(left, right, ["firstName"]))
+    return 1.0 if has_overlap(lv, rv) else 0.0
+
+
+def family_name_match(left: E, right: E) -> float:
+    """Matching family name between the two entities."""
+    lv, rv = tokenize_pair(props_pair(left, right, ["lastName"]))
+    return 1.0 if has_overlap(lv, rv) else 0.0
+
+
+def name_match(left: E, right: E) -> float:
+    """Check for exact name matches between the two entities."""
+    lv, rv = type_pair(left, right, registry.name)
+    lvn, rvn = normalize_names(lv), normalize_names(rv)
+    common = [len(n) for n in lvn.intersection(rvn)]
+    max_common = max(common, default=0)
+    if max_common == 0:
+        return 0.0
+    return float(max_common)
+
+
+def name_token_overlap(left: E, right: E) -> float:
+    """Evaluate the proportion of identical words in each name."""
+    lv, rv = tokenize_pair(type_pair(left, right, registry.name))
+    common = lv.intersection(rv)
+    tokens = min(len(lv), len(rv))
+    return float(len(common)) / float(max(2.0, tokens))
+
+
+def name_numbers(left: E, right: E) -> float:
+    """Find if names contain numbers, score if the numbers are different."""
+    lv, rv = type_pair(left, right, registry.name)
+    return 1.0 if is_disjoint(extract_numbers(lv), extract_numbers(rv)) else 0.0
+
+
+def name_length_similarity(left: E, right: E) -> float:
+    """Similarity in name lengths."""
+    lv, rv = type_pair(left, right, registry.name)
+    if not lv or not rv:
+        return 0.0
+    
+    max_left = max(len(name) for name in lv)
+    max_right = max(len(name) for name in rv)
+    
+    if max_left == 0 and max_right == 0:
+        return 1.0
+    
+    max_len = max(max_left, max_right)
+    min_len = min(max_left, max_right)
+    
+    return float(min_len) / float(max_len) if max_len > 0 else 0.0

nomenklatura/matching/svm_v1/train.py

@@ -0,0 +1,186 @@
+import logging
+import numpy as np
+import multiprocessing
+from typing import Iterable, List, Tuple, Optional
+from pprint import pprint
+from numpy.typing import NDArray
+from sklearn.pipeline import make_pipeline  # type: ignore
+from sklearn.preprocessing import StandardScaler  # type: ignore
+from sklearn.model_selection import train_test_split, GridSearchCV  # type: ignore
+from sklearn.svm import SVC  # type: ignore
+from sklearn import metrics # type: ignore
+from concurrent.futures import ProcessPoolExecutor
+from followthemoney.util import PathLike
+
+from nomenklatura.judgement import Judgement
+from nomenklatura.matching.pairs import read_pairs, JudgedPair
+from .model import SVMV1
+
+log = logging.getLogger(__name__)
+
+
+def pair_convert(pair: JudgedPair) -> Tuple[List[float], int]:
+    """Encode a pair of training data into features and target."""
+    judgement = 1 if pair.judgement == Judgement.POSITIVE else 0
+    features = SVMV1.encode_pair(pair.left, pair.right)
+    return features, judgement
+
+
+def pairs_to_arrays(
+    pairs: Iterable[JudgedPair],
+) -> Tuple[NDArray[np.float32], NDArray[np.float32]]:
+    """Parallelize feature computation for training data"""
+    xrows = []
+    yrows = []
+    workers = multiprocessing.cpu_count()
+    log.info("Using %d processes for feature computation...", workers)
+    with ProcessPoolExecutor(max_workers=workers) as excecutor:
+        results = excecutor.map(pair_convert, pairs)
+        for idx, (x, y) in enumerate(results):
+            if idx > 0 and idx % 10000 == 0:
+                log.info("Computing features: %s....", idx)
+            xrows.append(x)
+            yrows.append(y)
+
+    return np.array(xrows), np.array(yrows)
+
+
+def train_svm_matcher(
+    pairs_file: PathLike, 
+    kernel: str = "rbf",
+    optimize_hyperparameters: bool = True,
+    probability: bool = True
+) :
+    """Train an SVM matching model."""
+    pairs = []
+    for pair in read_pairs(pairs_file):
+        if pair.judgement == Judgement.UNSURE:
+            pair.judgement = Judgement.NEGATIVE
+        pairs.append(pair)
+    
+    positive = len([p for p in pairs if p.judgement == Judgement.POSITIVE])
+    negative = len([p for p in pairs if p.judgement == Judgement.NEGATIVE])
+    log.info("Total pairs loaded: %d (%d pos/%d neg)", len(pairs), positive, negative)
+    
+    X, y = pairs_to_arrays(pairs)
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
+    
+    log.info("Training SVM model with %s kernel...", kernel)
+    
+    # Create base SVM
+    if optimize_hyperparameters:
+        log.info("Optimizing hyperparameters with Grid Search...")
+        
+        # Define parameter grid
+        if kernel == "rbf":
+            param_grid = {
+                'svc__C': [0.1, 1, 10, 100],
+                'svc__gamma': ['scale', 'auto', 0.001, 0.01, 0.1, 1]
+            }
+        elif kernel == "linear":
+            param_grid = {
+                'svc__C': [0.1, 1, 10, 100]
+            }
+        elif kernel == "poly":
+            param_grid = {
+                'svc__C': [0.1, 1, 10, 100],
+                'svc__degree': [2, 3, 4],
+                'svc__gamma': ['scale', 'auto', 0.001, 0.01, 0.1]
+            }
+        else:
+            param_grid = {'svc__C': [0.1, 1, 10, 100]}
+        
+        # Create pipeline
+        base_svm = SVC(kernel=kernel, probability=probability, random_state=42)
+        pipeline = make_pipeline(StandardScaler(), base_svm)
+        
+        # Grid search
+        grid_search = GridSearchCV(
+            pipeline, 
+            param_grid, 
+            cv=5, 
+            scoring='roc_auc',
+            n_jobs=-1,
+            verbose=1
+        )
+        
+        grid_search.fit(X_train, y_train)
+        pipeline = grid_search.best_estimator_
+        
+        log.info("Best parameters: %s", grid_search.best_params_)
+        log.info("Best cross-validation score: %.4f", grid_search.best_score_)
+        
+    else:
+        # Use default parameters
+        svm = SVC(kernel=kernel, probability=probability, random_state=42)
+        pipeline = make_pipeline(StandardScaler(), svm)
+        pipeline.fit(X_train, y_train)
+    
+    # Get SVM object for coefficient extraction
+    svm_model = pipeline.named_steps['svc']
+    
+    # Create feature coefficients (for linear kernel, use actual coefficients)
+    coefficients = {}
+    if kernel == "linear" and hasattr(svm_model, 'coef_'):
+        # For linear SVM, we have actual feature coefficients
+        coef_values = svm_model.coef_[0]
+        for i, feature in enumerate(SVMV1.FEATURES):
+            coefficients[feature.__name__] = float(coef_values[i])
+    else:
+        # For non-linear kernels, use feature importance approximation
+        # This is a simplified approach - in practice, you might use SHAP or similar
+        for feature in SVMV1.FEATURES:
+            coefficients[feature.__name__] = 1.0  # Placeholder
+    
+    # Get support vector count
+    support_vectors_count = len(svm_model.support_vectors_) if hasattr(svm_model, 'support_vectors_') else None
+    
+    # Save the model
+    SVMV1.save(pipeline, coefficients, kernel, support_vectors_count)
+    path = SVMV1.MODEL_PATH
+    # Evaluation
+    log.info("Evaluating SVM model...")
+    
+    # Predictions
+    y_pred = pipeline.predict(X_test)
+    
+    if probability:
+        y_pred_proba = pipeline.predict_proba(X_test)[:, 1]
+    else:
+        # Use decision function and apply sigmoid
+        decision_scores = pipeline.decision_function(X_test)
+        y_pred_proba = 1.0 / (1.0 + np.exp(-decision_scores))
+    
+    # Print results
+    print("\nSVM Results:")
+    print(f"Kernel: {kernel}")
+    print(f"Support vectors count: {support_vectors_count}")
+    
+    if kernel == "linear":
+        print("Feature Coefficients:")
+        pprint(coefficients)
+    
+    cnf_matrix = metrics.confusion_matrix(y_test, y_pred)
+    print("Confusion matrix:\n", cnf_matrix)
+    print("Accuracy:", metrics.accuracy_score(y_test, y_pred))
+    print("Precision:", metrics.precision_score(y_test, y_pred))
+    print("Recall:", metrics.recall_score(y_test, y_pred))
+    print("F1-score:", metrics.f1_score(y_test, y_pred))
+    
+    auc = metrics.roc_auc_score(y_test, y_pred_proba)
+    print("Area under curve:", auc)
+    
+    # Additional SVM-specific metrics
+    if hasattr(svm_model, 'support_'):
+        print(f"Number of support vectors: {len(svm_model.support_)}")
+        print(f"Support vectors per class: {svm_model.n_support_}")
+    return SVMV1.MODEL_PATH
+
+
+def train_matcher(
+    pairs_file: PathLike, 
+    kernel: str = "rbf",
+    optimize: bool = True
+) -> None:
+    """Wrapper function to maintain compatibility with existing training interface."""
+    train_svm_matcher(pairs_file, kernel=kernel, optimize_hyperparameters=optimize)

nomenklatura/matching/svm_v1/util.py

@@ -0,0 +1,30 @@
+from normality.constants import WS
+from typing import Iterable, Set, Tuple
+from rigour.text.distance import levenshtein
+
+from nomenklatura.matching.compat import clean_name_ascii
+
+
+def tokenize(texts: Iterable[str]) -> Set[str]:
+    tokens: Set[str] = set()
+    for text in texts:
+        cleaned = clean_name_ascii(text)
+        if cleaned is None:
+            continue
+        for token in cleaned.split(WS):
+            token = token.strip()
+            if len(token) > 2:
+                tokens.add(token)
+    return tokens
+
+
+def tokenize_pair(
+    pair: Tuple[Iterable[str], Iterable[str]],
+) -> Tuple[Set[str], Set[str]]:
+    return tokenize(pair[0]), tokenize(pair[1])
+
+
+def compare_levenshtein(left: str, right: str) -> float:
+    distance = levenshtein(left, right)
+    base = max((1, len(left), len(right)))
+    return 1.0 - (distance / float(base))