nomenklatura-mpt 4.1.9__py3-none-any.whl

nomenklatura/matching/types.py

@@ -0,0 +1,227 @@
+from enum import Enum
+from pydantic import BaseModel
+from typing import List, Dict, Optional, Callable, Union, cast
+from followthemoney.proxy import E, EntityProxy
+
+from nomenklatura.matching.util import make_github_url, FNUL
+
+Encoded = List[float]
+CompareFunction = Callable[[EntityProxy, EntityProxy], float]
+FeatureCompareFunction = Callable[[EntityProxy, EntityProxy], "FtResult"]
+FeatureCompareConfigured = Callable[
+    [EntityProxy, EntityProxy, "ScoringConfig"], "FtResult"
+]
+
+
+class FeatureDoc(BaseModel):
+    """Documentation for a particular feature in the matching API model."""
+
+    description: Optional[str]
+    coefficient: float
+    url: str
+
+
+FeatureDocs = Dict[str, FeatureDoc]
+
+
+class ConfigVarType(str, Enum):
+    """The type of a configuration variable."""
+
+    INTEGER = "integer"
+    FLOAT = "float"
+    BOOLEAN = "boolean"
+
+
+class ConfigVar(BaseModel):
+    """A configuration variable for a scoring algorithm."""
+
+    type: ConfigVarType = ConfigVarType.FLOAT
+    description: Optional[str] = None
+    default: Union[str, int, float, bool] = 0
+
+
+class AlgorithmDocs(BaseModel):
+    """Documentation for a scoring algorithm."""
+
+    name: str
+    description: Optional[str] = None
+    config: Dict[str, ConfigVar]
+    features: FeatureDocs
+
+
+class FtResult(BaseModel):
+    """A explained score for a particular feature result."""
+
+    detail: Optional[str]
+    score: float
+
+    def empty(self) -> bool:
+        """Check if the result is empty."""
+        return self.detail is None and self.score == FNUL
+
+    @classmethod
+    def wrap(cls, func: CompareFunction) -> FeatureCompareFunction:
+        """Wrap a score and detail into a feature result."""
+
+        def wrapper(query: E, result: E) -> "FtResult":
+            return cls(score=func(query, result), detail=None)
+
+        wrapper.__name__ = func.__name__
+        wrapper.__doc__ = func.__doc__
+        return wrapper
+
+    @classmethod
+    def unwrap(cls, func: FeatureCompareConfigured) -> CompareFunction:
+        """Unwrap a feature result returned by a comparator into a score."""
+        config = ScoringConfig.defaults()
+
+        def wrapper(query: E, result: E) -> float:
+            return func(query, result, config).score
+
+        wrapper.__name__ = func.__name__
+        wrapper.__doc__ = func.__doc__
+        return wrapper
+
+    def __repr__(self) -> str:
+        """Return a string representation of the feature result."""
+        return f"<FtR({self.score}, {self.detail!r})>"
+
+
+class MatchingResult(BaseModel):
+    """Score and feature comparison results for matching comparison."""
+
+    score: float
+    features: Dict[str, float]
+    explanations: Dict[str, FtResult]
+
+    @classmethod
+    def make(cls, score: float, explanations: Dict[str, FtResult]) -> "MatchingResult":
+        """Create a new matching result."""
+        explanations = {k: v for k, v in explanations.items() if not v.empty()}
+        features = {k: v.score for k, v in explanations.items() if v.score != FNUL}
+        return cls(score=score, features=features, explanations=explanations)
+
+    def __repr__(self) -> str:
+        """Return a string representation of the matching result."""
+        return f"<MR({self.score}, expl={self.explanations})>"
+
+
+class ScoringConfig(BaseModel):
+    """Configuration for a scoring algorithm."""
+
+    weights: Dict[str, float]
+    config: Dict[str, Union[str, int, float, bool]]
+
+    @classmethod
+    def defaults(cls) -> "ScoringConfig":
+        """Return the default configuration."""
+        return cls(weights={}, config={})
+
+    def get_float(self, key: str) -> float:
+        """Get a float value from the configuration."""
+        return float(self.config[key])
+
+
+class ScoringAlgorithm(object):
+    """An implementation of a scoring system that compares two entities."""
+
+    NAME = "algorithm_name"
+    CONFIG: Dict[str, ConfigVar] = {}
+
+    @classmethod
+    def compare(cls, query: E, result: E, config: ScoringConfig) -> MatchingResult:
+        """Compare the two entities and return a score and feature comparison."""
+        raise NotImplementedError
+
+    @classmethod
+    def get_feature_docs(cls) -> FeatureDocs:
+        """Return an explanation of the features and their coefficients."""
+        raise NotImplementedError
+
+    @classmethod
+    def get_docs(cls) -> AlgorithmDocs:
+        """Return an explanation of the algorithm and its features."""
+        return AlgorithmDocs(
+            name=cls.NAME,
+            description=cls.__doc__,
+            config=cls.CONFIG,
+            features=cls.get_feature_docs(),
+        )
+
+    @classmethod
+    def default_config(cls) -> ScoringConfig:
+        """Return the default configuration for the algorithm."""
+        return ScoringConfig.defaults()
+
+
+class Feature(BaseModel):
+    func: Union[FeatureCompareFunction, FeatureCompareConfigured]
+    weight: float
+    qualifier: bool = False
+
+    @property
+    def name(self) -> str:
+        return self.func.__name__
+
+    @property
+    def doc(self) -> FeatureDoc:
+        description = self.func.__doc__
+        assert description is not None, self.func.__name__
+        return FeatureDoc(
+            description=description,
+            coefficient=self.weight,
+            url=make_github_url(self.func),
+        )
+
+    def invoke(self, query: E, result: E, config: ScoringConfig) -> FtResult:
+        """Invoke the feature function and return the result."""
+        if self.func.__code__.co_argcount == 3:
+            func = cast(FeatureCompareConfigured, self.func)
+            return func(query, result, config)
+        else:
+            func = cast(FeatureCompareFunction, self.func)  # type: ignore
+            return func(query, result)  # type: ignore
+
+
+class HeuristicAlgorithm(ScoringAlgorithm):
+    features: List[Feature]
+
+    @classmethod
+    def compute_score(
+        cls, scores: Dict[str, float], weights: Dict[str, float]
+    ) -> float:
+        raise NotImplementedError
+
+    @classmethod
+    def get_feature_docs(cls) -> FeatureDocs:
+        return {f.name: f.doc for f in cls.features}
+
+    @classmethod
+    def default_config(cls) -> ScoringConfig:
+        """Return the default configuration for the algorithm."""
+        config = ScoringConfig.defaults()
+        for name, var in cls.CONFIG.items():
+            config.config[name] = var.default
+        return config
+
+    @classmethod
+    def compare(cls, query: E, result: E, config: ScoringConfig) -> MatchingResult:
+        if not query.schema.can_match(result.schema):
+            if not query.schema.name == result.schema.name:
+                return MatchingResult.make(FNUL, {})
+
+        for name, var in cls.CONFIG.items():
+            if config.config.get(name) is None:
+                config.config[name] = var.default
+
+        explanations: Dict[str, FtResult] = {}
+        scores: Dict[str, float] = {}
+        weights: Dict[str, float] = {}
+        for feature in cls.features:
+            weights[feature.name] = config.weights.get(feature.name, feature.weight)
+            if weights[feature.name] != FNUL:
+                explanations[feature.name] = feature.invoke(query, result, config)
+                scores[feature.name] = explanations[feature.name].score
+        score = cls.compute_score(scores, weights)
+        score = min(1.0, max(FNUL, score))
+        return MatchingResult.make(score=score, explanations=explanations)

nomenklatura/matching/util.py

@@ -0,0 +1,62 @@
+from pathlib import Path
+from itertools import product
+from typing import List, Set, TypeVar, Tuple, Iterable, Optional, Callable, Any
+from followthemoney.proxy import E
+from followthemoney.types.common import PropertyType
+
+from nomenklatura import __version__
+from nomenklatura.util import DATA_PATH
+
+V = TypeVar("V")
+BASE_URL = "https://github.com/opensanctions/nomenklatura/blob/%s/nomenklatura/%s#L%s"
+CODE_PATH = DATA_PATH.joinpath("..").resolve()
+FNUL = 0.0
+
+
+def has_schema(left: E, right: E, schema: str) -> bool:
+    """Check if one of the entities has the required schema."""
+    if left.schema.is_a(schema) or right.schema.is_a(schema):
+        return True
+    return False
+
+
+def props_pair(left: E, right: E, props: List[str]) -> Tuple[Set[str], Set[str]]:
+    left_values: Set[str] = set()
+    right_values: Set[str] = set()
+    for prop in props:
+        left_values.update(left.get(prop, quiet=True))
+        right_values.update(right.get(prop, quiet=True))
+    return left_values, right_values
+
+
+def type_pair(left: E, right: E, type_: PropertyType) -> Tuple[List[str], List[str]]:
+    left_values = left.get_type_values(type_, matchable=True)
+    right_values = right.get_type_values(type_, matchable=True)
+    return left_values, right_values
+
+
+def max_in_sets(
+    left: Iterable[Optional[V]],
+    right: Iterable[Optional[V]],
+    compare_func: Callable[[V, V], float],
+    max_res: float = 1.0,
+) -> float:
+    """Compare two sets of values pair-wise and select the highest-scored result."""
+    res: float = 0.0
+    for le, ri in product(left, right):
+        if le is None or ri is None:
+            continue
+        v = compare_func(le, ri)
+        if v <= res:
+            continue
+        res = v
+        if res >= max_res:
+            return res
+    return res
+
+
+def make_github_url(func: Callable[..., Any]) -> str:
+    """Make a URL to the source code of a matching function."""
+    code_path = Path(func.__code__.co_filename).relative_to(CODE_PATH)
+    line_no = func.__code__.co_firstlineno
+    return BASE_URL % (__version__, code_path, line_no)

nomenklatura/publish/dates.py

@@ -0,0 +1,49 @@
+from functools import cache
+from typing import Iterable, List, Tuple
+from followthemoney import registry, SE
+
+
+PROV_MIN_DATES = ("createdAt", "authoredAt", "publishedAt")
+PROV_MAX_DATES = ("modifiedAt", "retrievedAt")
+
+
+def simplify_dates(entity: SE) -> SE:
+    """If an entity has multiple values for a date field, you may
+    want to remove all those that are prefixes of others. For example,
+    if a Person has both a birthDate of 1990 and of 1990-05-01, we'd
+    want to drop the mention of 1990."""
+    for prop in entity.iterprops():
+        if prop.type == registry.date:
+            # This is super unrolled in order to make it fast, its called
+            # a lot during data exports. We shouldn't re-use this function
+            # code in less perf critical contexts.
+            stmts = entity._statements[prop.name]
+            if len(stmts) < 2:
+                continue
+            values_in = tuple({s.value for s in stmts})
+            if len(values_in) < 2:
+                continue
+            values = remove_prefix_date_values(values_in)
+            if prop.name in PROV_MAX_DATES:
+                values = (max(values),)
+            elif prop.name in PROV_MIN_DATES:
+                values = (min(values),)
+
+            for stmt in list(stmts):
+                if stmt.value not in values:
+                    entity._statements[prop.name].remove(stmt)
+    return entity
+
+
+@cache
+def remove_prefix_date_values(values: Tuple[str]) -> Iterable[str]:
+    """See ``remove_prefix_dates``."""
+    kept: List[str] = []
+    values_list = sorted(values, reverse=True)
+    for index, value in enumerate(values_list):
+        if index > 0:
+            longer = values_list[index - 1]
+            if longer.startswith(value):
+                continue
+        kept.append(value)
+    return kept

nomenklatura/publish/edges.py

@@ -0,0 +1,32 @@
+from followthemoney import SE
+from nomenklatura.resolver import Identifier
+
+
+def simplify_undirected(entity: SE) -> SE:
+    """Simplify undirected edges by removing duplicate entity IDs on both
+    ends."""
+    # Problem: undirected relationships in which both
+    # entities are given as the source AND the target
+    if (
+        not entity.schema.edge
+        or entity.schema.edge_directed
+        or not entity.schema.edge_source
+        or not entity.schema.edge_target
+    ):
+        return entity
+    sources = entity.get_statements(entity.schema.edge_source)
+    targets = entity.get_statements(entity.schema.edge_target)
+    source_ids = set((s.value for s in sources))
+    target_ids = set((t.value for t in targets))
+    common = source_ids.intersection(target_ids)
+    if len(common) != 2:
+        return entity
+    identifiers = [Identifier.get(s) for s in common]
+    source_id, target_id = max(identifiers), min(identifiers)
+    for stmt in sources:
+        if stmt.value == target_id:
+            entity._statements[entity.schema.edge_source].remove(stmt)
+    for stmt in targets:
+        if stmt.value == source_id:
+            entity._statements[entity.schema.edge_target].remove(stmt)
+    return entity

nomenklatura/resolver/__init__.py

@@ -0,0 +1,6 @@
+from nomenklatura.resolver.identifier import Identifier, StrIdent, Pair
+from nomenklatura.resolver.edge import Edge
+from nomenklatura.resolver.linker import Linker
+from nomenklatura.resolver.resolver import Resolver
+
+__all__ = ["Identifier", "StrIdent", "Pair", "Edge", "Linker", "Resolver"]

nomenklatura/resolver/common.py

@@ -0,0 +1,2 @@
+class ResolverLogicError(Exception):
+    pass

nomenklatura/resolver/edge.py

@@ -0,0 +1,107 @@
+import json
+from typing import Any, Dict, Optional, Union
+
+from sqlalchemy.engine import RowMapping
+
+from nomenklatura.judgement import Judgement
+from nomenklatura.resolver.identifier import Identifier, StrIdent
+
+
+class Edge(object):
+    __slots__ = (
+        "key",
+        "source",
+        "target",
+        "judgement",
+        "score",
+        "user",
+        "created_at",
+        "deleted_at",
+    )
+
+    def __init__(
+        self,
+        left_id: StrIdent,
+        right_id: StrIdent,
+        judgement: Judgement = Judgement.NO_JUDGEMENT,
+        score: Optional[float] = None,
+        user: Optional[str] = None,
+        created_at: Optional[str] = None,
+        deleted_at: Optional[str] = None,
+    ):
+        self.key = Identifier.pair(left_id, right_id)
+        self.target, self.source = self.key
+        self.judgement = judgement
+        self.score = score
+        self.user = user
+        self.created_at = created_at
+        self.deleted_at = deleted_at
+
+    def other(self, cur: Identifier) -> Identifier:
+        if cur == self.target:
+            return self.source
+        return self.target
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "target": self.target.id,
+            "source": self.source.id,
+            "judgement": self.judgement.value,
+            "score": self.score,
+            "user": self.user,
+            "created_at": self.created_at,
+            "deleted_at": self.deleted_at,
+        }
+
+    def to_line(self) -> str:
+        row = [
+            self.target.id,
+            self.source.id,
+            self.judgement.value,
+            self.score,
+            self.user,
+            self.created_at,
+        ]
+        return json.dumps(row) + "\n"
+
+    def __str__(self) -> str:
+        return self.to_line()
+
+    def __hash__(self) -> int:
+        return hash(self.key)
+
+    def __eq__(self, other: Any) -> bool:
+        return hash(self) == hash(other)
+
+    def __lt__(self, other: Any) -> bool:
+        return bool(self.key < other.key)
+
+    def __repr__(self) -> str:
+        return f"<E({self.target.id}, {self.source.id}, {self.judgement.value})>"
+
+    @classmethod
+    def from_line(cls, line: str) -> "Edge":
+        data = json.loads(line)
+        edge = cls(
+            data[0],
+            data[1],
+            judgement=Judgement(data[2]),
+            score=data[3],
+            user=data[4],
+            created_at=data[5],
+        )
+        if len(data) > 6:
+            edge.deleted_at = data[6]
+        return edge
+
+    @classmethod
+    def from_dict(cls, data: Union[RowMapping, Dict[str, Any]]) -> "Edge":
+        return cls(
+            left_id=data["target"],
+            right_id=data["source"],
+            judgement=Judgement(data["judgement"]),
+            score=data["score"],
+            user=data["user"],
+            created_at=data.get("created_at"),
+            deleted_at=data.get("deleted_at"),
+        )

nomenklatura/resolver/identifier.py

@@ -0,0 +1,60 @@
+import shortuuid
+from typing import Any, Optional, Tuple, Union
+from rigour.ids.wikidata import is_qid
+
+from nomenklatura.resolver.common import ResolverLogicError
+
+StrIdent = Union[str, "Identifier"]
+Pair = Tuple["Identifier", "Identifier"]
+
+
+class Identifier(object):
+    PREFIX = "NK-"
+
+    __slots__ = ("id", "canonical", "weight")
+
+    def __init__(self, id: str):
+        self.id = id
+        self.weight: int = 1
+        if self.id.startswith(self.PREFIX):
+            self.weight = 2
+        elif is_qid(id):
+            self.weight = 3
+        self.canonical = self.weight > 1
+
+    def __eq__(self, other: Any) -> bool:
+        return self.id == str(other)
+
+    def __lt__(self, other: Any) -> bool:
+        return (self.weight, self.id) < (other.weight, other.id)
+
+    def __str__(self) -> str:
+        return self.id
+
+    def __hash__(self) -> int:
+        return hash(self.id)
+
+    def __len__(self) -> int:
+        return len(self.id)
+
+    def __repr__(self) -> str:
+        return f"<I({self.id})>"
+
+    @classmethod
+    def get(cls, id: StrIdent) -> "Identifier":
+        if isinstance(id, str):
+            return cls(id)
+        return id
+
+    @classmethod
+    def pair(cls, left_id: StrIdent, right_id: StrIdent) -> Pair:
+        left = cls.get(left_id)
+        right = cls.get(right_id)
+        if left == right:
+            raise ResolverLogicError("%s/%s" % (left, right))
+        return (max(left, right), min(left, right))
+
+    @classmethod
+    def make(cls, value: Optional[str] = None) -> "Identifier":
+        key = value or shortuuid.uuid()
+        return cls.get(f"{cls.PREFIX}{key}")

nomenklatura/resolver/linker.py

@@ -0,0 +1,101 @@
+from functools import lru_cache
+from typing import Dict, Generator, Generic, Set
+from followthemoney import registry, ValueEntity, Statement, SE
+
+from nomenklatura.resolver.identifier import Identifier, StrIdent
+
+
+class Linker(Generic[SE]):
+    """A class to manage the canonicalisation of entities. This stores only the positive
+    merges of entities and is used as a lightweight way to apply the harmonisation
+    post de-duplication."""
+
+    def __init__(self, entities: Dict[Identifier, Set[Identifier]] = {}) -> None:
+        """
+        Args:
+            entities: an entry for each entity with its connected set of entities.
+        """
+        self._entities: Dict[Identifier, Set[Identifier]] = entities
+
+    def connected(self, node: Identifier) -> Set[Identifier]:
+        return self._entities.get(node, set([node]))
+
+    @lru_cache(maxsize=1024)
+    def get_canonical(self, entity_id: StrIdent) -> str:
+        """Return the canonical identifier for the given entity ID."""
+        node = Identifier.get(entity_id)
+        best = max(self.connected(node))
+        if best.canonical:
+            return best.id
+        return node.id
+
+    def canonicals(self) -> Generator[Identifier, None, None]:
+        """Return all the canonical cluster identifiers."""
+        for node in self._entities.keys():
+            if not node.canonical:
+                continue
+            canonical = self.get_canonical(node)
+            if canonical == node.id:
+                yield node
+
+    def get_referents(
+        self, canonical_id: StrIdent, canonicals: bool = True
+    ) -> Set[str]:
+        """Get all the non-canonical entity identifiers which refer to a given
+        canonical identifier."""
+        node = Identifier.get(canonical_id)
+        referents: Set[str] = set()
+        for connected in self.connected(node):
+            if not canonicals and connected.canonical:
+                continue
+            if connected == node:
+                continue
+            referents.add(connected.id)
+        return referents
+
+    def apply(self, proxy: SE) -> SE:
+        """Replace all entity references in a given proxy with their canonical
+        identifiers. This is essentially the harmonisation post de-dupe."""
+        if proxy.id is None:
+            return proxy
+        proxy.id = self.get_canonical(proxy.id)
+        return self.apply_properties(proxy)
+
+    def apply_stream(self, proxy: ValueEntity) -> ValueEntity:
+        if proxy.id is None:
+            return proxy
+        proxy.id = self.get_canonical(proxy.id)
+        for prop in proxy.iterprops():
+            if prop.type == registry.entity:
+                values = proxy.pop(prop)
+                for value in values:
+                    proxy.unsafe_add(prop, self.get_canonical(value), cleaned=True)
+        return proxy
+
+    def apply_properties(self, proxy: SE) -> SE:
+        for stmt in proxy._iter_stmt():
+            if proxy.id is not None:
+                stmt.canonical_id = proxy.id
+            if stmt.prop_type == registry.entity.name:
+                canon_value = self.get_canonical(stmt.value)
+                if canon_value != stmt.value:
+                    if stmt.original_value is None:
+                        stmt.original_value = stmt.value
+                    # NOTE: this means the key is out of whack here now
+                    stmt.value = canon_value
+        return proxy
+
+    def apply_statement(self, stmt: Statement) -> Statement:
+        if stmt.entity_id is not None:
+            stmt.canonical_id = self.get_canonical(stmt.entity_id)
+        if stmt.prop_type == registry.entity.name:
+            canon_value = self.get_canonical(stmt.value)
+            if canon_value != stmt.value:
+                if stmt.original_value is None:
+                    stmt.original_value = stmt.value
+                # NOTE: this means the key is out of whack here now
+                stmt.value = canon_value
+        return stmt
+
+    def __repr__(self) -> str:
+        return f"<Merger({len(self._entities)})>"