followthemoney 1.3.6__py3-none-any.whl → 3.8.0__py3-none-any.whl

followthemoney/compare.py

@@ -1,64 +1,141 @@
+import math
 import itertools
-from Levenshtein import jaro
+from typing import Dict, Generator, Iterable, List, Optional
+import fingerprints
 from normality import normalize
-from followthemoney.types import registry
-from followthemoney.util import dampen
 from followthemoney.exc import InvalidData
+from followthemoney.model import Model
+from followthemoney.types import registry
+from followthemoney.proxy import EntityProxy
+from followthemoney.types.common import PropertyType
 
-# OK, Here's the plan: we have to find a way to get user judgements
-# on as many of these matches as we can, then build a regression
-# model which properly weights the value of a matching property
-# based upon it's type.
-FP_WEIGHT = 0.6
-MATCH_WEIGHTS = {
-    registry.text: 0,
-    registry.name: 0,  # because we already compare names
-    registry.identifier: 0.4,
-    registry.url: 0.1,
-    registry.email: 0.3,
-    registry.ip: 0.1,
-    registry.iban: 0.3,
-    registry.address: 0.2,
-    registry.date: 0.3,
-    registry.phone: 0.1,
-    registry.country: 0.1,
-    registry.language: 0.1,
+
+# Compare weights come from the glm-bernouli model in followthemoney-predict
+Weights = Dict[Optional[PropertyType], float]
+Scores = Dict[PropertyType, Optional[float]]
+COMPARE_WEIGHTS: Weights = {
+    registry.name: 12.275729155073371,
+    registry.country: 1.0494517476987815,
+    registry.date: 6.960245940274218,
+    registry.identifier: 5.2209896558064175,
+    registry.address: 6.456137299747168,
+    registry.phone: 3.538892687331418,
+    registry.email: 14.115925628770384,
+    registry.iban: 0.019140301711998726,
+    registry.url: 3.211995327345834,
+    None: -11.91521189545115,
 }
 
 
-def compare(model, left, right):
-    """Compare two entities and return number between 0 and 1.
-    Returned number indicates probability that two entities are the same.
-    """
-    left = model.get_proxy(left)
-    right = model.get_proxy(right)
-    if right.schema not in list(left.schema.matchable_schemata):
-        return 0
-    schema = model.common_schema(left.schema, right.schema)
-    score = compare_names(left, right) * FP_WEIGHT
-    for name, prop in schema.properties.items():
-        weight = MATCH_WEIGHTS.get(prop.type, 0)
-        if weight == 0:
-            continue
+def compare_scores(model: Model, left: EntityProxy, right: EntityProxy) -> Scores:
+    """Compare two entities and return a match score for each property."""
+    try:
+        model.common_schema(left.schema, right.schema)
+    except InvalidData:
+        return {}
+    scores: Scores = {}
+    left_inv = left.get_type_inverted(matchable=True)
+    right_inv = right.get_type_inverted(matchable=True)
+    left_groups = set(left_inv.keys())
+    right_groups = set(right_inv.keys())
+    for group_name in left_groups.intersection(right_groups):
+        group = registry.groups[group_name]
         try:
-            left_values = left.get(name)
-            right_values = right.get(name)
-        except InvalidData:
-            continue
-
-        if not len(left_values) or not len(right_values):
-            continue
-        prop_score = prop.type.compare_sets(left_values, right_values)
-        score += (prop_score * weight)
-    return max(0.0, min(1.0, score)) * 0.9
-
-
-def compare_names(left, right):
-    result = 0
-    left_list = [normalize(n, latinize=True) for n in left.names]
-    right_list = [normalize(n, latinize=True) for n in right.names]
-    for (left, right) in itertools.product(left_list, right_list):
-        similarity = jaro(left, right)
-        score = similarity * dampen(2, 20, min(left, right, key=len))
-        result = max(result, score)
+            if group == registry.name:
+                score = compare_names(left, right)
+            elif group == registry.country:
+                score = compare_countries(left, right)
+            else:
+                score = compare_group(
+                    group, left_inv[group_name], right_inv[group_name]
+                )
+            scores[group] = score
+        except ValueError:
+            pass
+    for group_name in left_groups.symmetric_difference(right_groups):
+        group = registry.groups[group_name]
+        scores[group] = None
+    return scores
+
+
+def _compare(scores: Scores, weights: Weights, n_std: int = 1) -> float:
+    if not scores or not any(scores.values()):
+        return 0.0
+    prob = 0.0
+    for field, weight in weights.items():
+        if field:
+            prob += weight * (scores.get(field) or 0.0)
+        else:
+            prob += weight
+    return 1.0 / (1.0 + math.exp(-prob))
+
+
+def compare(
+    model: Model,
+    left: EntityProxy,
+    right: EntityProxy,
+    weights: Weights = COMPARE_WEIGHTS,
+) -> float:
+    """Compare two entities and return a match score."""
+    scores = compare_scores(model, left, right)
+    return _compare(scores, weights)
+
+
+def _normalize_names(names: Iterable[str]) -> Generator[str, None, None]:
+    """Generate a sequence of comparable names for an entity. This also
+    generates a `fingerprint`, i.e. a version of the name where all tokens
+    are sorted alphabetically, and some parts, such as company suffixes,
+    have been removed."""
+    seen = set()
+    for name in names:
+        plain = normalize(name, ascii=True)
+        if plain is not None and plain not in seen:
+            seen.add(plain)
+            yield plain
+        fp = fingerprints.generate(name)
+        if fp is not None and len(fp) > 6 and fp not in seen:
+            seen.add(fp)
+            yield fp
+
+
+def compare_group(
+    group_type: PropertyType, left_values: List[str], right_values: List[str]
+) -> Optional[float]:
+    if not left_values and not right_values:
+        raise ValueError("At least one proxy must have property type: %s", group_type)
+    elif not left_values or not right_values:
+        return None
+    return group_type.compare_sets(left_values, right_values)
+
+
+def compare_names(
+    left: EntityProxy, right: EntityProxy, max_names: int = 200
+) -> Optional[float]:
+    result = 0.0
+    left_list = list(itertools.islice(_normalize_names(left.names), max_names))
+    right_list = list(itertools.islice(_normalize_names(right.names), max_names))
+    if not left_list and not right_list:
+        raise ValueError("At least one proxy must have name properties")
+    elif not left_list or not right_list:
+        return None
+    for (left_val, right_val) in itertools.product(left_list, right_list):
+        similarity = registry.name.compare(left_val, right_val)
+        result = max(result, similarity)
+        if result == 1.0:
+            break
+    result *= min(
+        1.0, 2 ** (-len(left_list) * len(right_list) / (max_names * max_names))
+    )
     return result
+
+
+def compare_countries(left: EntityProxy, right: EntityProxy) -> Optional[float]:
+    left_countries = left.country_hints
+    right_countries = right.country_hints
+    if not left_countries and not right_countries:
+        raise ValueError("At least one proxy must have country properties")
+    elif not left_countries or not right_countries:
+        return None
+    intersection = left_countries.intersection(right_countries)
+    union = left_countries.union(right_countries)
+    return len(intersection) / float(len(union))

followthemoney/exc.py

@@ -1,18 +1,31 @@
+from typing import Dict, Optional, TypedDict
 
 
-class InvalidData(Exception):
+class ErrorSpec(TypedDict, total=False):
+    properties: Dict[str, str]
+
+
+class FollowTheMoneyException(Exception):
+    """Catch-all exception for errors emitted by this library."""
+
+    pass
+
+
+class InvalidData(FollowTheMoneyException):
     """Schema validation errors will be caught by the API."""
 
-    def __init__(self, errors):
-        self.errors = errors
-        super(InvalidData, self).__init__(repr(errors))
+    def __init__(self, message: str, errors: Optional[ErrorSpec] = None) -> None:
+        super(InvalidData, self).__init__(message)
+        self.errors: ErrorSpec = errors or {}
 
 
-class InvalidModel(Exception):
+class InvalidModel(FollowTheMoneyException):
     """The schema model is not defined correctly."""
+
     pass
 
 
-class InvalidMapping(Exception):
+class InvalidMapping(FollowTheMoneyException):
     """A data mapping was invalid."""
+
     pass

followthemoney/export/common.py

@@ -0,0 +1,29 @@
+from typing import Generator, List, Optional, Tuple
+from followthemoney.property import Property
+from followthemoney.proxy import E
+from followthemoney.schema import Schema
+from followthemoney.types import registry
+
+
+class Exporter(object):
+    def __init__(self, export_all: bool = False) -> None:
+        self.export_all = export_all
+
+    def exportable_properties(self, schema: Schema) -> Generator[Property, None, None]:
+        for prop in schema.sorted_properties:
+            if not self.export_all:
+                if prop.hidden or prop.type == registry.entity:
+                    continue
+            yield prop
+
+    def exportable_fields(
+        self, proxy: E
+    ) -> Generator[Tuple[Property, List[str]], None, None]:
+        for prop in self.exportable_properties(proxy.schema):
+            yield prop, proxy.get(prop)
+
+    def write(self, proxy: E, extra: Optional[List[str]] = None) -> None:
+        raise NotImplementedError
+
+    def finalize(self) -> None:
+        pass

followthemoney/export/csv.py

@@ -0,0 +1,82 @@
+import csv
+
+try:
+    from _csv import _writer as csv_writer
+except ImportError:
+    # Python 3.8/3.9 work-around:
+    from _csv import writer as csv_writer  # type: ignore
+
+from io import TextIOWrapper
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+from followthemoney.proxy import E
+from followthemoney.export.common import Exporter
+from followthemoney.schema import Schema
+from followthemoney.util import PathLike
+
+CSVWriter = csv_writer
+
+
+class CSVMixin(object):
+    def _configure(
+        self,
+        directory: PathLike,
+        extra: Optional[List[str]] = None,
+    ) -> None:
+        self.directory = Path(directory)
+        self.extra = extra or []
+        self.handles: Dict[Schema, Tuple[TextIOWrapper, CSVWriter]] = {}
+
+    def _open_csv_file(self, name: str) -> Tuple[TextIOWrapper, CSVWriter]:
+        self.directory.mkdir(parents=True, exist_ok=True)
+        file_path = self.directory.joinpath("{0}.csv".format(name))
+        handle = open(file_path, mode="w")
+        writer = csv.writer(handle, dialect=csv.unix_dialect)
+        return handle, writer
+
+    def _write_header(self, writer: CSVWriter, schema: Schema) -> None:
+        raise NotImplementedError
+
+    def _get_writer(self, schema: Schema) -> CSVWriter:
+        if schema not in self.handles:
+            handle, writer = self._open_csv_file(schema.name)
+            self.handles[schema] = (handle, writer)
+            self._write_header(writer, schema)
+        handle, writer = self.handles[schema]
+        return writer
+
+    def close(self) -> None:
+        for handle, _ in self.handles.values():
+            handle.close()
+
+
+class CSVExporter(Exporter, CSVMixin):
+    def __init__(
+        self,
+        directory: PathLike,
+        export_all: bool = True,
+        extra: Optional[List[str]] = None,
+    ) -> None:
+        Exporter.__init__(self, export_all=export_all)
+        self._configure(directory, extra=extra)
+
+    def _write_header(self, writer: CSVWriter, schema: Schema) -> None:
+        headers = ["id"]
+        headers.extend(self.extra)
+        for prop in self.exportable_properties(schema):
+            # Not using label to make it more machine-readable:
+            headers.append(prop.name)
+        writer.writerow(headers)
+
+    def write(self, proxy: E, extra: Optional[List[str]] = None) -> None:
+        writer = self._get_writer(proxy.schema)
+        cells = [proxy.id]
+        cells.extend(extra or [])
+        for prop, values in self.exportable_fields(proxy):
+            cells.append(prop.type.join(values))
+
+        writer.writerow(cells)
+
+    def finalize(self) -> None:
+        self.close()

followthemoney/export/excel.py

@@ -0,0 +1,75 @@
+import logging
+from io import BytesIO
+from typing import Dict, List, Optional
+from openpyxl import Workbook
+from openpyxl.cell import WriteOnlyCell
+from openpyxl.styles import Font, PatternFill
+from openpyxl.worksheet.worksheet import Worksheet
+from openpyxl.utils.exceptions import IllegalCharacterError
+
+from followthemoney.export.common import Exporter
+from followthemoney.proxy import E
+from followthemoney.schema import Schema
+from followthemoney.util import PathLike, sanitize_text
+
+log = logging.getLogger(__name__)
+
+
+class ExcelWriter(object):
+    HEADER_FONT = Font(bold=True, color="FFFFFF")
+    HEADER_FILL = PatternFill(
+        start_color="982022", end_color="982022", fill_type="solid"
+    )
+
+    def __init__(self) -> None:
+        self.workbook = Workbook(write_only=True)
+
+    def make_sheet(self, title: str, headers: List[str]) -> Worksheet:
+        sheet: Worksheet = self.workbook.create_sheet(title=title)
+        sheet.freeze_panes = "A2"
+        sheet.sheet_properties.filterMode = True
+        cells = []
+        for header in headers:
+            header_ = sanitize_text(header)
+            cell = WriteOnlyCell(sheet, value=header_)
+            cell.font = self.HEADER_FONT
+            cell.fill = self.HEADER_FILL
+            cells.append(cell)
+        sheet.append(cells)
+        return sheet
+
+    def get_bytesio(self) -> BytesIO:
+        buffer = BytesIO()
+        self.workbook.save(buffer)
+        buffer.seek(0)
+        return buffer
+
+
+class ExcelExporter(ExcelWriter, Exporter):
+    def __init__(self, file_path: PathLike, extra: Optional[List[str]] = None):
+        ExcelWriter.__init__(self)
+        Exporter.__init__(self)
+        self.file_path = file_path
+        self.extra = extra or []
+        self.sheets: Dict[Schema, Worksheet] = {}
+
+    def write(self, proxy: E, extra: Optional[List[str]] = None) -> None:
+        if proxy.schema not in self.sheets:
+            headers = ["ID"]
+            headers.extend(self.extra)
+            for prop in self.exportable_properties(proxy.schema):
+                headers.append(prop.label)
+            sheet = self.make_sheet(proxy.schema.plural, headers)
+            self.sheets[proxy.schema] = sheet
+        sheet = self.sheets[proxy.schema]
+        try:
+            cells = [proxy.id]
+            cells.extend(extra or [])
+            for prop, values in self.exportable_fields(proxy):
+                cells.append(prop.type.join(values))
+            sheet.append(cells)
+        except IllegalCharacterError as ice:
+            log.error("Invalid text for Excel export: %s", ice)
+
+    def finalize(self) -> None:
+        self.workbook.save(self.file_path)

followthemoney/export/graph.py

@@ -0,0 +1,79 @@
+from typing import Dict, Iterable, List, Optional, TextIO, Union
+import networkx as nx  # type: ignore
+from networkx.readwrite.gexf import generate_gexf  # type: ignore
+
+from followthemoney.graph import Edge, Graph, Node
+from followthemoney.proxy import E
+from followthemoney.types import registry
+from followthemoney.export.common import Exporter
+
+DEFAULT_EDGE_TYPES = (registry.entity.name,)
+
+
+def edge_types() -> List[str]:
+    return [t.name for t in registry.matchable if t is not None]
+
+
+class GraphExporter(Exporter):
+    """Base functions for exporting a property graph from a stream
+    of entities."""
+
+    def __init__(self, edge_types: Iterable[str] = DEFAULT_EDGE_TYPES) -> None:
+        super(GraphExporter, self).__init__()
+        types = registry.get_types(edge_types)
+        self.graph = Graph(edge_types=types)
+
+    def get_attributes(self, element: Union[Node, Edge]) -> Dict[str, str]:
+        attributes = {}
+        if element.proxy:
+            for prop, values in self.exportable_fields(element.proxy):
+                attributes[prop.name] = prop.type.join(values)
+        return attributes
+
+    def write(self, proxy: E, extra: Optional[List[str]] = None) -> None:
+        self.graph.add(proxy)
+        self.write_graph()
+
+    def finalize(self) -> None:
+        self.finalize_graph()
+        self.graph.flush()
+
+    def write_graph(self) -> None:
+        pass
+
+    def finalize_graph(self) -> None:
+        pass
+
+
+class NXGraphExporter(GraphExporter):
+    """Write to NetworkX data structure, which in turn can be exported
+    to the file formats for Gephi (GEXF) and D3."""
+
+    def __init__(
+        self, fh: TextIO, edge_types: Iterable[str] = DEFAULT_EDGE_TYPES
+    ) -> None:
+        super(NXGraphExporter, self).__init__(edge_types=edge_types)
+        self.fh = fh
+
+    def finalize_graph(self) -> None:
+        """Convert from FtM graph model to NetworkX directed graph."""
+        digraph = nx.MultiDiGraph()
+
+        for node in self.graph.iternodes():
+            attributes = self.get_attributes(node)
+            attributes["schema"] = node.type.name
+            if node.caption is not None:
+                attributes["label"] = node.caption
+            if node.is_entity and node.schema is not None:
+                attributes["schema"] = node.schema.name
+            digraph.add_node(node.id, **attributes)
+
+        for edge in self.graph.iteredges():
+            attributes = self.get_attributes(edge)
+            attributes["schema"] = edge.type_name
+            attributes["weight"] = str(edge.weight)
+            digraph.add_edge(edge.source_id, edge.target_id, key=edge.id, **attributes)
+
+        for line in generate_gexf(digraph, prettyprint=True):
+            self.fh.write(line)
+            self.fh.write("\n")

followthemoney/export/neo4j.py

@@ -0,0 +1,182 @@
+import os
+import json
+import logging
+from typing import Any, Dict, Iterable, List, Optional, Set, TextIO
+import stringcase  # type: ignore
+
+from followthemoney.export.csv import CSVMixin, CSVWriter
+from followthemoney.export.graph import GraphExporter, DEFAULT_EDGE_TYPES
+from followthemoney.graph import Edge, Node
+from followthemoney.schema import Schema
+from followthemoney.util import PathLike
+
+log = logging.getLogger(__name__)
+NEO4J_ADMIN_PATH = os.environ.get("NEO4J_ADMIN_PATH", "neo4j-admin")
+NEO4J_DATABASE_NAME = os.environ.get("NEO4J_DATABASE_NAME", "graph.db")
+
+
+class Neo4JCSVExporter(CSVMixin, GraphExporter):
+    def __init__(
+        self,
+        directory: PathLike,
+        extra: Optional[List[str]] = None,
+        edge_types: Iterable[str] = DEFAULT_EDGE_TYPES,
+    ) -> None:
+        super(Neo4JCSVExporter, self).__init__(edge_types=edge_types)
+        self._configure(directory, extra=extra)
+
+        self.links_handler, self.links_writer = self._open_csv_file("_links")
+        self.links_writer.writerow([":TYPE", ":START_ID", ":END_ID", "weight"])
+
+        self.nodes_handler, self.nodes_writer = self._open_csv_file("_nodes")
+        self.nodes_writer.writerow(["id:ID", ":LABEL", "caption"])
+        self.nodes_seen: Set[str] = set()
+
+    def _write_header(self, writer: CSVWriter, schema: Schema) -> None:
+        headers = []
+        if not schema.edge:
+            headers = ["id:ID", ":LABEL", "caption"]
+        else:
+            headers = ["id", ":TYPE", ":START_ID", ":END_ID"]
+
+        headers.extend(self.extra)
+        for prop in self.exportable_properties(schema):
+            headers.append(prop.name)
+        writer.writerow(headers)
+
+    def write_graph(self, extra: Optional[List[str]] = None) -> None:
+        extra_ = extra or []
+        for node in self.graph.iternodes():
+            self.write_node(node, extra_)
+
+        for edge in self.graph.iteredges():
+            self.write_edge(edge, extra_)
+
+        self.graph.flush()
+
+    def write_node(self, node: Node, extra: List[str]) -> None:
+        if node.id is None:
+            return None
+        if not node.is_entity and node.id not in self.nodes_seen:
+            row = [node.id, node.type.name, node.caption]
+            self.nodes_writer.writerow(row)
+            self.nodes_seen.add(node.id)
+        if node.proxy is not None and node.schema is not None:
+            label = ";".join(node.schema.names)
+            cells = [node.id, label, node.caption]
+            cells.extend(extra or [])
+            for prop, values in self.exportable_fields(node.proxy):
+                cells.append(prop.type.join(values))
+            writer = self._get_writer(node.schema)
+            writer.writerow(cells)
+
+    def write_edge(self, edge: Edge, extra: List[str]) -> None:
+        if edge.prop is not None:
+            type_ = stringcase.constcase(edge.prop.name)
+            row = [type_, edge.source_id, edge.target_id, edge.weight]
+            self.links_writer.writerow(row)
+        if edge.proxy is not None:
+            proxy = edge.proxy
+            type_ = stringcase.constcase(proxy.schema.name)
+            # That potentially may lead to multiple edges with same id
+            cells = [proxy.id, type_, edge.source_id, edge.target_id]
+            cells.extend(extra or [])
+
+            for prop, values in self.exportable_fields(edge.proxy):
+                cells.append(prop.type.join(values))
+
+            writer = self._get_writer(proxy.schema)
+            writer.writerow(cells)
+
+    def finalize_graph(self) -> None:
+        script_path = self.directory.joinpath("neo4j_import.sh")
+        with open(script_path, mode="w") as fp:
+            cmd = "{} import --id-type=STRING --database={} \\\n"
+            fp.write(cmd.format(NEO4J_ADMIN_PATH, NEO4J_DATABASE_NAME))
+            fp.write("\t--multiline-fields=true \\\n")
+            cmd = "\t--relationships={} \\\n"
+            fp.write(cmd.format(os.path.basename(self.links_handler.name)))
+            cmd = "\t--nodes={} \\\n"
+            fp.write(cmd.format(os.path.basename(self.nodes_handler.name)))
+
+            for schema, (handle, writer) in self.handles.items():
+                file_name = os.path.basename(handle.name)
+                if schema.edge:
+                    cmd = "\t--relationships={} \\\n"
+                    fp.write(cmd.format(file_name))
+                else:
+                    cmd = "\t--nodes={} \\\n"
+                    fp.write(cmd.format(file_name))
+
+        self.links_handler.close()
+        self.nodes_handler.close()
+        self.close()
+
+
+class CypherGraphExporter(GraphExporter):
+    """Cypher query format, used for import to Neo4J. This is a bit like
+    writing SQL with individual statements - so for large datasets it
+    might be a better idea to do a CSV-based import."""
+
+    # https://www.opencypher.org/
+    # MATCH (n) DETACH DELETE n;
+
+    def __init__(self, fh: TextIO, edge_types: Iterable[str] = DEFAULT_EDGE_TYPES):
+        super(CypherGraphExporter, self).__init__(edge_types=edge_types)
+        self.fh = fh
+        self.proxy_nodes: Set[str] = set()
+
+    def _to_map(self, data: Dict[str, Any]) -> str:
+        values = []
+        for key, value in data.items():
+            if value:
+                value = "%s: %s" % (key, json.dumps(value))
+                values.append(value)
+        return ", ".join(values)
+
+    def write_graph(self) -> None:
+        """Export queries for each graph element."""
+        for node in self.graph.iternodes():
+            if node.value in self.proxy_nodes:
+                continue
+            if node.id is None:
+                continue
+            if node.proxy is not None:
+                self.proxy_nodes.add(node.value)
+            attributes = self.get_attributes(node)
+            attributes["id"] = node.id
+            if node.caption is not None:
+                attributes["caption"] = node.caption
+            if node.schema:
+                labels = list(node.schema.names)
+            else:
+                labels = [node.type.name]
+            cypher = "MERGE (p { %(id)s }) " "SET p += { %(map)s } SET p :%(label)s;\n"
+            self.fh.write(
+                cypher
+                % {
+                    "id": self._to_map({"id": node.id}),
+                    "map": self._to_map(attributes),
+                    "label": ":".join(labels),
+                }
+            )
+
+        for edge in self.graph.iteredges():
+            attributes = self.get_attributes(edge)
+            attributes["id"] = edge.id
+            attributes["weight"] = str(edge.weight)
+            cypher = (
+                "MATCH (s { %(source)s }), (t { %(target)s }) "
+                "MERGE (s)-[:%(type)s { %(map)s }]->(t);\n"
+            )
+            self.fh.write(
+                cypher
+                % {
+                    "source": self._to_map({"id": edge.source_id}),
+                    "target": self._to_map({"id": edge.target_id}),
+                    "type": stringcase.constcase(edge.type_name),
+                    "map": self._to_map(attributes),
+                }
+            )
+
+        self.graph.flush()