followthemoney 1.3.6__py3-none-any.whl → 3.8.0__py3-none-any.whl

followthemoney/export/rdf.py

@@ -0,0 +1,26 @@
+import logging
+from rdflib import Graph
+from typing import List, Optional, TextIO
+
+from followthemoney.export.common import Exporter
+from followthemoney.proxy import E
+
+log = logging.getLogger(__name__)
+
+
+class RDFExporter(Exporter):
+    def __init__(self, fh: TextIO, qualified: bool = True) -> None:
+        super(RDFExporter, self).__init__()
+        self.fh = fh
+        self.qualified = qualified
+
+    def write(self, proxy: E, extra: Optional[List[str]] = None) -> None:
+        graph = Graph()
+
+        for triple in proxy.triples(qualified=self.qualified):
+            graph.add(triple)
+        try:
+            nt = graph.serialize(format="nt11").strip()
+            self.fh.write(nt + "\n")
+        except Exception:
+            log.exception("Failed to serialize ntriples.")

followthemoney/graph.py

@@ -0,0 +1,308 @@
+"""
+Converting FtM data to a property graph data model.
+
+This module provides an abstract data object that represents a property
+graph. This is used by the exporter modules to convert data
+to a specific output format, like Cypher or NetworkX.
+"""
+import logging
+from typing import Any, Dict, Generator, Iterable, List, Optional
+
+from followthemoney.types import registry
+from followthemoney.types.common import PropertyType
+from followthemoney.schema import Schema
+from followthemoney.proxy import EntityProxy
+from followthemoney.property import Property
+from followthemoney.exc import InvalidModel
+
+log = logging.getLogger(__name__)
+
+
+class Node(object):
+    """A node represents either an entity that can be rendered as a
+    node in a graph, or as a re-ified value, like a name, email
+    address or phone number."""
+
+    __slots__ = ["type", "value", "id", "proxy", "schema"]
+
+    def __init__(
+        self,
+        type_: PropertyType,
+        value: str,
+        proxy: Optional[EntityProxy] = None,
+        schema: Optional[Schema] = None,
+    ) -> None:
+        self.type = type_
+        self.value = value
+        # _id = type_.node_id_safe(value)
+        # if _id is None:
+        #     raise InvalidData("No ID for node")
+        self.id = type_.node_id_safe(value)
+        self.proxy = proxy
+        self.schema = schema if proxy is None else proxy.schema
+
+    @property
+    def is_entity(self) -> bool:
+        """Check to see if the node represents an entity. If this is false, the
+        node represents a non-entity property value that has been reified, like
+        a phone number or a name."""
+        return self.type == registry.entity
+
+    @property
+    def caption(self) -> str:
+        """A user-facing label for the current node."""
+        if self.type == registry.entity and self.proxy is not None:
+            return self.proxy.caption
+        caption = self.type.caption(self.value)
+        return caption or self.value
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Return a simple dictionary to reflect this graph node."""
+        return {
+            "id": self.id,
+            "type": self.type.name,
+            "value": self.value,
+            "caption": self.caption,
+        }
+
+    @classmethod
+    def from_proxy(cls, proxy: EntityProxy) -> "Node":
+        """For a given :class:`~followthemoney.proxy.EntityProxy`, return a node
+        based on the entity."""
+        return cls(registry.entity, proxy.id, proxy=proxy)
+
+    def __str__(self) -> str:
+        return self.caption
+
+    def __repr__(self) -> str:
+        return "<Node(%r, %r, %r)>" % (self.id, self.type, self.caption)
+
+    def __hash__(self) -> int:
+        return hash(self.id)
+
+    def __eq__(self, other: Any) -> bool:
+        return bool(self.id == other.id)
+
+
+class Edge(object):
+    """A link between two nodes."""
+
+    __slots__ = [
+        "id",
+        "weight",
+        "source_id",
+        "target_id",
+        "prop",
+        "proxy",
+        "schema",
+        "graph",
+    ]
+
+    def __init__(
+        self,
+        graph: "Graph",
+        source: Node,
+        target: Node,
+        proxy: Optional[EntityProxy] = None,
+        prop: Optional[Property] = None,
+        value: Optional[str] = None,
+    ):
+        self.graph = graph
+        self.id = f"{source.id}<>{target.id}"
+        self.source_id = source.id
+        self.target_id = target.id
+        self.weight = 1.0
+        self.prop = prop
+        self.proxy = proxy
+        self.schema: Optional[Schema] = None
+        if prop is not None and value is not None:
+            self.weight = prop.specificity(value)
+        if proxy is not None:
+            self.id = f"{source.id}<{proxy.id}>{target.id}"
+            self.schema = proxy.schema
+
+    @property
+    def source(self) -> Optional[Node]:
+        """The graph node from which the edge originates."""
+        if self.source_id is None:
+            return None
+        return self.graph.nodes.get(self.source_id)
+
+    @property
+    def source_prop(self) -> Property:
+        """Get the entity property originating this edge."""
+        if self.schema is not None and self.schema.source_prop is not None:
+            if self.schema.source_prop.reverse is not None:
+                return self.schema.source_prop.reverse
+        if self.prop is None:
+            raise InvalidModel("Contradiction: %r" % self)
+        return self.prop
+
+    @property
+    def target(self) -> Optional[Node]:
+        """The graph node to which the edge points."""
+        if self.target_id is None:
+            return None
+        return self.graph.nodes.get(self.target_id)
+
+    @property
+    def target_prop(self) -> Optional[Property]:
+        """Get the entity property originating this edge."""
+        if self.schema is not None and self.schema.target_prop is not None:
+            return self.schema.target_prop.reverse
+        if self.prop is not None:
+            return self.prop.reverse
+        # NOTE: this edge points at a value node.
+        return None
+
+    @property
+    def type_name(self) -> str:
+        """Return a machine-readable description of the type of the edge.
+        This is either a property name or a schema name."""
+        if self.schema is not None:
+            return self.schema.name
+        if self.prop is None:
+            raise InvalidModel("Invalid edge: %r" % self)
+        return self.prop.name
+
+    def to_dict(self) -> Dict[str, Optional[str]]:
+        return {
+            "id": self.id,
+            "source_id": self.source_id,
+            "target_id": self.target_id,
+            "type_name": self.type_name,
+        }
+
+    def __repr__(self) -> str:
+        return "<Edge(%r)>" % self.id
+
+    def __hash__(self) -> int:
+        return hash(self.id)
+
+    def __eq__(self, other: Any) -> bool:
+        return bool(self.id == other.id)
+
+
+class Graph(object):
+    """A set of nodes and edges, derived from entities and their properties.
+    This represents an alternative interpretation of FtM data as a property
+    graph.
+
+    This class is meant to be extensible in order to support additional
+    backends, like Aleph.
+    """
+
+    def __init__(self, edge_types: Iterable[PropertyType] = registry.pivots) -> None:
+        types = registry.get_types(edge_types)
+        self.edge_types = [t for t in types if t.matchable]
+        self.flush()
+
+    def flush(self) -> None:
+        """Remove all nodes, edges and proxies from the graph."""
+        self.edges: Dict[str, Edge] = {}
+        self.nodes: Dict[str, Node] = {}
+        self.proxies: Dict[str, Optional[EntityProxy]] = {}
+
+    def queue(self, id_: str, proxy: Optional[EntityProxy] = None) -> None:
+        """Register a reference to an entity in the graph."""
+        if id_ not in self.proxies or proxy is not None:
+            self.proxies[id_] = proxy
+
+    @property
+    def queued(self) -> List[str]:
+        """Return a list of all the entities which are referenced from the graph
+        but that haven't been loaded yet. This can be used to get a list of
+        entities that should be included to expand the whole graph by one degree.
+        """
+        return [i for (i, p) in self.proxies.items() if p is None]
+
+    def _get_node_stub(self, prop: Property, value: str) -> Node:
+        if prop.type == registry.entity:
+            self.queue(value)
+        node = Node(prop.type, value, schema=prop.range)
+        if node.id is None:
+            return node
+        if node.id not in self.nodes:
+            self.nodes[node.id] = node
+        return self.nodes[node.id]
+
+    def _add_edge(self, proxy: EntityProxy, source: str, target: str) -> None:
+        if proxy.schema.source_prop is None:
+            raise InvalidModel("Invalid edge entity: %r" % proxy)
+        source_node = self._get_node_stub(proxy.schema.source_prop, source)
+        if proxy.schema.target_prop is None:
+            raise InvalidModel("Invalid edge entity: %r" % proxy)
+        target_node = self._get_node_stub(proxy.schema.target_prop, target)
+        if source_node.id is not None and target_node.id is not None:
+            edge = Edge(self, source_node, target_node, proxy=proxy)
+            self.edges[edge.id] = edge
+
+    def _add_node(self, proxy: EntityProxy) -> None:
+        """Derive a node and its value edges from the given proxy."""
+        entity = Node.from_proxy(proxy)
+        if entity.id is not None:
+            self.nodes[entity.id] = entity
+        for prop, value in proxy.itervalues():
+            if prop.type not in self.edge_types:
+                continue
+            node = self._get_node_stub(prop, value)
+            if node.id is None:
+                continue
+            edge = Edge(self, entity, node, prop=prop, value=value)
+            if edge.weight > 0:
+                self.edges[edge.id] = edge
+
+    def add(self, proxy: EntityProxy) -> None:
+        """Add an :class:`~followthemoney.proxy.EntityProxy` to the graph and make
+        it either a :class:`~followthemoney.graph.Node` or an
+        :class:`~followthemoney.graph.Edge`."""
+        if proxy is None:
+            return
+        self.queue(proxy.id, proxy)
+        if proxy.schema.edge:
+            for (source, target) in proxy.edgepairs():
+                self._add_edge(proxy, source, target)
+        else:
+            self._add_node(proxy)
+
+    def iternodes(self) -> Iterable[Node]:
+        """Iterate all :class:`nodes <followthemoney.graph.Node>` in the graph."""
+        return self.nodes.values()
+
+    def iteredges(self) -> Iterable[Edge]:
+        """Iterate all :class:`edges <followthemoney.graph.Edge>` in the graph."""
+        return self.edges.values()
+
+    def get_outbound(
+        self, node: Node, prop: Optional[Property] = None
+    ) -> Generator[Edge, None, None]:
+        """Get all edges pointed out from the given node."""
+        for edge in self.iteredges():
+            if edge.source == node:
+                if prop and edge.source_prop != prop:
+                    continue
+                yield edge
+
+    def get_inbound(
+        self, node: Node, prop: Optional[Property] = None
+    ) -> Generator[Edge, None, None]:
+        """Get all edges pointed at the given node."""
+        for edge in self.iteredges():
+            if edge.target == node:
+                if prop and edge.target_prop != prop:
+                    continue
+                yield edge
+
+    def get_adjacent(
+        self, node: Node, prop: Optional[Property] = None
+    ) -> Generator[Edge, None, None]:
+        "Get all adjacent edges of the given node."
+        yield from self.get_outbound(node, prop=prop)
+        yield from self.get_inbound(node, prop=prop)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Return a dictionary with the graph nodes and edges."""
+        return {
+            "nodes": [n.to_dict() for n in self.iternodes()],
+            "edges": [e.to_dict() for e in self.iteredges()],
+        }

followthemoney/helpers.py

@@ -0,0 +1,212 @@
+# This module violates the boundary between the role of code and
+# YAML in the rest of followthemoney. It handles normalisations
+# which would be much harder to express in abstract, especially
+# those thet simplify the data based on their pragmatics.
+#
+# If anyone were to swap out the default model, this would
+# probably be the first place to break.
+from os.path import splitext
+from typing import Iterable, List, Optional, Set
+from normality import safe_filename
+from mimetypes import guess_extension
+from itertools import product
+from datetime import datetime, timedelta
+
+from followthemoney.types import registry
+from followthemoney.proxy import E
+from followthemoney.util import join_text
+
+PROV_MIN_DATES = ("createdAt", "authoredAt", "publishedAt")
+PROV_MAX_DATES = ("modifiedAt", "retrievedAt")
+
+
+def remove_checksums(proxy: E) -> E:
+    """When accepting entities via a web API, it would consistute
+    a security risk to allow a user to submit checksum-type properties.
+    These can be traded in for access to said files if they exist in the
+    underlying content-addressed storage. It seems safest to just remove
+    all checksums from entities when they are untrusted user input."""
+    for prop in proxy.iterprops():
+        if prop.type == registry.checksum:
+            proxy.pop(prop)
+    return proxy
+
+
+def simplify_provenance(proxy: E) -> E:
+    """If there are multiple dates given for some of the provenance
+    fields, we can logically conclude which one is the most meaningful."""
+    for prop_name in PROV_MAX_DATES:
+        values = proxy.pop(prop_name, quiet=True)
+        if len(values):
+            proxy.set(prop_name, max(values), cleaned=True)
+    for prop_name in PROV_MIN_DATES:
+        values = proxy.pop(prop_name, quiet=True)
+        if len(values):
+            proxy.set(prop_name, min(values), cleaned=True)
+    return proxy
+
+
+def entity_filename(
+    proxy: E, base_name: Optional[str] = None, extension: Optional[str] = None
+) -> Optional[str]:
+    """Derive a safe filename for the given entity."""
+    if proxy.schema.is_a("Document"):
+        for extension_ in proxy.get("extension", quiet=True):
+            if extension is not None:
+                break
+            extension = extension_
+        for file_name in proxy.get("fileName", quiet=True):
+            base_name_, extension_ = splitext(file_name)
+            if base_name is None and len(base_name_):
+                base_name = base_name_
+            if extension is None and len(extension_):
+                extension = extension_
+        for mime_type in proxy.get("mimeType", quiet=True):
+            if extension is not None:
+                break
+            extension = guess_extension(mime_type)
+    base_name = base_name or proxy.id
+    return safe_filename(base_name, extension=extension)
+
+
+def name_entity(entity: E) -> E:
+    """If an entity has multiple names, pick the most central one
+    and set all the others as aliases. This is awkward given that
+    names are not special and may not always be the caption."""
+    if entity.schema.is_a("Thing"):
+        names = entity.get("name")
+        if len(names) > 1:
+            name = registry.name.pick(names)
+            if name in names:
+                names.remove(name)
+            entity.set("name", name)
+            entity.add("alias", names)
+    return entity
+
+
+def check_person_cutoff(
+    entity: E,
+    death_cutoff: datetime = datetime(2000, 1, 1),
+    birth_cutoff: Optional[datetime] = None,
+) -> bool:
+    """Check if a person has been dead long enough to not be relevant for
+    investigations any more."""
+    if not entity.schema.is_a("Person"):
+        return False
+    death_dates = entity.get("deathDate", quiet=True)
+    death_cutoff_ = death_cutoff.isoformat()
+    if len(death_dates) and max(death_dates) < death_cutoff_:
+        return True
+    birth_dates = entity.get("birthDate", quiet=True)
+    if birth_cutoff is None:
+        birth_cutoff = death_cutoff - timedelta(days=100 * 365)
+    birth_cutoff_ = birth_cutoff.isoformat()
+    if len(birth_dates) and min(birth_dates) < birth_cutoff_:
+        return True
+    return False
+
+
+def remove_prefix_dates(entity: E) -> E:
+    """If an entity has multiple values for a date field, you may
+    want to remove all those that are prefixes of others. For example,
+    if a Person has both a birthDate of 1990 and of 1990-05-01, we'd
+    want to drop the mention of 1990."""
+    for prop in entity.iterprops():
+        if prop.type == registry.date:
+            values = remove_prefix_date_values(entity.get(prop))
+            entity.set(prop, values)
+    return entity
+
+
+def remove_prefix_date_values(values: Iterable[str]) -> List[str]:
+    """See ``remove_prefix_dates``."""
+    kept: List[str] = []
+    values = sorted(values, key=len, reverse=True)
+    for index, value in enumerate(values):
+        keep = True
+        for longer in values[:index]:
+            if longer.startswith(value):
+                keep = False
+                break
+        if keep:
+            kept.append(value)
+    return kept
+
+
+def inline_names(entity: E, related: E) -> None:
+    """Attempt to solve a weird UI problem. Imagine we are showing a list of
+    payments between a sender and a beneficiary to a user. They may now conduct
+    a search for a term present in the sender or recipient name, but there will
+    be no result, because the name is only indexed with the parties, but not in
+    the payment. This is part of a partial work-around to that.
+
+    This is really bad in theory, but really useful in practice. Shoot me.
+    """
+    prop = entity.schema.get("namesMentioned")
+    if prop is not None:
+        entity.add(prop, related.get_type_values(registry.name))
+
+
+def combine_names(entity: E) -> E:
+    """This function will try to build names from name parts provided as part
+    of a person entity. This is of course impossible to do culturally correctly
+    for the whole planet at once, so it should be mostly used for internal-facing
+    (e.g. matching) processes."""
+    if entity.schema.is_a("Person"):
+        first_names = entity.get("firstName")
+        second_names = entity.get("secondName")
+        second_names.append("")
+        middle_names = entity.get("middleName")
+        middle_names.append("")
+        father_names = entity.get("fatherName")
+        father_names.append("")
+        last_names = entity.get("lastName")
+        for (first, second, middle, father, last) in product(
+            first_names, second_names, middle_names, father_names, last_names
+        ):
+            name = join_text(first, second, middle, father, last)
+            if name is not None:
+                entity.add("alias", name)
+
+        # If no first name is given, at least add the last name:
+        if not entity.get_type_values(registry.name) and len(last_names):
+            entity.add("alias", last_names)
+    return entity
+
+
+def dates_years(dates: Iterable[Optional[str]]) -> Set[str]:
+    """Get the unique years from a set of date strings."""
+    cleaned: Set[str] = set()
+    for date in dates:
+        if date is not None:
+            cleaned.add(date[:4])
+    return cleaned
+
+
+def post_summary(
+    organization: str,
+    role: Optional[str],
+    start_dates: Iterable[Optional[str]],
+    end_dates: Iterable[Optional[str]],
+    dates: Iterable[Optional[str]],
+) -> str:
+    """Make a string summary for a Post object."""
+    position = organization
+    start = min(dates_years(start_dates), default="")
+    end = min(dates_years(end_dates), default="")
+    date_range = None
+    if len(start) or len(end):
+        date_range = f"{start}-{end}"
+    dates_ = dates_years(dates)
+    if date_range is None and len(dates_):
+        date_range = ", ".join(sorted(dates_))
+
+    bracketed = None
+    if date_range and role:
+        bracketed = f"{role}, {date_range}"
+    else:
+        bracketed = role or date_range
+
+    if bracketed:
+        position = f"{position} ({bracketed})"
+    return position

followthemoney/mapping/__init__.py

@@ -1,3 +1,3 @@
 from followthemoney.mapping.query import QueryMapping
 
-__all__ = [QueryMapping]
+__all__ = ["QueryMapping"]

followthemoney/mapping/csv.py

@@ -1,65 +1,97 @@
 import io
 import os
 import logging
+from banal.lists import ensure_list
 import requests
 from csv import DictReader
-from banal import ensure_list
-from normality import stringify
+from urllib.parse import urlparse
+from banal import keys_values
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Generator,
+    ItemsView,
+    Iterable,
+    List,
+    Optional,
+    Set,
+    Tuple,
+    cast,
+)
 
-from followthemoney.mapping.source import Source
+from followthemoney.mapping.source import Record, Source
+from followthemoney.util import sanitize_text
 from followthemoney.exc import InvalidMapping
 
+if TYPE_CHECKING:
+    from followthemoney.mapping.query import QueryMapping
+
 log = logging.getLogger(__name__)
+FilterList = List[Tuple[str, Set[Optional[str]]]]
 
 
 class CSVSource(Source):
     """Special case for entity loading directly from a CSV URL"""
 
-    def __init__(self, query, data):
-        super(CSVSource, self).__init__(query, data)
-        urls = ensure_list(data.get('csv_url'))
-        urls.extend(ensure_list(data.get('csv_urls')))
-        self.urls = set()
-        for url in urls:
-            self.urls.add(os.path.expandvars(url))
+    def __init__(self, query: "QueryMapping", data: Dict[str, Any]) -> None:
+        super().__init__(query, data)
+        self.urls: Set[str] = set()
+        for url in keys_values(data, "csv_url", "csv_urls"):
+            self.urls.add(cast(str, os.path.expandvars(url)))
 
         if not len(self.urls):
             raise InvalidMapping("No CSV URLs are specified.")
 
-    def read_csv(self, url):
-        parsed_url = requests.utils.urlparse(url)
+        self.filters_set = self._parse_filters(self.filters)
+        self.filters_not_set = self._parse_filters(self.filters_not)
+
+    def _parse_filters(self, filters: ItemsView[str, Any]) -> FilterList:
+        filters_set: FilterList = []
+        for (key, value) in filters:
+            values = set(cast(List[Optional[str]], ensure_list(value)))
+            filters_set.append((key, values))
+        return filters_set
+
+    def check_filters(self, data: Record) -> bool:
+        for (k, v) in self.filters_set:
+            if data.get(k) not in v:
+                return False
+        for (k, v) in self.filters_not_set:
+            if data.get(k) in v:
+                return False
+        return True
+
+    @classmethod
+    def read_csv(cls, fh: Iterable[str]) -> Generator[Record, None, None]:
+        for row in DictReader(fh, skipinitialspace=True):
+            data: Record = {}
+            for ref, ref_value in row.items():
+                value = sanitize_text(ref_value)
+                if value is not None:
+                    data[ref] = value
+            yield data
+
+    def read_csv_url(self, url: str) -> Generator[Record, None, None]:
+        parsed_url = urlparse(url)
         log.info("Loading: %s", url)
-        if parsed_url.scheme in ['http', 'https']:
+        if parsed_url.scheme in ["http", "https"]:
             res = requests.get(url, stream=True)
             if not res.ok:
                 raise InvalidMapping("Failed to open CSV: %s" % url)
             # if res.encoding is None:
-            res.encoding = 'utf-8'
+            res.encoding = "utf-8"
             # log.info("Detected encoding: %s", res.encoding)
             lines = res.iter_lines(decode_unicode=True)
-            for row in DictReader(lines, skipinitialspace=True):
-                yield row
+            yield from self.read_csv(lines)
         else:
-            with io.open(parsed_url.path, 'r') as fh:
-                for row in DictReader(fh, skipinitialspace=True):
-                    yield row
-
-    def check_filters(self, data):
-        for (k, v) in self.filters:
-            if v != data.get(k):
-                return False
-        for (k, v) in self.filters_not:
-            if v == data.get(k):
-                return False
-        return True
+            with io.open(parsed_url.path, "r") as fh:
+                yield from self.read_csv(fh)
 
     @property
-    def records(self):
+    def records(self) -> Generator[Record, None, None]:
         """Iterate through the table applying filters on-the-go."""
         for url in self.urls:
-            for row in self.read_csv(url):
-                data = {}
-                for ref in self.query.refs:
-                    data[ref] = stringify(row.get(ref))
-                if self.check_filters(data):
-                    yield data
+            for record in self.read_csv_url(url):
+                if self.check_filters(record):
+                    yield record