followthemoney 1.3.7__py3-none-any.whl → 3.8.1__py3-none-any.whl

followthemoney/export/neo4j.py

@@ -0,0 +1,182 @@
+import os
+import json
+import logging
+from typing import Any, Dict, Iterable, List, Optional, Set, TextIO
+import stringcase  # type: ignore
+
+from followthemoney.export.csv import CSVMixin, CSVWriter
+from followthemoney.export.graph import GraphExporter, DEFAULT_EDGE_TYPES
+from followthemoney.graph import Edge, Node
+from followthemoney.schema import Schema
+from followthemoney.util import PathLike
+
+log = logging.getLogger(__name__)
+NEO4J_ADMIN_PATH = os.environ.get("NEO4J_ADMIN_PATH", "neo4j-admin")
+NEO4J_DATABASE_NAME = os.environ.get("NEO4J_DATABASE_NAME", "graph.db")
+
+
+class Neo4JCSVExporter(CSVMixin, GraphExporter):
+    def __init__(
+        self,
+        directory: PathLike,
+        extra: Optional[List[str]] = None,
+        edge_types: Iterable[str] = DEFAULT_EDGE_TYPES,
+    ) -> None:
+        super(Neo4JCSVExporter, self).__init__(edge_types=edge_types)
+        self._configure(directory, extra=extra)
+
+        self.links_handler, self.links_writer = self._open_csv_file("_links")
+        self.links_writer.writerow([":TYPE", ":START_ID", ":END_ID", "weight"])
+
+        self.nodes_handler, self.nodes_writer = self._open_csv_file("_nodes")
+        self.nodes_writer.writerow(["id:ID", ":LABEL", "caption"])
+        self.nodes_seen: Set[str] = set()
+
+    def _write_header(self, writer: CSVWriter, schema: Schema) -> None:
+        headers = []
+        if not schema.edge:
+            headers = ["id:ID", ":LABEL", "caption"]
+        else:
+            headers = ["id", ":TYPE", ":START_ID", ":END_ID"]
+
+        headers.extend(self.extra)
+        for prop in self.exportable_properties(schema):
+            headers.append(prop.name)
+        writer.writerow(headers)
+
+    def write_graph(self, extra: Optional[List[str]] = None) -> None:
+        extra_ = extra or []
+        for node in self.graph.iternodes():
+            self.write_node(node, extra_)
+
+        for edge in self.graph.iteredges():
+            self.write_edge(edge, extra_)
+
+        self.graph.flush()
+
+    def write_node(self, node: Node, extra: List[str]) -> None:
+        if node.id is None:
+            return None
+        if not node.is_entity and node.id not in self.nodes_seen:
+            row = [node.id, node.type.name, node.caption]
+            self.nodes_writer.writerow(row)
+            self.nodes_seen.add(node.id)
+        if node.proxy is not None and node.schema is not None:
+            label = ";".join(node.schema.names)
+            cells = [node.id, label, node.caption]
+            cells.extend(extra or [])
+            for prop, values in self.exportable_fields(node.proxy):
+                cells.append(prop.type.join(values))
+            writer = self._get_writer(node.schema)
+            writer.writerow(cells)
+
+    def write_edge(self, edge: Edge, extra: List[str]) -> None:
+        if edge.prop is not None:
+            type_ = stringcase.constcase(edge.prop.name)
+            row = [type_, edge.source_id, edge.target_id, edge.weight]
+            self.links_writer.writerow(row)
+        if edge.proxy is not None:
+            proxy = edge.proxy
+            type_ = stringcase.constcase(proxy.schema.name)
+            # That potentially may lead to multiple edges with same id
+            cells = [proxy.id, type_, edge.source_id, edge.target_id]
+            cells.extend(extra or [])
+
+            for prop, values in self.exportable_fields(edge.proxy):
+                cells.append(prop.type.join(values))
+
+            writer = self._get_writer(proxy.schema)
+            writer.writerow(cells)
+
+    def finalize_graph(self) -> None:
+        script_path = self.directory.joinpath("neo4j_import.sh")
+        with open(script_path, mode="w") as fp:
+            cmd = "{} import --id-type=STRING --database={} \\\n"
+            fp.write(cmd.format(NEO4J_ADMIN_PATH, NEO4J_DATABASE_NAME))
+            fp.write("\t--multiline-fields=true \\\n")
+            cmd = "\t--relationships={} \\\n"
+            fp.write(cmd.format(os.path.basename(self.links_handler.name)))
+            cmd = "\t--nodes={} \\\n"
+            fp.write(cmd.format(os.path.basename(self.nodes_handler.name)))
+
+            for schema, (handle, writer) in self.handles.items():
+                file_name = os.path.basename(handle.name)
+                if schema.edge:
+                    cmd = "\t--relationships={} \\\n"
+                    fp.write(cmd.format(file_name))
+                else:
+                    cmd = "\t--nodes={} \\\n"
+                    fp.write(cmd.format(file_name))
+
+        self.links_handler.close()
+        self.nodes_handler.close()
+        self.close()
+
+
+class CypherGraphExporter(GraphExporter):
+    """Cypher query format, used for import to Neo4J. This is a bit like
+    writing SQL with individual statements - so for large datasets it
+    might be a better idea to do a CSV-based import."""
+
+    # https://www.opencypher.org/
+    # MATCH (n) DETACH DELETE n;
+
+    def __init__(self, fh: TextIO, edge_types: Iterable[str] = DEFAULT_EDGE_TYPES):
+        super(CypherGraphExporter, self).__init__(edge_types=edge_types)
+        self.fh = fh
+        self.proxy_nodes: Set[str] = set()
+
+    def _to_map(self, data: Dict[str, Any]) -> str:
+        values = []
+        for key, value in data.items():
+            if value:
+                value = "%s: %s" % (key, json.dumps(value))
+                values.append(value)
+        return ", ".join(values)
+
+    def write_graph(self) -> None:
+        """Export queries for each graph element."""
+        for node in self.graph.iternodes():
+            if node.value in self.proxy_nodes:
+                continue
+            if node.id is None:
+                continue
+            if node.proxy is not None:
+                self.proxy_nodes.add(node.value)
+            attributes = self.get_attributes(node)
+            attributes["id"] = node.id
+            if node.caption is not None:
+                attributes["caption"] = node.caption
+            if node.schema:
+                labels = list(node.schema.names)
+            else:
+                labels = [node.type.name]
+            cypher = "MERGE (p { %(id)s }) " "SET p += { %(map)s } SET p :%(label)s;\n"
+            self.fh.write(
+                cypher
+                % {
+                    "id": self._to_map({"id": node.id}),
+                    "map": self._to_map(attributes),
+                    "label": ":".join(labels),
+                }
+            )
+
+        for edge in self.graph.iteredges():
+            attributes = self.get_attributes(edge)
+            attributes["id"] = edge.id
+            attributes["weight"] = str(edge.weight)
+            cypher = (
+                "MATCH (s { %(source)s }), (t { %(target)s }) "
+                "MERGE (s)-[:%(type)s { %(map)s }]->(t);\n"
+            )
+            self.fh.write(
+                cypher
+                % {
+                    "source": self._to_map({"id": edge.source_id}),
+                    "target": self._to_map({"id": edge.target_id}),
+                    "type": stringcase.constcase(edge.type_name),
+                    "map": self._to_map(attributes),
+                }
+            )
+
+        self.graph.flush()

followthemoney/export/rdf.py

@@ -0,0 +1,26 @@
+import logging
+from rdflib import Graph
+from typing import List, Optional, TextIO
+
+from followthemoney.export.common import Exporter
+from followthemoney.proxy import E
+
+log = logging.getLogger(__name__)
+
+
+class RDFExporter(Exporter):
+    def __init__(self, fh: TextIO, qualified: bool = True) -> None:
+        super(RDFExporter, self).__init__()
+        self.fh = fh
+        self.qualified = qualified
+
+    def write(self, proxy: E, extra: Optional[List[str]] = None) -> None:
+        graph = Graph()
+
+        for triple in proxy.triples(qualified=self.qualified):
+            graph.add(triple)
+        try:
+            nt = graph.serialize(format="nt11").strip()
+            self.fh.write(nt + "\n")
+        except Exception:
+            log.exception("Failed to serialize ntriples.")

followthemoney/graph.py

@@ -0,0 +1,308 @@
+"""
+Converting FtM data to a property graph data model.
+
+This module provides an abstract data object that represents a property
+graph. This is used by the exporter modules to convert data
+to a specific output format, like Cypher or NetworkX.
+"""
+import logging
+from typing import Any, Dict, Generator, Iterable, List, Optional
+
+from followthemoney.types import registry
+from followthemoney.types.common import PropertyType
+from followthemoney.schema import Schema
+from followthemoney.proxy import EntityProxy
+from followthemoney.property import Property
+from followthemoney.exc import InvalidModel
+
+log = logging.getLogger(__name__)
+
+
+class Node(object):
+    """A node represents either an entity that can be rendered as a
+    node in a graph, or as a re-ified value, like a name, email
+    address or phone number."""
+
+    __slots__ = ["type", "value", "id", "proxy", "schema"]
+
+    def __init__(
+        self,
+        type_: PropertyType,
+        value: str,
+        proxy: Optional[EntityProxy] = None,
+        schema: Optional[Schema] = None,
+    ) -> None:
+        self.type = type_
+        self.value = value
+        # _id = type_.node_id_safe(value)
+        # if _id is None:
+        #     raise InvalidData("No ID for node")
+        self.id = type_.node_id_safe(value)
+        self.proxy = proxy
+        self.schema = schema if proxy is None else proxy.schema
+
+    @property
+    def is_entity(self) -> bool:
+        """Check to see if the node represents an entity. If this is false, the
+        node represents a non-entity property value that has been reified, like
+        a phone number or a name."""
+        return self.type == registry.entity
+
+    @property
+    def caption(self) -> str:
+        """A user-facing label for the current node."""
+        if self.type == registry.entity and self.proxy is not None:
+            return self.proxy.caption
+        caption = self.type.caption(self.value)
+        return caption or self.value
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Return a simple dictionary to reflect this graph node."""
+        return {
+            "id": self.id,
+            "type": self.type.name,
+            "value": self.value,
+            "caption": self.caption,
+        }
+
+    @classmethod
+    def from_proxy(cls, proxy: EntityProxy) -> "Node":
+        """For a given :class:`~followthemoney.proxy.EntityProxy`, return a node
+        based on the entity."""
+        return cls(registry.entity, proxy.id, proxy=proxy)
+
+    def __str__(self) -> str:
+        return self.caption
+
+    def __repr__(self) -> str:
+        return "<Node(%r, %r, %r)>" % (self.id, self.type, self.caption)
+
+    def __hash__(self) -> int:
+        return hash(self.id)
+
+    def __eq__(self, other: Any) -> bool:
+        return bool(self.id == other.id)
+
+
+class Edge(object):
+    """A link between two nodes."""
+
+    __slots__ = [
+        "id",
+        "weight",
+        "source_id",
+        "target_id",
+        "prop",
+        "proxy",
+        "schema",
+        "graph",
+    ]
+
+    def __init__(
+        self,
+        graph: "Graph",
+        source: Node,
+        target: Node,
+        proxy: Optional[EntityProxy] = None,
+        prop: Optional[Property] = None,
+        value: Optional[str] = None,
+    ):
+        self.graph = graph
+        self.id = f"{source.id}<>{target.id}"
+        self.source_id = source.id
+        self.target_id = target.id
+        self.weight = 1.0
+        self.prop = prop
+        self.proxy = proxy
+        self.schema: Optional[Schema] = None
+        if prop is not None and value is not None:
+            self.weight = prop.specificity(value)
+        if proxy is not None:
+            self.id = f"{source.id}<{proxy.id}>{target.id}"
+            self.schema = proxy.schema
+
+    @property
+    def source(self) -> Optional[Node]:
+        """The graph node from which the edge originates."""
+        if self.source_id is None:
+            return None
+        return self.graph.nodes.get(self.source_id)
+
+    @property
+    def source_prop(self) -> Property:
+        """Get the entity property originating this edge."""
+        if self.schema is not None and self.schema.source_prop is not None:
+            if self.schema.source_prop.reverse is not None:
+                return self.schema.source_prop.reverse
+        if self.prop is None:
+            raise InvalidModel("Contradiction: %r" % self)
+        return self.prop
+
+    @property
+    def target(self) -> Optional[Node]:
+        """The graph node to which the edge points."""
+        if self.target_id is None:
+            return None
+        return self.graph.nodes.get(self.target_id)
+
+    @property
+    def target_prop(self) -> Optional[Property]:
+        """Get the entity property originating this edge."""
+        if self.schema is not None and self.schema.target_prop is not None:
+            return self.schema.target_prop.reverse
+        if self.prop is not None:
+            return self.prop.reverse
+        # NOTE: this edge points at a value node.
+        return None
+
+    @property
+    def type_name(self) -> str:
+        """Return a machine-readable description of the type of the edge.
+        This is either a property name or a schema name."""
+        if self.schema is not None:
+            return self.schema.name
+        if self.prop is None:
+            raise InvalidModel("Invalid edge: %r" % self)
+        return self.prop.name
+
+    def to_dict(self) -> Dict[str, Optional[str]]:
+        return {
+            "id": self.id,
+            "source_id": self.source_id,
+            "target_id": self.target_id,
+            "type_name": self.type_name,
+        }
+
+    def __repr__(self) -> str:
+        return "<Edge(%r)>" % self.id
+
+    def __hash__(self) -> int:
+        return hash(self.id)
+
+    def __eq__(self, other: Any) -> bool:
+        return bool(self.id == other.id)
+
+
+class Graph(object):
+    """A set of nodes and edges, derived from entities and their properties.
+    This represents an alternative interpretation of FtM data as a property
+    graph.
+
+    This class is meant to be extensible in order to support additional
+    backends, like Aleph.
+    """
+
+    def __init__(self, edge_types: Iterable[PropertyType] = registry.pivots) -> None:
+        types = registry.get_types(edge_types)
+        self.edge_types = [t for t in types if t.matchable]
+        self.flush()
+
+    def flush(self) -> None:
+        """Remove all nodes, edges and proxies from the graph."""
+        self.edges: Dict[str, Edge] = {}
+        self.nodes: Dict[str, Node] = {}
+        self.proxies: Dict[str, Optional[EntityProxy]] = {}
+
+    def queue(self, id_: str, proxy: Optional[EntityProxy] = None) -> None:
+        """Register a reference to an entity in the graph."""
+        if id_ not in self.proxies or proxy is not None:
+            self.proxies[id_] = proxy
+
+    @property
+    def queued(self) -> List[str]:
+        """Return a list of all the entities which are referenced from the graph
+        but that haven't been loaded yet. This can be used to get a list of
+        entities that should be included to expand the whole graph by one degree.
+        """
+        return [i for (i, p) in self.proxies.items() if p is None]
+
+    def _get_node_stub(self, prop: Property, value: str) -> Node:
+        if prop.type == registry.entity:
+            self.queue(value)
+        node = Node(prop.type, value, schema=prop.range)
+        if node.id is None:
+            return node
+        if node.id not in self.nodes:
+            self.nodes[node.id] = node
+        return self.nodes[node.id]
+
+    def _add_edge(self, proxy: EntityProxy, source: str, target: str) -> None:
+        if proxy.schema.source_prop is None:
+            raise InvalidModel("Invalid edge entity: %r" % proxy)
+        source_node = self._get_node_stub(proxy.schema.source_prop, source)
+        if proxy.schema.target_prop is None:
+            raise InvalidModel("Invalid edge entity: %r" % proxy)
+        target_node = self._get_node_stub(proxy.schema.target_prop, target)
+        if source_node.id is not None and target_node.id is not None:
+            edge = Edge(self, source_node, target_node, proxy=proxy)
+            self.edges[edge.id] = edge
+
+    def _add_node(self, proxy: EntityProxy) -> None:
+        """Derive a node and its value edges from the given proxy."""
+        entity = Node.from_proxy(proxy)
+        if entity.id is not None:
+            self.nodes[entity.id] = entity
+        for prop, value in proxy.itervalues():
+            if prop.type not in self.edge_types:
+                continue
+            node = self._get_node_stub(prop, value)
+            if node.id is None:
+                continue
+            edge = Edge(self, entity, node, prop=prop, value=value)
+            if edge.weight > 0:
+                self.edges[edge.id] = edge
+
+    def add(self, proxy: EntityProxy) -> None:
+        """Add an :class:`~followthemoney.proxy.EntityProxy` to the graph and make
+        it either a :class:`~followthemoney.graph.Node` or an
+        :class:`~followthemoney.graph.Edge`."""
+        if proxy is None:
+            return
+        self.queue(proxy.id, proxy)
+        if proxy.schema.edge:
+            for (source, target) in proxy.edgepairs():
+                self._add_edge(proxy, source, target)
+        else:
+            self._add_node(proxy)
+
+    def iternodes(self) -> Iterable[Node]:
+        """Iterate all :class:`nodes <followthemoney.graph.Node>` in the graph."""
+        return self.nodes.values()
+
+    def iteredges(self) -> Iterable[Edge]:
+        """Iterate all :class:`edges <followthemoney.graph.Edge>` in the graph."""
+        return self.edges.values()
+
+    def get_outbound(
+        self, node: Node, prop: Optional[Property] = None
+    ) -> Generator[Edge, None, None]:
+        """Get all edges pointed out from the given node."""
+        for edge in self.iteredges():
+            if edge.source == node:
+                if prop and edge.source_prop != prop:
+                    continue
+                yield edge
+
+    def get_inbound(
+        self, node: Node, prop: Optional[Property] = None
+    ) -> Generator[Edge, None, None]:
+        """Get all edges pointed at the given node."""
+        for edge in self.iteredges():
+            if edge.target == node:
+                if prop and edge.target_prop != prop:
+                    continue
+                yield edge
+
+    def get_adjacent(
+        self, node: Node, prop: Optional[Property] = None
+    ) -> Generator[Edge, None, None]:
+        "Get all adjacent edges of the given node."
+        yield from self.get_outbound(node, prop=prop)
+        yield from self.get_inbound(node, prop=prop)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Return a dictionary with the graph nodes and edges."""
+        return {
+            "nodes": [n.to_dict() for n in self.iternodes()],
+            "edges": [e.to_dict() for e in self.iteredges()],
+        }