followthemoney 1.3.7__py3-none-any.whl → 3.8.1__py3-none-any.whl

followthemoney/__init__.py

@@ -3,12 +3,14 @@ import os
 from followthemoney.model import Model
 from followthemoney.util import set_model_locale
 
-__version__ = '1.3.7'
+__version__ = "3.8.1"
+
 
 model_path = os.path.dirname(__file__)
-model_path = os.path.join(model_path, 'schema')
+model_path = os.path.join(model_path, "schema")
+model_path = os.environ.get("FTM_MODEL_PATH", model_path)
 
 # Data model singleton
 model = Model(model_path)
 
-__all__ = [model, set_model_locale]
+__all__ = ["model", "set_model_locale"]

followthemoney/cli/__init__.py

@@ -0,0 +1,17 @@
+import sys
+
+
+def load_entry_points() -> None:
+    if sys.version_info[0] >= 3 and sys.version_info[1] >= 10:
+        from importlib.metadata import entry_points
+
+        for ep in entry_points().select(group="followthemoney.cli"):
+            ep.load()
+    else:
+        from pkg_resources import iter_entry_points
+
+        for ep_ in iter_entry_points("followthemoney.cli"):
+            ep_.load()
+
+
+load_entry_points()

followthemoney/cli/aggregate.py

@@ -0,0 +1,56 @@
+import click
+from pathlib import Path
+from typing import Dict, Optional, Type
+
+from followthemoney.proxy import EntityProxy, E
+from followthemoney.namespace import Namespace
+from followthemoney.cli.cli import cli
+from followthemoney.cli.util import InPath, OutPath, path_entities
+from followthemoney.cli.util import path_writer, write_entity
+
+
+def sorted_aggregate(path: Path, outpath: Path, entity_type: Type[E]) -> None:
+    """Aggregate entities based on the premise that the fragments in the source
+    stream are sorted by their ID."""
+    entity: Optional[E] = None
+    with path_writer(outpath) as outfh:
+        for next_entity in path_entities(path, entity_type=entity_type):
+            if entity is None:
+                entity = next_entity
+                continue
+            if next_entity.id == entity.id:
+                entity = entity.merge(next_entity)
+                continue
+            write_entity(outfh, entity)
+            entity = next_entity
+
+        if entity is not None:
+            write_entity(outfh, entity)
+
+
+@cli.command("aggregate", help="Aggregate multiple fragments of entities")
+@click.option("-i", "--infile", type=InPath, default="-")
+@click.option("-o", "--outfile", type=OutPath, default="-")
+def aggregate(infile: Path, outfile: Path) -> None:
+    buffer: Dict[str, EntityProxy] = {}
+    namespace = Namespace(None)
+    try:
+        with path_writer(outfile) as outfh:
+            for entity in path_entities(infile, EntityProxy):
+                entity = namespace.apply(entity)
+                if entity.id in buffer:
+                    buffer[entity.id].merge(entity)
+                else:
+                    buffer[entity.id] = entity
+
+            for entity in buffer.values():
+                write_entity(outfh, entity)
+    except BrokenPipeError:
+        raise click.Abort()
+
+
+@cli.command("sorted-aggregate", help="Aggregate sorted fragments of entities")
+@click.option("-i", "--infile", type=InPath, default="-")
+@click.option("-o", "--outfile", type=OutPath, default="-")
+def sorted_aggregate_(infile: Path, outfile: Path) -> None:
+    sorted_aggregate(infile, outfile, EntityProxy)

followthemoney/cli/cli.py

@@ -0,0 +1,88 @@
+import sys
+import click
+import orjson
+import logging
+from pathlib import Path
+from typing import Optional, BinaryIO, List, Any, Dict
+from banal import ensure_list
+
+from followthemoney import model
+from followthemoney.namespace import Namespace
+from followthemoney.cli.util import InPath, OutPath, path_entities
+from followthemoney.cli.util import path_writer, write_entity
+from followthemoney.proxy import EntityProxy
+
+
+@click.group(help="Utility for FollowTheMoney graph data")
+def cli() -> None:
+    fmt = "%(name)s [%(levelname)s] %(message)s"
+    logging.basicConfig(stream=sys.stderr, level=logging.INFO, format=fmt)
+
+
+@cli.command("dump-model", help="Export the current schema model")
+@click.option("-o", "--outfile", type=click.File("wb"), default="-")
+def dump_model(outfile: BinaryIO) -> None:
+    f = orjson.OPT_INDENT_2 | orjson.OPT_SORT_KEYS
+    outfile.write(orjson.dumps(model.to_dict(), option=f))
+
+
+@cli.command("validate", help="Re-parse and validate the given data")
+@click.option("-i", "--infile", type=InPath, default="-")
+@click.option("-o", "--outfile", type=OutPath, default="-")
+def validate(infile: Path, outfile: Path) -> None:
+    try:
+        with path_writer(outfile) as outfh:
+            for entity in path_entities(infile, EntityProxy, cleaned=False):
+                clean = model.make_entity(entity.schema)
+                clean.id = entity.id
+                for prop, value in entity.itervalues():
+                    clean.add(prop, value)
+                write_entity(outfh, clean)
+    except BrokenPipeError:
+        raise click.Abort()
+
+
+@cli.command("import-vis", help="Load a .VIS file and get entities")
+@click.option("-i", "--infile", type=InPath, default="-")  # noqa
+@click.option("-o", "--outfile", type=OutPath, default="-")  # noqa
+def import_vis(infile: Path, outfile: Path) -> None:
+    with path_writer(outfile) as outfh:
+        with open(infile, "rb") as infh:
+            data: Dict[str, Any] = orjson.loads(infh.read())
+            if "entities" in data:
+                entities: List[Dict[str, Any]] = data.get("entities", data)
+            elif "layout" in data:
+                entities = data.get("layout", {}).get("entities", data)
+            else:
+                raise click.ClickException("No entities found in VIS file")
+            for entity_data in ensure_list(entities):
+                entity = EntityProxy.from_dict(model, entity_data)
+                write_entity(outfh, entity)
+
+
+@cli.command("sign", help="Apply a HMAC signature to entity IDs")
+@click.option("-i", "--infile", type=InPath, default="-")  # noqa
+@click.option("-o", "--outfile", type=OutPath, default="-")  # noqa
+@click.option("-s", "--signature", default=None, help="HMAC signature key")  # noqa
+def sign(infile: Path, outfile: Path, signature: Optional[str]) -> None:
+    ns = Namespace(signature)
+    try:
+        with path_writer(outfile) as outfh:
+            for entity in path_entities(infile, EntityProxy):
+                signed = ns.apply(entity)
+                write_entity(outfh, signed)
+    except BrokenPipeError:
+        raise click.Abort()
+
+
+@cli.command(help="Format a stream of entities to make it readable")
+@click.option("-i", "--infile", type=InPath, default="-")  # noqa
+def pretty(infile: Path) -> None:
+    stdout = click.get_binary_stream("stdout")
+    try:
+        f = orjson.OPT_INDENT_2 | orjson.OPT_APPEND_NEWLINE
+        for entity in path_entities(infile, EntityProxy):
+            data = orjson.dumps(entity.to_dict(), option=f)
+            stdout.write(data)
+    except BrokenPipeError:
+        raise click.Abort()

followthemoney/cli/exports.py

@@ -0,0 +1,121 @@
+import click
+from typing import List, TextIO, Generator
+from pathlib import Path
+from contextlib import contextmanager
+
+from followthemoney.cli.cli import cli
+from followthemoney.cli.util import InPath, OutPath, export_stream
+from followthemoney.export.csv import CSVExporter
+from followthemoney.export.rdf import RDFExporter
+from followthemoney.export.excel import ExcelExporter
+from followthemoney.export.graph import edge_types, DEFAULT_EDGE_TYPES
+from followthemoney.export.graph import NXGraphExporter
+from followthemoney.export.neo4j import Neo4JCSVExporter
+from followthemoney.export.neo4j import CypherGraphExporter
+
+
+@contextmanager
+def text_out(path: Path) -> Generator[TextIO, None, None]:
+    if str(path) == "-":
+        yield click.get_text_stream("stdout")
+        return
+    with open(path, "w") as fh:
+        yield fh
+
+
+@cli.command("export-csv", help="Export to CSV")
+@click.option("-i", "--infile", type=InPath, default="-")  # noqa
+@click.option(
+    "-o",
+    "--outdir",
+    type=click.Path(file_okay=False, writable=True, path_type=Path),
+    default=".",
+    help="output directory",
+)
+def export_csv(infile: Path, outdir: Path) -> None:
+    exporter = CSVExporter(outdir)
+    export_stream(exporter, infile)
+
+
+@cli.command("export-excel", help="Export to Excel")
+@click.option("-i", "--infile", type=InPath, default="-")
+@click.option(
+    "-o",
+    "--outfile",
+    type=click.Path(dir_okay=False, writable=True, path_type=Path),
+    required=True,
+)
+def export_excel(infile: Path, outfile: Path) -> None:
+    exporter = ExcelExporter(outfile)
+    export_stream(exporter, infile)
+
+
+@cli.command("export-rdf", help="Export to RDF NTriples")
+@click.option("-i", "--infile", type=InPath, default="-")
+@click.option("-o", "--outfile", type=OutPath, default="-")
+@click.option(
+    "--qualified/--unqualified",
+    is_flag=True,
+    default=True,
+    help="Generate full predicates",
+)
+def export_rdf(infile: Path, outfile: Path, qualified: bool = True) -> None:
+    with text_out(outfile) as fh:
+        exporter = RDFExporter(fh, qualified=qualified)
+        export_stream(exporter, infile)
+
+
+@cli.command("export-gexf", help="Export to GEXF (Gephi) format")
+@click.option("-i", "--infile", type=InPath, default="-")
+@click.option("-o", "--outfile", type=OutPath, default="-")
+@click.option(
+    "-e",
+    "--edge-types",
+    type=click.Choice(edge_types()),
+    multiple=True,
+    default=DEFAULT_EDGE_TYPES,
+    help="Property types to be reified into graph edges.",
+)
+def export_gexf(infile: Path, outfile: Path, edge_types: List[str]) -> None:
+    with text_out(outfile) as fh:
+        exporter = NXGraphExporter(fh, edge_types=edge_types)
+        export_stream(exporter, infile)
+
+
+@cli.command("export-cypher", help="Export to Cypher script")
+@click.option("-i", "--infile", type=InPath, default="-")  # noqa
+@click.option("-o", "--outfile", type=OutPath, default="-")  # noqa
+@click.option(
+    "-e",
+    "--edge-types",
+    type=click.Choice(edge_types()),
+    multiple=True,
+    default=DEFAULT_EDGE_TYPES,
+    help="Property types to be reified into graph edges.",
+)
+def export_cypher(infile: Path, outfile: Path, edge_types: List[str]) -> None:
+    with text_out(outfile) as fh:
+        exporter = CypherGraphExporter(fh, edge_types=edge_types)
+        export_stream(exporter, infile)
+
+
+@cli.command("export-neo4j-bulk", help="Export to Neo4J bulk import")
+@click.option("-i", "--infile", type=InPath, default="-")  # noqa
+@click.option(
+    "-o",
+    "--outdir",
+    type=click.Path(file_okay=False, writable=True, path_type=Path),
+    required=True,
+    help="Output directory for Neo4J import script",
+)
+@click.option(
+    "-e",
+    "--edge-types",
+    type=click.Choice(edge_types()),
+    multiple=True,
+    default=DEFAULT_EDGE_TYPES,
+    help="Property types to be reified into graph edges.",
+)
+def export_neo4j_bulk(infile: Path, outdir: Path, edge_types: List[str]) -> None:
+    exporter = Neo4JCSVExporter(outdir, edge_types=edge_types)
+    export_stream(exporter, infile)

followthemoney/cli/mapping.py

@@ -0,0 +1,85 @@
+import sys
+import click
+from pathlib import Path
+from banal import keys_values
+from typing import Generator, List, TextIO, Tuple
+from contextlib import contextmanager
+
+from followthemoney import model
+from followthemoney.namespace import Namespace
+from followthemoney.mapping.query import QueryMapping
+from followthemoney.mapping.csv import CSVSource
+from followthemoney.cli.cli import cli
+from followthemoney.cli.util import InPath, OutPath, load_mapping_file
+from followthemoney.cli.util import path_writer, write_entity
+
+
+@contextmanager
+def input_file(path: Path) -> Generator[TextIO, None, None]:
+    if str(path) == "-":
+        yield sys.stdin
+        return
+    with open(path, "r") as fh:
+        yield fh
+
+
+@cli.command("map", help="Execute a mapping file and emit objects")
+@click.option("-o", "--outfile", type=OutPath, default="-")
+@click.option(
+    "--sign/--no-sign",
+    is_flag=True,
+    default=True,
+    help="Apply HMAC signature",
+)
+@click.argument("mapping_yaml", type=click.Path(exists=True, path_type=Path))
+def run_mapping(outfile: Path, mapping_yaml: Path, sign: bool = True) -> None:
+    config = load_mapping_file(mapping_yaml)
+    try:
+        with path_writer(outfile) as outfh:
+            for dataset, meta in config.items():
+                ns = Namespace(dataset)
+                for mapping in keys_values(meta, "queries", "query"):
+                    entities = model.map_entities(mapping, key_prefix=dataset)
+                    for entity in entities:
+                        if sign:
+                            entity = ns.apply(entity)
+                        write_entity(outfh, entity)
+    except BrokenPipeError:
+        raise click.Abort()
+    except Exception as exc:
+        raise click.ClickException(str(exc))
+
+
+@cli.command("map-csv", help="Map CSV data from stdin and emit objects")
+@click.option("-i", "--infile", type=InPath, default="-")
+@click.option("-o", "--outfile", type=OutPath, default="-")
+@click.option(
+    "--sign/--no-sign", is_flag=True, default=True, help="Apply HMAC signature"
+)
+@click.argument("mapping_yaml", type=click.Path(exists=True, path_type=Path))
+def stream_mapping(
+    infile: Path, outfile: Path, mapping_yaml: Path, sign: bool = True
+) -> None:
+    queries: List[Tuple[str, QueryMapping]] = []
+    config = load_mapping_file(mapping_yaml)
+    for dataset, meta in config.items():
+        for data in keys_values(meta, "queries", "query"):
+            data.pop("database", None)
+            data["csv_url"] = "/dev/null"
+            query = model.make_mapping(data, key_prefix=dataset)
+            queries.append((dataset, query))
+
+    try:
+        with path_writer(outfile) as outfh:
+            with input_file(infile) as fh:
+                for record in CSVSource.read_csv(fh):
+                    for (dataset, query) in queries:
+                        ns = Namespace(dataset)
+                        if query.source.check_filters(record):  # type: ignore
+                            entities = query.map(record)
+                            for entity in entities.values():
+                                if sign:
+                                    entity = ns.apply(entity)
+                                write_entity(outfh, entity)
+    except BrokenPipeError:
+        raise click.Abort()

followthemoney/cli/sieve.py

@@ -0,0 +1,67 @@
+import click
+from pathlib import Path
+from typing import Iterable, Optional
+
+from followthemoney import model
+from followthemoney.proxy import E, EntityProxy
+from followthemoney.types import registry
+from followthemoney.cli.cli import cli
+from followthemoney.cli.util import InPath, OutPath, path_entities
+from followthemoney.cli.util import path_writer, write_entity
+
+
+def sieve_entity(
+    entity: EntityProxy,
+    schemata: Iterable[str],
+    properties: Iterable[str],
+    types: Iterable[str],
+) -> Optional[EntityProxy]:
+    for schema in schemata:
+        if entity.schema.is_a(schema):
+            return None
+    for prop in entity.iterprops():
+        if prop.name in properties or prop.qname in properties:
+            entity.pop(prop, quiet=True)
+        elif prop.type.name in types:
+            entity.pop(prop, quiet=True)
+    return entity
+
+
+@cli.command("sieve", help="Filter out parts of entities.")
+@click.option("-i", "--infile", type=InPath, default="-")
+@click.option("-o", "--outfile", type=OutPath, default="-")
+@click.option(
+    "-s",
+    "--schema",
+    type=click.Choice(list(model.schemata.keys())),
+    multiple=True,
+    help="Filter out the given schemata.",
+)
+@click.option(
+    "-p",
+    "--property",
+    multiple=True,
+    help="Filter out the given property names.",
+)
+@click.option(
+    "-t",
+    "--type",
+    type=click.Choice([t.name for t in registry.types]),
+    multiple=True,
+    help="Filter out the given property types.",
+)
+def sieve(
+    infile: Path,
+    outfile: Path,
+    schema: Iterable[str],
+    property: Iterable[str],
+    type: Iterable[str],
+) -> None:
+    try:
+        with path_writer(outfile) as outfh:
+            for entity in path_entities(infile, EntityProxy):
+                sieved = sieve_entity(entity, schema, property, type)
+                if sieved is not None:
+                    write_entity(outfh, sieved)
+    except BrokenPipeError:
+        raise click.Abort()

followthemoney/cli/util.py

@@ -0,0 +1,142 @@
+from contextlib import contextmanager
+import os
+import json
+import yaml
+import click
+import orjson
+from pathlib import Path
+from warnings import warn
+from typing import Any, BinaryIO, Generator, Optional, TextIO, Type
+from banal import is_mapping, is_listish, ensure_list
+
+from followthemoney import model
+from followthemoney.export.common import Exporter
+from followthemoney.proxy import E, EntityProxy
+from followthemoney.util import MEGABYTE, PathLike
+
+MAX_LINE = 200 * MEGABYTE
+InPath = click.Path(dir_okay=False, readable=True, path_type=Path, allow_dash=True)
+OutPath = click.Path(dir_okay=False, writable=True, path_type=Path, allow_dash=True)
+
+
+def write_object(stream: TextIO, obj: Any) -> None:
+    warn("write_object() is deprecated.", DeprecationWarning, stacklevel=2)
+    if hasattr(obj, "to_dict"):
+        obj = obj.to_dict()
+    data = json.dumps(obj)
+    stream.write(data + "\n")
+
+
+def write_entity(fh: BinaryIO, entity: E) -> None:
+    data = entity.to_dict()
+    entity_id = data.pop("id")
+    assert entity_id is not None, data
+    sort_data = dict(id=entity_id)
+    sort_data.update(data)
+    out = orjson.dumps(sort_data, option=orjson.OPT_APPEND_NEWLINE)
+    fh.write(out)
+
+
+def _read_one(data: Any, cleaned: bool = True) -> Generator[EntityProxy, None, None]:
+    if is_mapping(data) and "schema" in data:
+        yield model.get_proxy(data, cleaned=cleaned)
+
+
+def read_entities(
+    stream: TextIO, cleaned: bool = True, max_line: int = MAX_LINE
+) -> Generator[EntityProxy, None, None]:
+    warn("read_entities() is deprecated.", DeprecationWarning, stacklevel=2)
+    while True:
+        line = stream.readline(max_line)
+        if not line:
+            return
+        data = json.loads(line)
+        entities = ensure_list(data)
+        if is_mapping(data):
+            if "entities" in data:
+                entities = data.get("entities", data)
+            if "layout" in data:
+                entities = data.get("layout", {}).get("entities", data)
+        for entity in ensure_list(entities):
+            yield from _read_one(entity, cleaned=cleaned)
+
+
+def read_entity(
+    stream: TextIO, cleaned: bool = True, max_line: int = MAX_LINE
+) -> Optional[Any]:
+    warn("read_entity() is deprecated.", DeprecationWarning, stacklevel=2)
+    line = stream.readline(max_line)
+    if not line:
+        return None
+    data = json.loads(line)
+    for entity in _read_one(data, cleaned=cleaned):
+        return entity
+    return data
+
+
+def binary_entities(
+    fh: BinaryIO, entity_type: Type[E], cleaned: bool = True, max_line: int = MAX_LINE
+) -> Generator[E, None, None]:
+    while line := fh.readline(max_line):
+        data = orjson.loads(line)
+        yield entity_type.from_dict(model, data, cleaned=cleaned)
+
+
+def path_entities(
+    path: PathLike,
+    entity_type: Type[E],
+    cleaned: bool = True,
+    max_line: int = MAX_LINE,
+) -> Generator[E, None, None]:
+    if str(path) == "-":
+        fh = click.get_binary_stream("stdin")
+        yield from binary_entities(fh, entity_type, cleaned=cleaned, max_line=max_line)
+        return
+    with open(path, "rb") as fh:
+        yield from binary_entities(fh, entity_type, cleaned=cleaned, max_line=max_line)
+
+
+@contextmanager
+def path_writer(path: PathLike) -> Generator[BinaryIO, None, None]:
+    """Open a file for writing binary content, or use stdout."""
+    if str(path) == "-":
+        yield click.get_binary_stream("stdout")
+        return
+    with open(path, "wb") as fh:
+        yield fh
+
+
+def export_stream(exporter: Exporter, path: Path) -> None:
+    try:
+        for entity in path_entities(path, EntityProxy):
+            exporter.write(entity)
+    except BrokenPipeError:
+        raise click.Abort()
+    finally:
+        exporter.finalize()
+
+
+def load_mapping_file(file_path: PathLike) -> Any:
+    """Load a YAML (or JSON) bulk load mapping file."""
+    file_path = os.path.abspath(file_path)
+    with open(file_path, "r") as fh:
+        data = yaml.safe_load(fh) or {}
+    return resolve_includes(file_path, data)
+
+
+def resolve_includes(file_path: PathLike, data: Any) -> Any:
+    """Handle include statements in the graph configuration file.
+
+    This allows the YAML graph configuration to be broken into
+    multiple smaller fragments that are easier to maintain."""
+    if is_listish(data):
+        return [resolve_includes(file_path, i) for i in data]
+    if is_mapping(data):
+        include_paths = ensure_list(data.pop("include", []))
+        for include_path in include_paths:
+            dir_prefix = os.path.dirname(file_path)
+            include_path = os.path.join(dir_prefix, include_path)
+            data.update(load_mapping_file(include_path))
+        for key, value in data.items():
+            data[key] = resolve_includes(file_path, value)
+    return data