followthemoney 1.3.7__py3-none-any.whl → 3.8.0__py3-none-any.whl

followthemoney/helpers.py

@@ -0,0 +1,212 @@
+# This module violates the boundary between the role of code and
+# YAML in the rest of followthemoney. It handles normalisations
+# which would be much harder to express in abstract, especially
+# those thet simplify the data based on their pragmatics.
+#
+# If anyone were to swap out the default model, this would
+# probably be the first place to break.
+from os.path import splitext
+from typing import Iterable, List, Optional, Set
+from normality import safe_filename
+from mimetypes import guess_extension
+from itertools import product
+from datetime import datetime, timedelta
+
+from followthemoney.types import registry
+from followthemoney.proxy import E
+from followthemoney.util import join_text
+
+PROV_MIN_DATES = ("createdAt", "authoredAt", "publishedAt")
+PROV_MAX_DATES = ("modifiedAt", "retrievedAt")
+
+
+def remove_checksums(proxy: E) -> E:
+    """When accepting entities via a web API, it would consistute
+    a security risk to allow a user to submit checksum-type properties.
+    These can be traded in for access to said files if they exist in the
+    underlying content-addressed storage. It seems safest to just remove
+    all checksums from entities when they are untrusted user input."""
+    for prop in proxy.iterprops():
+        if prop.type == registry.checksum:
+            proxy.pop(prop)
+    return proxy
+
+
+def simplify_provenance(proxy: E) -> E:
+    """If there are multiple dates given for some of the provenance
+    fields, we can logically conclude which one is the most meaningful."""
+    for prop_name in PROV_MAX_DATES:
+        values = proxy.pop(prop_name, quiet=True)
+        if len(values):
+            proxy.set(prop_name, max(values), cleaned=True)
+    for prop_name in PROV_MIN_DATES:
+        values = proxy.pop(prop_name, quiet=True)
+        if len(values):
+            proxy.set(prop_name, min(values), cleaned=True)
+    return proxy
+
+
+def entity_filename(
+    proxy: E, base_name: Optional[str] = None, extension: Optional[str] = None
+) -> Optional[str]:
+    """Derive a safe filename for the given entity."""
+    if proxy.schema.is_a("Document"):
+        for extension_ in proxy.get("extension", quiet=True):
+            if extension is not None:
+                break
+            extension = extension_
+        for file_name in proxy.get("fileName", quiet=True):
+            base_name_, extension_ = splitext(file_name)
+            if base_name is None and len(base_name_):
+                base_name = base_name_
+            if extension is None and len(extension_):
+                extension = extension_
+        for mime_type in proxy.get("mimeType", quiet=True):
+            if extension is not None:
+                break
+            extension = guess_extension(mime_type)
+    base_name = base_name or proxy.id
+    return safe_filename(base_name, extension=extension)
+
+
+def name_entity(entity: E) -> E:
+    """If an entity has multiple names, pick the most central one
+    and set all the others as aliases. This is awkward given that
+    names are not special and may not always be the caption."""
+    if entity.schema.is_a("Thing"):
+        names = entity.get("name")
+        if len(names) > 1:
+            name = registry.name.pick(names)
+            if name in names:
+                names.remove(name)
+            entity.set("name", name)
+            entity.add("alias", names)
+    return entity
+
+
+def check_person_cutoff(
+    entity: E,
+    death_cutoff: datetime = datetime(2000, 1, 1),
+    birth_cutoff: Optional[datetime] = None,
+) -> bool:
+    """Check if a person has been dead long enough to not be relevant for
+    investigations any more."""
+    if not entity.schema.is_a("Person"):
+        return False
+    death_dates = entity.get("deathDate", quiet=True)
+    death_cutoff_ = death_cutoff.isoformat()
+    if len(death_dates) and max(death_dates) < death_cutoff_:
+        return True
+    birth_dates = entity.get("birthDate", quiet=True)
+    if birth_cutoff is None:
+        birth_cutoff = death_cutoff - timedelta(days=100 * 365)
+    birth_cutoff_ = birth_cutoff.isoformat()
+    if len(birth_dates) and min(birth_dates) < birth_cutoff_:
+        return True
+    return False
+
+
+def remove_prefix_dates(entity: E) -> E:
+    """If an entity has multiple values for a date field, you may
+    want to remove all those that are prefixes of others. For example,
+    if a Person has both a birthDate of 1990 and of 1990-05-01, we'd
+    want to drop the mention of 1990."""
+    for prop in entity.iterprops():
+        if prop.type == registry.date:
+            values = remove_prefix_date_values(entity.get(prop))
+            entity.set(prop, values)
+    return entity
+
+
+def remove_prefix_date_values(values: Iterable[str]) -> List[str]:
+    """See ``remove_prefix_dates``."""
+    kept: List[str] = []
+    values = sorted(values, key=len, reverse=True)
+    for index, value in enumerate(values):
+        keep = True
+        for longer in values[:index]:
+            if longer.startswith(value):
+                keep = False
+                break
+        if keep:
+            kept.append(value)
+    return kept
+
+
+def inline_names(entity: E, related: E) -> None:
+    """Attempt to solve a weird UI problem. Imagine we are showing a list of
+    payments between a sender and a beneficiary to a user. They may now conduct
+    a search for a term present in the sender or recipient name, but there will
+    be no result, because the name is only indexed with the parties, but not in
+    the payment. This is part of a partial work-around to that.
+
+    This is really bad in theory, but really useful in practice. Shoot me.
+    """
+    prop = entity.schema.get("namesMentioned")
+    if prop is not None:
+        entity.add(prop, related.get_type_values(registry.name))
+
+
+def combine_names(entity: E) -> E:
+    """This function will try to build names from name parts provided as part
+    of a person entity. This is of course impossible to do culturally correctly
+    for the whole planet at once, so it should be mostly used for internal-facing
+    (e.g. matching) processes."""
+    if entity.schema.is_a("Person"):
+        first_names = entity.get("firstName")
+        second_names = entity.get("secondName")
+        second_names.append("")
+        middle_names = entity.get("middleName")
+        middle_names.append("")
+        father_names = entity.get("fatherName")
+        father_names.append("")
+        last_names = entity.get("lastName")
+        for (first, second, middle, father, last) in product(
+            first_names, second_names, middle_names, father_names, last_names
+        ):
+            name = join_text(first, second, middle, father, last)
+            if name is not None:
+                entity.add("alias", name)
+
+        # If no first name is given, at least add the last name:
+        if not entity.get_type_values(registry.name) and len(last_names):
+            entity.add("alias", last_names)
+    return entity
+
+
+def dates_years(dates: Iterable[Optional[str]]) -> Set[str]:
+    """Get the unique years from a set of date strings."""
+    cleaned: Set[str] = set()
+    for date in dates:
+        if date is not None:
+            cleaned.add(date[:4])
+    return cleaned
+
+
+def post_summary(
+    organization: str,
+    role: Optional[str],
+    start_dates: Iterable[Optional[str]],
+    end_dates: Iterable[Optional[str]],
+    dates: Iterable[Optional[str]],
+) -> str:
+    """Make a string summary for a Post object."""
+    position = organization
+    start = min(dates_years(start_dates), default="")
+    end = min(dates_years(end_dates), default="")
+    date_range = None
+    if len(start) or len(end):
+        date_range = f"{start}-{end}"
+    dates_ = dates_years(dates)
+    if date_range is None and len(dates_):
+        date_range = ", ".join(sorted(dates_))
+
+    bracketed = None
+    if date_range and role:
+        bracketed = f"{role}, {date_range}"
+    else:
+        bracketed = role or date_range
+
+    if bracketed:
+        position = f"{position} ({bracketed})"
+    return position

followthemoney/mapping/__init__.py

@@ -1,3 +1,3 @@
 from followthemoney.mapping.query import QueryMapping
 
-__all__ = [QueryMapping]
+__all__ = ["QueryMapping"]

followthemoney/mapping/csv.py

@@ -1,65 +1,97 @@
 import io
 import os
 import logging
+from banal.lists import ensure_list
 import requests
 from csv import DictReader
-from banal import ensure_list
-from normality import stringify
+from urllib.parse import urlparse
+from banal import keys_values
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Generator,
+    ItemsView,
+    Iterable,
+    List,
+    Optional,
+    Set,
+    Tuple,
+    cast,
+)
 
-from followthemoney.mapping.source import Source
+from followthemoney.mapping.source import Record, Source
+from followthemoney.util import sanitize_text
 from followthemoney.exc import InvalidMapping
 
+if TYPE_CHECKING:
+    from followthemoney.mapping.query import QueryMapping
+
 log = logging.getLogger(__name__)
+FilterList = List[Tuple[str, Set[Optional[str]]]]
 
 
 class CSVSource(Source):
     """Special case for entity loading directly from a CSV URL"""
 
-    def __init__(self, query, data):
-        super(CSVSource, self).__init__(query, data)
-        urls = ensure_list(data.get('csv_url'))
-        urls.extend(ensure_list(data.get('csv_urls')))
-        self.urls = set()
-        for url in urls:
-            self.urls.add(os.path.expandvars(url))
+    def __init__(self, query: "QueryMapping", data: Dict[str, Any]) -> None:
+        super().__init__(query, data)
+        self.urls: Set[str] = set()
+        for url in keys_values(data, "csv_url", "csv_urls"):
+            self.urls.add(cast(str, os.path.expandvars(url)))
 
         if not len(self.urls):
             raise InvalidMapping("No CSV URLs are specified.")
 
-    def read_csv(self, url):
-        parsed_url = requests.utils.urlparse(url)
+        self.filters_set = self._parse_filters(self.filters)
+        self.filters_not_set = self._parse_filters(self.filters_not)
+
+    def _parse_filters(self, filters: ItemsView[str, Any]) -> FilterList:
+        filters_set: FilterList = []
+        for (key, value) in filters:
+            values = set(cast(List[Optional[str]], ensure_list(value)))
+            filters_set.append((key, values))
+        return filters_set
+
+    def check_filters(self, data: Record) -> bool:
+        for (k, v) in self.filters_set:
+            if data.get(k) not in v:
+                return False
+        for (k, v) in self.filters_not_set:
+            if data.get(k) in v:
+                return False
+        return True
+
+    @classmethod
+    def read_csv(cls, fh: Iterable[str]) -> Generator[Record, None, None]:
+        for row in DictReader(fh, skipinitialspace=True):
+            data: Record = {}
+            for ref, ref_value in row.items():
+                value = sanitize_text(ref_value)
+                if value is not None:
+                    data[ref] = value
+            yield data
+
+    def read_csv_url(self, url: str) -> Generator[Record, None, None]:
+        parsed_url = urlparse(url)
         log.info("Loading: %s", url)
-        if parsed_url.scheme in ['http', 'https']:
+        if parsed_url.scheme in ["http", "https"]:
             res = requests.get(url, stream=True)
             if not res.ok:
                 raise InvalidMapping("Failed to open CSV: %s" % url)
             # if res.encoding is None:
-            res.encoding = 'utf-8'
+            res.encoding = "utf-8"
             # log.info("Detected encoding: %s", res.encoding)
             lines = res.iter_lines(decode_unicode=True)
-            for row in DictReader(lines, skipinitialspace=True):
-                yield row
+            yield from self.read_csv(lines)
         else:
-            with io.open(parsed_url.path, 'r') as fh:
-                for row in DictReader(fh, skipinitialspace=True):
-                    yield row
-
-    def check_filters(self, data):
-        for (k, v) in self.filters:
-            if v != data.get(k):
-                return False
-        for (k, v) in self.filters_not:
-            if v == data.get(k):
-                return False
-        return True
+            with io.open(parsed_url.path, "r") as fh:
+                yield from self.read_csv(fh)
 
     @property
-    def records(self):
+    def records(self) -> Generator[Record, None, None]:
         """Iterate through the table applying filters on-the-go."""
         for url in self.urls:
-            for row in self.read_csv(url):
-                data = {}
-                for ref in self.query.refs:
-                    data[ref] = stringify(row.get(ref))
-                if self.check_filters(data):
-                    yield data
+            for record in self.read_csv_url(url):
+                if self.check_filters(record):
+                    yield record

followthemoney/mapping/entity.py

@@ -1,62 +1,111 @@
+import logging
 from hashlib import sha1
-from banal import ensure_list
+from warnings import warn
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set
+from banal import keys_values
+from normality import stringify
 
-from followthemoney.mapping.property import PropertyMapping
 from followthemoney.types import registry
 from followthemoney.util import key_bytes
+from followthemoney.proxy import EntityProxy
+from followthemoney.mapping.property import PropertyMapping
+from followthemoney.mapping.source import Record
 from followthemoney.exc import InvalidMapping
 
+if TYPE_CHECKING:
+    from followthemoney.model import Model
+    from followthemoney.mapping.query import QueryMapping
+
+log = logging.getLogger(__name__)
+
 
 class EntityMapping(object):
+    __slots__ = (
+        "model",
+        "name",
+        "seed",
+        "keys",
+        "id_column",
+        "schema",
+        "refs",
+        "dependencies",
+        "properties",
+    )
 
-    def __init__(self, model, query, name, data, key_prefix=None):
+    def __init__(
+        self,
+        model: "Model",
+        query: "QueryMapping",
+        name: str,
+        data: Dict[str, Any],
+        key_prefix: Optional[str] = None,
+    ) -> None:
         self.model = model
         self.name = name
-        self.data = data
 
         self.seed = sha1(key_bytes(key_prefix))
-        self.seed.update(key_bytes(data.get('key_literal')))
+        self.seed.update(key_bytes(data.get("key_literal")))
 
-        self.keys = ensure_list(data.get('key'))
-        self.keys.extend(ensure_list(data.get('keys')))
-        if not len(self.keys):
-            raise InvalidMapping("No keys: %r" % name)
+        self.keys = keys_values(data, "key", "keys")
+        self.id_column = stringify(data.get("id_column"))
+        if not len(self.keys) and self.id_column is None:
+            raise InvalidMapping("No keys or ID: %r" % name)
+        if len(self.keys) and self.id_column is not None:
+            msg = "Please use only keys or id_column, not both: %r" % name
+            raise InvalidMapping(msg)
 
-        self.schema = model.get(data.get('schema'))
-        if self.schema is None:
-            raise InvalidMapping("Invalid schema: %s" % data.get('schema'))
+        schema_name = stringify(data.get("schema"))
+        if schema_name is None:
+            raise InvalidMapping("No schema: %s" % name)
+        schema = model.get(schema_name)
+        if schema is None:
+            raise InvalidMapping("Invalid schema: %s" % schema_name)
+        if schema.deprecated:
+            warn(
+                "Mapping uses a deprecated schema: %r" % schema,
+                DeprecationWarning,
+                stacklevel=2,
+            )
+        self.schema = schema
 
         self.refs = set(self.keys)
-        self.dependencies = set()
-        self.properties = []
-        for name, prop in data.get('properties', {}).items():
-            prop_schema = self.schema.get(name)
-            if prop_schema is None:
+        if self.id_column:
+            self.refs.add(self.id_column)
+        self.dependencies: Set[str] = set()
+        self.properties: List[PropertyMapping] = []
+        for name, prop_mapping in data.get("properties", {}).items():
+            prop = self.schema.get(name)
+            if prop is None:
                 raise InvalidMapping("Invalid property: %s" % name)
-            prop = PropertyMapping(query, prop, prop_schema)
-            self.properties.append(prop)
-            self.refs.update(prop.refs)
-            if prop.entity:
-                self.dependencies.add(prop.entity)
+            mapping = PropertyMapping(query, prop_mapping, prop)
+            self.properties.append(mapping)
+            self.refs.update(mapping.refs)
+            if mapping.entity:
+                self.dependencies.add(mapping.entity)
 
-    def bind(self):
+    def bind(self) -> None:
         for prop in self.properties:
             prop.bind()
 
-    def compute_key(self, record):
+    def compute_key(self, record: Record) -> Optional[str]:
         """Generate a key for this entity, based on the given fields."""
+        if self.id_column is not None:
+            return record.get(self.id_column)
         values = [key_bytes(record.get(k)) for k in self.keys]
         digest = self.seed.copy()
+        has_value = False
         for value in sorted(values):
-            digest.update(value)
-        if digest.digest() != self.seed.digest():
+            if len(value):
+                has_value = True
+                digest.update(value)
+        if has_value:
             return digest.hexdigest()
+        return None
 
-    def map(self, record, entities):
+    def map(
+        self, record: Record, entities: Dict[str, EntityProxy]
+    ) -> Optional[EntityProxy]:
         proxy = self.model.make_entity(self.schema)
-        proxy.id = self.compute_key(record)
-        if proxy.id is None:
-            return
 
         # THIS IS HACKY
         # Some of the converters, e.g. for phone numbers, work better if they
@@ -64,23 +113,46 @@ class EntityMapping(object):
         # detail, we are first running country fields, then making the data
         # from that accessible to phone and address parsers.
         for prop in self.properties:
-            if prop.schema.type == registry.country:
-                proxy.add(prop.schema, prop.map(record, entities))
+            if prop.prop.type == registry.country:
+                discarded_values = prop.map(proxy, record, entities)
+                for value in discarded_values:
+                    log.warning(
+                        f'[{self.name}] Discarded unclean value "{value}" for property "{prop.prop.qname}".'
+                    )
 
         for prop in self.properties:
-            if prop.schema.type != registry.country:
-                proxy.add(prop.schema, prop.map(record, entities,
-                                                countries=proxy.countries))
+            if prop.prop.type != registry.country:
+                discarded_values = prop.map(proxy, record, entities)
+                for value in discarded_values:
+                    log.warning(
+                        f'[{self.name}] Discarding unclean value "{value}" for property "{prop.prop.qname}".'
+                    )
+
+        # Generate the ID at the end to avoid self-reference checks on empty
+        # keys.
+        proxy.id = self.compute_key(record)
+        if proxy.id is None:
+            if self.id_column:
+                log.warning(
+                    f'[{self.name}] Skipping entity because no ID could be computed. Make sure that there are no empty values in the "{self.id_column}" column.'
+                )
+            if self.keys:
+                log.warning(
+                    f"[{self.name}] Skipping entity because no ID could be computed. Make sure that there are no empty values in key columns."
+                )
+            return None
 
         for prop in self.properties:
-            if prop.required:
-                if not len(proxy.get(prop.schema)):
-                    # This is a bit weird, it flags fields to be required in
-                    # the mapping, not in the model. Basically it means: if
-                    # this row of source data doesn't have that field, then do
-                    # not map it again.
-                    return
+            if prop.required and not proxy.has(prop.prop):
+                # This is a bit weird, it flags fields to be required in
+                # the mapping, not in the model. Basically it means: if
+                # this row of source data doesn't have that field, then do
+                # not map it again.
+                log.warning(
+                    f'[{self.name}] Skipping entity because required property "{prop.prop.name}" is empty.'
+                )
+                return None
         return proxy
 
-    def __repr__(self):
-        return '<EntityMapping(%r)>' % self.name
+    def __repr__(self) -> str:
+        return "<EntityMapping(%r)>" % self.name