followthemoney 1.3.6__py3-none-any.whl → 3.8.0__py3-none-any.whl

followthemoney/mapping/entity.py

@@ -1,62 +1,111 @@
+import logging
 from hashlib import sha1
-from banal import ensure_list
+from warnings import warn
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set
+from banal import keys_values
+from normality import stringify
 
-from followthemoney.mapping.property import PropertyMapping
 from followthemoney.types import registry
 from followthemoney.util import key_bytes
+from followthemoney.proxy import EntityProxy
+from followthemoney.mapping.property import PropertyMapping
+from followthemoney.mapping.source import Record
 from followthemoney.exc import InvalidMapping
 
+if TYPE_CHECKING:
+    from followthemoney.model import Model
+    from followthemoney.mapping.query import QueryMapping
+
+log = logging.getLogger(__name__)
+
 
 class EntityMapping(object):
+    __slots__ = (
+        "model",
+        "name",
+        "seed",
+        "keys",
+        "id_column",
+        "schema",
+        "refs",
+        "dependencies",
+        "properties",
+    )
 
-    def __init__(self, model, query, name, data, key_prefix=None):
+    def __init__(
+        self,
+        model: "Model",
+        query: "QueryMapping",
+        name: str,
+        data: Dict[str, Any],
+        key_prefix: Optional[str] = None,
+    ) -> None:
         self.model = model
         self.name = name
-        self.data = data
 
         self.seed = sha1(key_bytes(key_prefix))
-        self.seed.update(key_bytes(data.get('key_literal')))
+        self.seed.update(key_bytes(data.get("key_literal")))
 
-        self.keys = ensure_list(data.get('key'))
-        self.keys.extend(ensure_list(data.get('keys')))
-        if not len(self.keys):
-            raise InvalidMapping("No keys: %r" % name)
+        self.keys = keys_values(data, "key", "keys")
+        self.id_column = stringify(data.get("id_column"))
+        if not len(self.keys) and self.id_column is None:
+            raise InvalidMapping("No keys or ID: %r" % name)
+        if len(self.keys) and self.id_column is not None:
+            msg = "Please use only keys or id_column, not both: %r" % name
+            raise InvalidMapping(msg)
 
-        self.schema = model.get(data.get('schema'))
-        if self.schema is None:
-            raise InvalidMapping("Invalid schema: %s" % data.get('schema'))
+        schema_name = stringify(data.get("schema"))
+        if schema_name is None:
+            raise InvalidMapping("No schema: %s" % name)
+        schema = model.get(schema_name)
+        if schema is None:
+            raise InvalidMapping("Invalid schema: %s" % schema_name)
+        if schema.deprecated:
+            warn(
+                "Mapping uses a deprecated schema: %r" % schema,
+                DeprecationWarning,
+                stacklevel=2,
+            )
+        self.schema = schema
 
         self.refs = set(self.keys)
-        self.dependencies = set()
-        self.properties = []
-        for name, prop in data.get('properties', {}).items():
-            prop_schema = self.schema.get(name)
-            if prop_schema is None:
+        if self.id_column:
+            self.refs.add(self.id_column)
+        self.dependencies: Set[str] = set()
+        self.properties: List[PropertyMapping] = []
+        for name, prop_mapping in data.get("properties", {}).items():
+            prop = self.schema.get(name)
+            if prop is None:
                 raise InvalidMapping("Invalid property: %s" % name)
-            prop = PropertyMapping(query, prop, prop_schema)
-            self.properties.append(prop)
-            self.refs.update(prop.refs)
-            if prop.entity:
-                self.dependencies.add(prop.entity)
+            mapping = PropertyMapping(query, prop_mapping, prop)
+            self.properties.append(mapping)
+            self.refs.update(mapping.refs)
+            if mapping.entity:
+                self.dependencies.add(mapping.entity)
 
-    def bind(self):
+    def bind(self) -> None:
         for prop in self.properties:
             prop.bind()
 
-    def compute_key(self, record):
+    def compute_key(self, record: Record) -> Optional[str]:
         """Generate a key for this entity, based on the given fields."""
+        if self.id_column is not None:
+            return record.get(self.id_column)
         values = [key_bytes(record.get(k)) for k in self.keys]
         digest = self.seed.copy()
+        has_value = False
         for value in sorted(values):
-            digest.update(value)
-        if digest.digest() != self.seed.digest():
+            if len(value):
+                has_value = True
+                digest.update(value)
+        if has_value:
             return digest.hexdigest()
+        return None
 
-    def map(self, record, entities):
+    def map(
+        self, record: Record, entities: Dict[str, EntityProxy]
+    ) -> Optional[EntityProxy]:
         proxy = self.model.make_entity(self.schema)
-        proxy.id = self.compute_key(record)
-        if proxy.id is None:
-            return
 
         # THIS IS HACKY
         # Some of the converters, e.g. for phone numbers, work better if they
@@ -64,23 +113,46 @@ class EntityMapping(object):
         # detail, we are first running country fields, then making the data
         # from that accessible to phone and address parsers.
         for prop in self.properties:
-            if prop.schema.type == registry.country:
-                proxy.add(prop.schema, prop.map(record, entities))
+            if prop.prop.type == registry.country:
+                discarded_values = prop.map(proxy, record, entities)
+                for value in discarded_values:
+                    log.warning(
+                        f'[{self.name}] Discarded unclean value "{value}" for property "{prop.prop.qname}".'
+                    )
 
         for prop in self.properties:
-            if prop.schema.type != registry.country:
-                proxy.add(prop.schema, prop.map(record, entities,
-                                                countries=proxy.countries))
+            if prop.prop.type != registry.country:
+                discarded_values = prop.map(proxy, record, entities)
+                for value in discarded_values:
+                    log.warning(
+                        f'[{self.name}] Discarding unclean value "{value}" for property "{prop.prop.qname}".'
+                    )
+
+        # Generate the ID at the end to avoid self-reference checks on empty
+        # keys.
+        proxy.id = self.compute_key(record)
+        if proxy.id is None:
+            if self.id_column:
+                log.warning(
+                    f'[{self.name}] Skipping entity because no ID could be computed. Make sure that there are no empty values in the "{self.id_column}" column.'
+                )
+            if self.keys:
+                log.warning(
+                    f"[{self.name}] Skipping entity because no ID could be computed. Make sure that there are no empty values in key columns."
+                )
+            return None
 
         for prop in self.properties:
-            if prop.required:
-                if not len(proxy.get(prop.schema)):
-                    # This is a bit weird, it flags fields to be required in
-                    # the mapping, not in the model. Basically it means: if
-                    # this row of source data doesn't have that field, then do
-                    # not map it again.
-                    return
+            if prop.required and not proxy.has(prop.prop):
+                # This is a bit weird, it flags fields to be required in
+                # the mapping, not in the model. Basically it means: if
+                # this row of source data doesn't have that field, then do
+                # not map it again.
+                log.warning(
+                    f'[{self.name}] Skipping entity because required property "{prop.prop.name}" is empty.'
+                )
+                return None
         return proxy
 
-    def __repr__(self):
-        return '<EntityMapping(%r)>' % self.name
+    def __repr__(self) -> str:
+        return "<EntityMapping(%r)>" % self.name

followthemoney/mapping/property.py

@@ -1,46 +1,76 @@
 import re
 from copy import deepcopy
+from warnings import warn
 from normality import stringify
-from banal import unique_list, ensure_list
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, cast
+from banal import keys_values, as_bool
 
+from followthemoney.helpers import inline_names
 from followthemoney.exc import InvalidMapping
+from followthemoney.proxy import EntityProxy
+from followthemoney.util import sanitize_text
+from followthemoney.property import Property
+from followthemoney.mapping.source import Record
+
+if TYPE_CHECKING:
+    from followthemoney.mapping.query import QueryMapping
 
 
 class PropertyMapping(object):
     """Map values from a given record (e.g. a CSV row or SQL result) to the
     schema form."""
-    FORMAT_PATTERN = re.compile('{{([^(}})]*)}}')
 
-    def __init__(self, query, data, schema):
+    __slots__ = (
+        "query",
+        "prop",
+        "refs",
+        "join",
+        "split",
+        "entity",
+        "format",
+        "fuzzy",
+        "required",
+        "literals",
+        "template",
+        "replacements",
+    )
+
+    FORMAT_PATTERN = re.compile("{{([^(}})]*)}}")
+
+    def __init__(
+        self, query: "QueryMapping", data: Dict[str, Any], prop: Property
+    ) -> None:
         self.query = query
         data = deepcopy(data)
-        self.data = data
-        self.schema = schema
-        self.name = schema.name
-        self.type = schema.type
-
-        self.refs = ensure_list(data.pop('column', []))
-        self.refs.extend(ensure_list(data.pop('columns', [])))
-
-        self.literals = ensure_list(data.pop('literal', []))
-        self.literals.extend(ensure_list(data.pop('literals', [])))
-
-        self.join = data.pop('join', None)
-        self.split = data.pop('split', None)
-        self.entity = data.pop('entity', None)
-        self.required = data.pop('required', False)
-
-        self.template = stringify(data.pop('template', None))
-        self.replacements = {}
+        self.prop = prop
+
+        self.refs = cast(List[str], keys_values(data, "column", "columns"))
+        self.join = cast(Optional[str], data.pop("join", None))
+        self.split = cast(Optional[str], data.pop("split", None))
+        self.entity = stringify(data.pop("entity", None))
+        self.format = stringify(data.pop("format", None))
+        self.fuzzy = as_bool(data.pop("fuzzy", False))
+        self.required = as_bool(data.pop("required", False))
+        self.literals = cast(List[str], keys_values(data, "literal", "literals"))
+
+        self.template = sanitize_text(data.pop("template", None))
+        self.replacements: Dict[str, str] = {}
         if self.template is not None:
             # this is hacky, trying to generate refs from template
             for ref in self.FORMAT_PATTERN.findall(self.template):
                 self.refs.append(ref)
-                self.replacements['{{%s}}' % ref] = ref
+                self.replacements["{{%s}}" % ref] = ref
 
-    def bind(self):
-        if self.schema.stub:
-            raise InvalidMapping("Property for [%s] is a stub" % self.name)
+    def bind(self) -> None:
+        if self.prop.stub:
+            raise InvalidMapping("Property for [%r] is a stub" % self.prop)
+
+        if self.prop.deprecated:
+            warn(
+                "Mapping uses a deprecated property: %r" % self.prop,
+                DeprecationWarning,
+                stacklevel=2,
+            )
 
         if self.entity is None:
             return
@@ -52,44 +82,47 @@ class PropertyMapping(object):
         for entity in self.query.entities:
             if entity.name != self.entity:
                 continue
-            if not entity.schema.is_a(self.schema.range):
-                raise InvalidMapping("The entity [%s] must be a %s (not %s)" %
-                                     (self.name, self.schema.range, entity.schema.name))  # noqa
+            if not self.prop.range or not entity.schema.is_a(self.prop.range):
+                raise InvalidMapping(
+                    "The entity [%r] must be a %s (not %s)"
+                    % (self.prop, self.prop.range, entity.schema.name)
+                )  # noqa
             return
 
-        raise InvalidMapping("No entity [%s] for property [%s]"
-                             % (self.entity, self.name))
+        raise InvalidMapping(
+            "No entity [%s] for property [%r]" % (self.entity, self.prop)
+        )
 
-    def record_values(self, record):
+    def record_values(self, record: Record) -> List[str]:
         if self.template is not None:
             # replace mentions of any refs with the values present in the
             # current record
             value = self.template
             for repl, ref in self.replacements.items():
-                ref_value = record.get(ref) or ''
+                ref_value = record.get(ref) or ""
                 value = value.replace(repl, ref_value)
             return [value.strip()]
 
         values = list(self.literals)
-        values.extend([record.get(r) for r in self.refs])
+        for ref in self.refs:
+            rec_value = record.get(ref)
+            if rec_value is not None:
+                values.append(rec_value)
         return values
 
-    def map(self, record, entities, **kwargs):
-        kwargs.update(self.data)
-
+    def map(
+        self, proxy: EntityProxy, record: Record, entities: Dict[str, EntityProxy]
+    ) -> List[str]:
         if self.entity is not None:
             entity = entities.get(self.entity)
             if entity is not None:
-                return [entity.id]
+                proxy.unsafe_add(self.prop, entity.id, cleaned=True)
+                inline_names(proxy, entity)
             return []
 
         # clean the values returned by the query, or by using literals, or
         # formats.
-        values = []
-        for value in self.record_values(record):
-            value = self.type.clean(value, **kwargs)
-            if value is not None:
-                values.append(value)
+        values: List[str] = self.record_values(record)
 
         if self.join is not None:
             values = [self.join.join(values)]
@@ -97,7 +130,20 @@ class PropertyMapping(object):
         if self.split is not None:
             splote = []
             for value in values:
-                splote = splote + value.split(self.split)
+                splote.extend(value.split(self.split))
             values = splote
 
-        return unique_list(values)
+        discarded_values: List[str] = []
+
+        for value in values:
+            added_value = proxy.unsafe_add(
+                prop=self.prop,
+                value=value,
+                fuzzy=self.fuzzy,
+                format=self.format,
+            )
+
+            if value is not None and added_value is None:
+                discarded_values.append(value)
+
+        return discarded_values

followthemoney/mapping/query.py

@@ -1,20 +1,27 @@
+from followthemoney.mapping.source import Record, Source
+from typing import TYPE_CHECKING, Any, List, Optional, Set, Dict
+
+from followthemoney.proxy import EntityProxy
 from followthemoney.mapping.entity import EntityMapping
 from followthemoney.mapping.sql import SQLSource
 from followthemoney.mapping.csv import CSVSource
 from followthemoney.exc import InvalidMapping
 
+if TYPE_CHECKING:
+    from followthemoney.model import Model
 
-class QueryMapping(object):
 
-    def __init__(self, model, data, key_prefix=None):
-        self.model = model
-        self.data = data
+class QueryMapping:
+    __slots__ = ("model", "data", "refs", "entities", "source")
 
-        self.refs = set()
-        self.entities = []
-        for name, data in data.get('entities', {}).items():
-            entity = EntityMapping(model, self, name, data,
-                                   key_prefix=key_prefix)
+    def __init__(
+        self, model: "Model", data: Dict[str, Any], key_prefix: Optional[str] = None
+    ) -> None:
+        self.model = model
+        self.refs: Set[str] = set()
+        self.entities: List[EntityMapping] = []
+        for name, edata in data.get("entities", {}).items():
+            entity = EntityMapping(model, self, name, edata, key_prefix=key_prefix)
 
             self.entities.append(entity)
             self.refs.update(entity.refs)
@@ -32,7 +39,7 @@ class QueryMapping(object):
         # in dependent entities.
         entities = self.entities
         self.entities = []
-        resolved = set()
+        resolved: Set[str] = set()
         while len(entities) > 0:
             before = len(entities)
             for entity in entities:
@@ -44,16 +51,17 @@ class QueryMapping(object):
             if before == len(entities):
                 raise InvalidMapping("Circular entity dependency detected.")
 
-    @property
-    def source(self):
-        if 'database' in self.data:
-            return SQLSource(self, self.data)
-        elif 'csv_url' in self.data or 'csv_urls' in self.data:
-            return CSVSource(self, self.data)
-        raise InvalidMapping("Cannot determine mapping type")
+        self.source = self._get_source(data)
+
+    def _get_source(self, data: Dict[str, Any]) -> Source:
+        if "database" in data:
+            return SQLSource(self, data)
+        if "csv_url" in data or "csv_urls" in data:
+            return CSVSource(self, data)
+        raise InvalidMapping("Cannot determine mapping type: %r" % data)
 
-    def map(self, record):
-        data = {}
+    def map(self, record: Record) -> Dict[str, EntityProxy]:
+        data: Dict[str, EntityProxy] = {}
         for entity in self.entities:
             mapped = entity.map(record, data)
             if mapped is not None:

followthemoney/mapping/source.py

@@ -1,11 +1,21 @@
+from typing import TYPE_CHECKING, Any, Dict, Generator, Optional, Set, cast
 
+if TYPE_CHECKING:
+    from followthemoney.mapping.query import QueryMapping
+
+Filter = Set[Optional[str]]
+Record = Dict[str, str]
 
-class Source(object):
 
-    def __init__(self, query, data):
+class Source(object):
+    def __init__(self, query: "QueryMapping", data: Dict[str, Any]) -> None:
         self.query = query
-        self.filters = query.data.get('filters', {}).items()
-        self.filters_not = query.data.get('filters_not', {}).items()
+        self.filters = cast(Dict[str, Any], data.get("filters", {})).items()
+        self.filters_not = cast(Dict[str, Any], data.get("filters_not", {})).items()
+
+    @property
+    def records(self) -> Generator[Record, None, None]:
+        raise NotImplementedError
 
-    def __len__(self):
+    def __len__(self) -> int:
         return 0