followthemoney 1.3.6__py3-none-any.whl → 3.8.0__py3-none-any.whl

followthemoney/mapping/sql.py

@@ -1,18 +1,24 @@
 import os
-import six
 import logging
 from uuid import uuid4
-from banal import ensure_list
-from normality import stringify
-from sqlalchemy import create_engine, MetaData
-from sqlalchemy import select, func
-# from sqlalchemy import text as sql_text
+from typing import TYPE_CHECKING, Any, Dict, Generator, List, Optional, Union, cast
+from banal import ensure_list, is_listish, keys_values
+from sqlalchemy import MetaData, func
+from sqlalchemy.future import select
+from sqlalchemy.engine import Engine, create_engine
+from sqlalchemy.sql.elements import Label
 from sqlalchemy.pool import NullPool
 from sqlalchemy.schema import Table
+from sqlalchemy.sql.expression import Select
 
-from followthemoney.mapping.source import Source
+from followthemoney.mapping.source import Record, Source
+from followthemoney.util import sanitize_text
 from followthemoney.exc import InvalidMapping
 
+if TYPE_CHECKING:
+    from followthemoney.mapping.query import QueryMapping
+
+
 log = logging.getLogger(__name__)
 DATA_PAGE = 1000
 
@@ -20,20 +26,22 @@ DATA_PAGE = 1000
 class QueryTable(object):
     """A table to be joined in."""
 
-    def __init__(self, query, data):
-        self.query = query
-        if isinstance(data, six.string_types):
-            data = {'table': data}
-        self.data = data
-        self.table_ref = data.get('table')
-        self.alias_ref = data.get('alias', self.table_ref)
-        self.table = Table(self.table_ref, self.query.meta, autoload=True)
-        self.alias = self.table.alias(self.alias_ref)
-
-        self.refs = {}
+    def __init__(
+        self, meta: MetaData, engine: Engine, data: Union[str, Dict[str, str]]
+    ) -> None:
+        if isinstance(data, str):
+            data = {"table": data}
+        table_ref = data.get("table")
+        if table_ref is None:
+            raise InvalidMapping("Query has no table!")
+        alias_ref = data.get("alias", table_ref)
+        self.table = Table(table_ref, meta, autoload_with=engine)
+        self.alias = self.table.alias(alias_ref)
+
+        self.refs: Dict[str, Label[Any]] = {}
         for column in self.alias.columns:
-            name = '%s.%s' % (self.alias_ref, column.name)
-            labeled_column = column.label('col_%s' % uuid4().hex[:10])
+            name = "%s.%s" % (alias_ref, column.name)
+            labeled_column = column.label("col_%s" % uuid4().hex[:10])
             self.refs[name] = labeled_column
             self.refs[column.name] = labeled_column
 
@@ -41,70 +49,76 @@ class QueryTable(object):
 class SQLSource(Source):
     """Query mapper for loading data from a SQL query."""
 
-    def __init__(self, query, data):
+    def __init__(self, query: "QueryMapping", data: Dict[str, Any]) -> None:
         super(SQLSource, self).__init__(query, data)
-        self.database_uri = os.path.expandvars(data.get('database'))
-        kwargs = {}
-        if self.database_uri.lower().startswith('postgres'):
-            kwargs['server_side_cursors'] = True
-        self.engine = create_engine(self.database_uri,
-                                    poolclass=NullPool,
-                                    **kwargs)
+        database = data.get("database")
+        if database is None:
+            raise InvalidMapping("No database in SQL mapping!")
+        self.database_uri = cast(str, os.path.expandvars(database))
+        self.engine = create_engine(self.database_uri, poolclass=NullPool)
         self.meta = MetaData()
-        self.meta.bind = self.engine
 
-        tables = ensure_list(data.get('table'))
-        tables.extend(ensure_list(data.get('tables')))
-        self.tables = [QueryTable(self, f) for f in tables]
-        self.joins = ensure_list(data.get('joins'))
+        tables = keys_values(data, "table", "tables")
+        self.tables = [QueryTable(self.meta, self.engine, f) for f in tables]
+        self.joins = cast(List[Dict[str, str]], ensure_list(data.get("joins")))
 
-    def get_column(self, ref):
+    def get_column(self, ref: Optional[str]) -> Label[Any]:
         for table in self.tables:
             if ref in table.refs:
-                return table.refs.get(ref)
+                return table.refs[ref]
         raise InvalidMapping("Missing reference: %s" % ref)
 
-    def apply_filters(self, q):
+    def apply_filters(self, q: Select) -> Select:
         for col, val in self.filters:
-            q = q.where(self.get_column(col) == val)
+            if is_listish(val):
+                q = q.where(self.get_column(col).in_(val))
+            else:
+                q = q.where(self.get_column(col) == val)
         for col, val in self.filters_not:
-            q = q.where(self.get_column(col) != val)
+            if is_listish(val):
+                q = q.where(self.get_column(col).notin_(val))
+            else:
+                q = q.where(self.get_column(col) != val)
         # not sure this is a great idea:
         # if self.data.get('where'):
         #    q = q.where(sql_text(self.data.get('where')))
         for join in self.joins:
-            left = self.get_column(join.get('left'))
-            right = self.get_column(join.get('right'))
+            left = self.get_column(join.get("left"))
+            right = self.get_column(join.get("right"))
             q = q.where(left == right)
         return q
 
-    def compose_query(self):
-        from_clause = [t.alias for t in self.tables]
+    def compose_query(self) -> Select:
         columns = [self.get_column(r) for r in self.query.refs]
-        q = select(columns=columns, from_obj=from_clause, use_labels=True)
+        q = select(*columns)
+        q = q.select_from(*[t.alias for t in self.tables])
         return self.apply_filters(q)
 
     @property
-    def records(self):
+    def records(self) -> Generator[Record, None, None]:
         """Compose the actual query and return an iterator of ``Record``."""
         mapping = [(r, self.get_column(r).name) for r in self.query.refs]
         q = self.compose_query()
         log.info("Query: %s", q)
-        rp = self.engine.execute(q)
-        while True:
-            rows = rp.fetchmany(size=DATA_PAGE)
-            if not len(rows):
-                break
-            for row in rows:
-                data = {}
-                for ref, name in mapping:
-                    data[ref] = stringify(row[name])
-                yield data
-
-    def __len__(self):
-        from_clause = [t.alias for t in self.tables]
-        columns = [func.count('*')]
-        q = select(columns=columns, from_obj=from_clause, use_labels=True)
+        with self.engine.connect() as conn:
+            rp = conn.execution_options(stream_results=True).execute(q)
+            while True:
+                rows = rp.fetchmany(size=DATA_PAGE)
+                if not len(rows):
+                    break
+                for row in rows:
+                    row_map = row._mapping
+                    data: Record = {}
+                    for ref, name in mapping:
+                        value = sanitize_text(row_map[name])
+                        if value is not None:
+                            data[ref] = value
+                    yield data
+
+    def __len__(self) -> int:
+        q = select(func.count("*"))
+        q = q.select_from(*[t.alias for t in self.tables])
         q = self.apply_filters(q)
-        rp = self.engine.execute(q)
-        return rp.scalar()
+        with self.engine.connect() as conn:
+            rp = conn.execute(q)
+            return int(rp.scalar() or 0)

followthemoney/messages.py

@@ -1,19 +1,25 @@
-import six
 import yaml
+from typing import Any, Dict, Generator, List, TextIO, Tuple
 
+Message = Tuple[Any, Any, List[str], List[str]]
 
-def extract_object(data, path):
+
+def extract_object(
+    data: Dict[str, Any], path: List[str]
+) -> Generator[Message, None, None]:
     for key, value in data.items():
         subpath = path + [key]
-        if isinstance(value, six.string_types):
-            if key in ['label', 'reverse', 'description', 'plural']:
-                comment = '.'.join(subpath)
+        if isinstance(value, str):
+            if key in ["label", "reverse", "description", "plural"]:
+                comment = ".".join(subpath)
                 yield (None, None, [value], [comment])
         if isinstance(value, dict):
             for res in extract_object(value, subpath):
                 yield res
 
 
-def extract_yaml(fileobj, keywords, comment_tags, options):
-    data = yaml.load(fileobj)
+def extract_yaml(
+    fileobj: TextIO, keywords: Any, comment_tags: Any, options: Any
+) -> Generator[Message, None, None]:
+    data = yaml.safe_load(fileobj)
     return extract_object(data, [])

followthemoney/model.py

@@ -1,66 +1,109 @@
 import os
 import yaml
+from typing import Any, Dict, Generator, Iterator, Optional, Set, TypedDict, Union
 
-from followthemoney.schema import Schema
+from followthemoney.types import registry
+from followthemoney.types.common import PropertyType, PropertyTypeToDict
+from followthemoney.schema import Schema, SchemaToDict
+from followthemoney.property import Property
 from followthemoney.mapping import QueryMapping
 from followthemoney.proxy import EntityProxy
 from followthemoney.exc import InvalidModel, InvalidData
 
 
+class ModelToDict(TypedDict):
+    schemata: Dict[str, SchemaToDict]
+    types: Dict[str, PropertyTypeToDict]
+
+
 class Model(object):
-    """A collection of schemata."""
+    """A collection of all the schemata available in followthemoney. The model
+    provides some helper functions to find schemata, properties or to instantiate
+    entity proxies based on the schema metadata."""
+
+    __slots__ = ("path", "schemata", "properties", "qnames")
 
-    def __init__(self, path):
+    def __init__(self, path: str) -> None:
         self.path = path
-        self.schemata = {}
+
+        #: A mapping with all schemata, organised by their name.
+        self.schemata: Dict[str, Schema] = {}
+
+        #: All properties defined in the model.
+        self.properties: Set[Property] = set()
+        self.qnames: Dict[str, Property] = {}
         for (path, _, filenames) in os.walk(self.path):
             for filename in filenames:
                 self._load(os.path.join(path, filename))
         self.generate()
 
-    def generate(self):
-        self.properties = set()
+    def generate(self) -> None:
+        """Loading the model is a weird process because the schemata reference
+        each other in complex ways, so the generation process cannot be fully
+        run as schemata are being instantiated. Hence this process needs to be
+        called once all schemata are loaded to finalise dereferencing the
+        schemata."""
         for schema in self:
-            schema.generate()
-            for prop in schema.properties.values():
-                self.properties.add(prop)
-
-    def _load(self, filepath):
-        with open(filepath, 'r') as fh:
-            data = yaml.load(fh)
+            schema.generate(self)
+        for prop in self.properties:
+            self.qnames[prop.qname] = prop
+            for schema in prop.schema.descendants:
+                if prop.name not in schema.properties:
+                    schema.properties[prop.name] = prop
+
+    def _load(self, filepath: str) -> None:
+        with open(filepath, "r", encoding="utf-8") as fh:
+            data = yaml.safe_load(fh)
             if not isinstance(data, dict):
-                raise InvalidModel('Model file is not a mapping.')
+                raise InvalidModel("Model file is not a mapping: %s" % filepath)
             for name, config in data.items():
                 self.schemata[name] = Schema(self, name, config)
 
-    def get(self, name):
-        if isinstance(name, Schema):
-            return name
-        return self.schemata.get(name)
+    def get(self, name: Union[str, Schema]) -> Optional[Schema]:
+        """Get a schema object based on a schema name. If the input is already
+        a schema object, it will just be returned."""
+        if isinstance(name, str):
+            return self.schemata.get(name)
+        return name
 
-    def get_qname(self, qname):
-        if not hasattr(self, '_qnames'):
-            self._qnames = {p.qname: p for p in self.properties}
-        return self._qnames.get(qname)
+    def get_qname(self, qname: str) -> Optional[Property]:
+        """Get a property object based on a qualified name (i.e. schema:property)."""
+        return self.qnames.get(qname)
 
-    def __getitem__(self, name):
+    def __getitem__(self, name: str) -> Schema:
+        """Same as get(), but throws an exception when the given name does not exist."""
         schema = self.get(name)
         if schema is None:
             raise KeyError("No such schema: %s" % name)
         return schema
 
-    def make_mapping(self, mapping, key_prefix=None):
+    def get_type_schemata(self, type_: PropertyType) -> Set[Schema]:
+        """Return all the schemata which have a property of the given type."""
+        schemata = set()
+        for schema in self.schemata.values():
+            for prop in schema.properties.values():
+                if prop.type == type_:
+                    schemata.add(schema)
+        return schemata
+
+    def make_mapping(
+        self, mapping: Dict[str, Any], key_prefix: Optional[str] = None
+    ) -> QueryMapping:
         """Parse a mapping that applies (tabular) source data to the model."""
         return QueryMapping(self, mapping, key_prefix=key_prefix)
 
-    def map_entities(self, mapping, key_prefix=None):
+    def map_entities(
+        self, mapping: Dict[str, Any], key_prefix: Optional[str] = None
+    ) -> Generator[EntityProxy, None, None]:
         """Given a mapping, yield a series of entities from the data source."""
-        mapping = self.make_mapping(mapping, key_prefix=key_prefix)
-        for record in mapping.source.records:
-            for entity in mapping.map(record).values():
+        gen = self.make_mapping(mapping, key_prefix=key_prefix)
+        for record in gen.source.records:
+            for entity in gen.map(record).values():
                 yield entity
 
-    def common_schema(self, left, right):
+    def common_schema(
+        self, left: Union[str, Schema], right: Union[str, Schema]
+    ) -> Schema:
         """Select the most narrow of two schemata.
 
         When indexing data from a dataset, an entity may be declared as a
@@ -68,33 +111,42 @@ class Model(object):
         will select the most specific of two schemata offered. In the example,
         that would be Person.
         """
-        left = self.get(left) or self.get(right)
-        right = self.get(right) or self.get(left)
-        left_schemata = list(left.schemata)
-        right_schemata = list(right.schemata)
-        if right in left_schemata:
-            return left
-        if left in right_schemata:
-            return right
-
-        # Find a common ancestor:
-        for left in left_schemata:
-            for right in right_schemata:
-                if left == right:
-                    return left
-
-        msg = "No common ancestor: %s and %s"
+        left_schema = self.get(left) or self.get(right)
+        right_schema = self.get(right) or self.get(left)
+        if left_schema is None or right_schema is None:
+            raise InvalidData("Invalid schema")
+        if left_schema.is_a(right_schema):
+            return left_schema
+        if right_schema.is_a(left_schema):
+            return right_schema
+        # for schema in self.schemata.values():
+        #     if schema.is_a(left) and schema.is_a(right):
+        #         return schema
+        msg = "No common schema: %s and %s"
         raise InvalidData(msg % (left, right))
 
-    def make_entity(self, schema, key_prefix=None):
-        schema = self.get(schema)
-        return EntityProxy(schema, None, None, key_prefix=key_prefix)
-
-    def get_proxy(self, data):
-        return EntityProxy.from_dict(self, data)
-
-    def to_dict(self):
-        return {n: s.to_dict() for (n, s) in self.schemata.items()}
-
-    def __iter__(self):
+    def make_entity(
+        self, schema: Union[str, Schema], key_prefix: Optional[str] = None
+    ) -> EntityProxy:
+        """Instantiate an empty entity proxy of the given schema type."""
+        return EntityProxy(self, {"schema": schema}, key_prefix=key_prefix)
+
+    def get_proxy(self, data: Dict[str, Any], cleaned: bool = True) -> EntityProxy:
+        """Create an entity proxy to reflect the entity data in the given
+        dictionary. If ``cleaned`` is disabled, all property values are
+        fully re-validated and normalised. Use this if handling input data
+        from an untrusted source."""
+        if isinstance(data, EntityProxy):
+            return data
+        return EntityProxy.from_dict(self, data, cleaned=cleaned)
+
+    def to_dict(self) -> ModelToDict:
+        """Return metadata for all schemata and properties, in a serializable form."""
+        return {
+            "schemata": {s.name: s.to_dict() for s in self.schemata.values()},
+            "types": {t.name: t.to_dict() for t in registry.types},
+        }
+
+    def __iter__(self) -> Iterator[Schema]:
+        """Iterate across all schemata."""
         return iter(self.schemata.values())

followthemoney/namespace.py

@@ -0,0 +1,119 @@
+"""
+*We like our abstractions like our offshore banks: leaky.*
+
+Entity ID namespaces are a security mechanism related to the Aleph search index.
+
+Aleph allows the user (via mappings or the API) to create arbitrary entity IDs.
+Entity IDs that are controlled by the user and not the system are unusual.
+However, this makes it possible to generate bulk data outside Aleph,
+and then load entities into the system as a continuous :ref:`streams`.
+
+The problem is that having user controlled entity IDs increases the chance
+of conflict in the search index.
+
+Namespacing works around this by making each entity ID consist of two parts:
+one controlled by the client, the other controlled by the system. The second
+part of the ID is called its `signature`::
+
+    entity_id.a40a29300ac6bb79dd2f911e77bbda7a3b502126
+
+The signature is generated as ``hmac(entity_id, dataset_id)``. This guarantees
+that the combined ID is specific to a dataset, without needing an (expensive)
+index look up of each ID first. It can also be generated on the client or
+the server without compromising isolation.
+"""
+import hmac
+from typing import Any, Optional, Tuple, Union
+
+from followthemoney.types import registry
+from followthemoney.proxy import E
+from followthemoney.util import key_bytes, get_entity_id
+
+
+class Namespace(object):
+    """Namespaces are used to partition entity IDs into different units,
+    which traditionally represent a dataset, collection or source.
+
+    See module docstring for details."""
+
+    SEP = "."
+
+    def __init__(self, name: Optional[str] = None) -> None:
+        self.bname = key_bytes(name) if name else b""
+        self.hmac = hmac.new(self.bname, digestmod="sha1")
+
+    @classmethod
+    def parse(cls, entity_id: str) -> Tuple[Optional[str], Optional[str]]:
+        """Split up an entity ID into the plain ID and the namespace
+        signature. If either part is missing, return None instead."""
+        clean_id = registry.entity.clean(entity_id)
+        if clean_id is None:
+            return (None, None)
+        try:
+            plain_id, checksum = clean_id.rsplit(cls.SEP, 1)
+            return (plain_id, checksum)
+        except ValueError:
+            return (clean_id, None)
+
+    @classmethod
+    def strip(cls, entity_id: str) -> Optional[str]:
+        plain_id, _ = cls.parse(entity_id)
+        return plain_id
+
+    def signature(self, entity_id: str) -> Optional[str]:
+        """Generate a namespace-specific signature."""
+        if not len(self.bname) or entity_id is None:
+            return None
+        digest = self.hmac.copy()
+        digest.update(key_bytes(entity_id))
+        return digest.hexdigest()
+
+    def sign(self, entity_id: str) -> Optional[str]:
+        """Apply a namespace signature to an entity ID, removing any
+        previous namespace marker."""
+        parsed_id, _ = self.parse(entity_id)
+        if not len(self.bname):
+            return parsed_id
+        if parsed_id is None:
+            return None
+        digest = self.signature(parsed_id)
+        if digest is None:
+            return None
+        return self.SEP.join((parsed_id, digest))
+
+    def verify(self, entity_id: str) -> bool:
+        """Check if the signature matches the current namespace."""
+        parsed_id, digest = self.parse(entity_id)
+        if digest is None or parsed_id is None:
+            return False
+        signature = self.signature(parsed_id)
+        if signature is None:
+            return False
+        return hmac.compare_digest(digest, signature)
+
+    def apply(self, proxy: E, shallow: bool = False) -> E:
+        """Rewrite an entity proxy so all IDs mentioned are limited to
+        the namespace."""
+        signed = proxy.clone()
+        signed.id = self.sign(proxy.id)
+        if not shallow:
+            for prop in proxy.iterprops():
+                if prop.type != registry.entity:
+                    continue
+                for value in signed.pop(prop):
+                    entity_id = get_entity_id(value)
+                    if entity_id is not None:
+                        signed.add(prop, self.sign(entity_id))
+        return signed
+
+    @classmethod
+    def make(cls, name: Union[str, "Namespace"]) -> "Namespace":
+        if isinstance(name, str):
+            return cls(name)
+        return name
+
+    def __eq__(self, other: Any) -> bool:
+        return bool(self.bname == other.bname)
+
+    def __repr__(self) -> str:
+        return "<Namespace(%r)>" % self.bname

followthemoney/offshore.py

@@ -0,0 +1,48 @@
+from followthemoney.proxy import E
+
+# Derived from: https://fsi.taxjustice.net/en/introduction/fsi-results
+OFFSHORE_COUNTRIES = set(
+    (
+        "ky",
+        "ch",
+        "sg",
+        "lu",
+        "vg",
+        "gg",
+        "pa",
+        "je",
+        "mt",
+        "bs",
+        "cy",
+        "gi",
+        "mo",
+        "bm",
+        "im",
+        "mh",
+        "mu",
+        "li",
+        "ai",
+        "kn",
+        "tc",
+        "vu",
+        "mc",
+        "sc",
+        "ag",
+        "dm",
+        "ms",
+        "lc",
+        "ck",
+    )
+)
+
+
+def offshore_from_jurisdiction(proxy: E) -> E:
+    """Tag organizations linked to a well-known offshore jurisdiction as
+    offshores automatically. Complete generalization, use only in experiments."""
+    if not proxy.schema.is_a("Organization"):
+        return proxy
+    countries = set(proxy.get("country", quiet=True))
+    countries.update(proxy.get("jurisdiction", quiet=True))
+    if len(countries.intersection(OFFSHORE_COUNTRIES)) > 0:
+        proxy.add("topics", "corp.offshore")
+    return proxy