followthemoney 1.3.7__py3-none-any.whl → 3.8.0__py3-none-any.whl

followthemoney/model.py

@@ -1,66 +1,109 @@
 import os
 import yaml
+from typing import Any, Dict, Generator, Iterator, Optional, Set, TypedDict, Union
 
-from followthemoney.schema import Schema
+from followthemoney.types import registry
+from followthemoney.types.common import PropertyType, PropertyTypeToDict
+from followthemoney.schema import Schema, SchemaToDict
+from followthemoney.property import Property
 from followthemoney.mapping import QueryMapping
 from followthemoney.proxy import EntityProxy
 from followthemoney.exc import InvalidModel, InvalidData
 
 
+class ModelToDict(TypedDict):
+    schemata: Dict[str, SchemaToDict]
+    types: Dict[str, PropertyTypeToDict]
+
+
 class Model(object):
-    """A collection of schemata."""
+    """A collection of all the schemata available in followthemoney. The model
+    provides some helper functions to find schemata, properties or to instantiate
+    entity proxies based on the schema metadata."""
+
+    __slots__ = ("path", "schemata", "properties", "qnames")
 
-    def __init__(self, path):
+    def __init__(self, path: str) -> None:
         self.path = path
-        self.schemata = {}
+
+        #: A mapping with all schemata, organised by their name.
+        self.schemata: Dict[str, Schema] = {}
+
+        #: All properties defined in the model.
+        self.properties: Set[Property] = set()
+        self.qnames: Dict[str, Property] = {}
         for (path, _, filenames) in os.walk(self.path):
             for filename in filenames:
                 self._load(os.path.join(path, filename))
         self.generate()
 
-    def generate(self):
-        self.properties = set()
+    def generate(self) -> None:
+        """Loading the model is a weird process because the schemata reference
+        each other in complex ways, so the generation process cannot be fully
+        run as schemata are being instantiated. Hence this process needs to be
+        called once all schemata are loaded to finalise dereferencing the
+        schemata."""
         for schema in self:
-            schema.generate()
-            for prop in schema.properties.values():
-                self.properties.add(prop)
-
-    def _load(self, filepath):
-        with open(filepath, 'r') as fh:
-            data = yaml.load(fh)
+            schema.generate(self)
+        for prop in self.properties:
+            self.qnames[prop.qname] = prop
+            for schema in prop.schema.descendants:
+                if prop.name not in schema.properties:
+                    schema.properties[prop.name] = prop
+
+    def _load(self, filepath: str) -> None:
+        with open(filepath, "r", encoding="utf-8") as fh:
+            data = yaml.safe_load(fh)
             if not isinstance(data, dict):
-                raise InvalidModel('Model file is not a mapping.')
+                raise InvalidModel("Model file is not a mapping: %s" % filepath)
             for name, config in data.items():
                 self.schemata[name] = Schema(self, name, config)
 
-    def get(self, name):
-        if isinstance(name, Schema):
-            return name
-        return self.schemata.get(name)
+    def get(self, name: Union[str, Schema]) -> Optional[Schema]:
+        """Get a schema object based on a schema name. If the input is already
+        a schema object, it will just be returned."""
+        if isinstance(name, str):
+            return self.schemata.get(name)
+        return name
 
-    def get_qname(self, qname):
-        if not hasattr(self, '_qnames'):
-            self._qnames = {p.qname: p for p in self.properties}
-        return self._qnames.get(qname)
+    def get_qname(self, qname: str) -> Optional[Property]:
+        """Get a property object based on a qualified name (i.e. schema:property)."""
+        return self.qnames.get(qname)
 
-    def __getitem__(self, name):
+    def __getitem__(self, name: str) -> Schema:
+        """Same as get(), but throws an exception when the given name does not exist."""
         schema = self.get(name)
         if schema is None:
             raise KeyError("No such schema: %s" % name)
         return schema
 
-    def make_mapping(self, mapping, key_prefix=None):
+    def get_type_schemata(self, type_: PropertyType) -> Set[Schema]:
+        """Return all the schemata which have a property of the given type."""
+        schemata = set()
+        for schema in self.schemata.values():
+            for prop in schema.properties.values():
+                if prop.type == type_:
+                    schemata.add(schema)
+        return schemata
+
+    def make_mapping(
+        self, mapping: Dict[str, Any], key_prefix: Optional[str] = None
+    ) -> QueryMapping:
         """Parse a mapping that applies (tabular) source data to the model."""
         return QueryMapping(self, mapping, key_prefix=key_prefix)
 
-    def map_entities(self, mapping, key_prefix=None):
+    def map_entities(
+        self, mapping: Dict[str, Any], key_prefix: Optional[str] = None
+    ) -> Generator[EntityProxy, None, None]:
         """Given a mapping, yield a series of entities from the data source."""
-        mapping = self.make_mapping(mapping, key_prefix=key_prefix)
-        for record in mapping.source.records:
-            for entity in mapping.map(record).values():
+        gen = self.make_mapping(mapping, key_prefix=key_prefix)
+        for record in gen.source.records:
+            for entity in gen.map(record).values():
                 yield entity
 
-    def common_schema(self, left, right):
+    def common_schema(
+        self, left: Union[str, Schema], right: Union[str, Schema]
+    ) -> Schema:
         """Select the most narrow of two schemata.
 
         When indexing data from a dataset, an entity may be declared as a
@@ -68,33 +111,42 @@ class Model(object):
         will select the most specific of two schemata offered. In the example,
         that would be Person.
         """
-        left = self.get(left) or self.get(right)
-        right = self.get(right) or self.get(left)
-        left_schemata = list(left.schemata)
-        right_schemata = list(right.schemata)
-        if right in left_schemata:
-            return left
-        if left in right_schemata:
-            return right
-
-        # Find a common ancestor:
-        for left in left_schemata:
-            for right in right_schemata:
-                if left == right:
-                    return left
-
-        msg = "No common ancestor: %s and %s"
+        left_schema = self.get(left) or self.get(right)
+        right_schema = self.get(right) or self.get(left)
+        if left_schema is None or right_schema is None:
+            raise InvalidData("Invalid schema")
+        if left_schema.is_a(right_schema):
+            return left_schema
+        if right_schema.is_a(left_schema):
+            return right_schema
+        # for schema in self.schemata.values():
+        #     if schema.is_a(left) and schema.is_a(right):
+        #         return schema
+        msg = "No common schema: %s and %s"
         raise InvalidData(msg % (left, right))
 
-    def make_entity(self, schema, key_prefix=None):
-        schema = self.get(schema)
-        return EntityProxy(schema, None, None, key_prefix=key_prefix)
-
-    def get_proxy(self, data):
-        return EntityProxy.from_dict(self, data)
-
-    def to_dict(self):
-        return {n: s.to_dict() for (n, s) in self.schemata.items()}
-
-    def __iter__(self):
+    def make_entity(
+        self, schema: Union[str, Schema], key_prefix: Optional[str] = None
+    ) -> EntityProxy:
+        """Instantiate an empty entity proxy of the given schema type."""
+        return EntityProxy(self, {"schema": schema}, key_prefix=key_prefix)
+
+    def get_proxy(self, data: Dict[str, Any], cleaned: bool = True) -> EntityProxy:
+        """Create an entity proxy to reflect the entity data in the given
+        dictionary. If ``cleaned`` is disabled, all property values are
+        fully re-validated and normalised. Use this if handling input data
+        from an untrusted source."""
+        if isinstance(data, EntityProxy):
+            return data
+        return EntityProxy.from_dict(self, data, cleaned=cleaned)
+
+    def to_dict(self) -> ModelToDict:
+        """Return metadata for all schemata and properties, in a serializable form."""
+        return {
+            "schemata": {s.name: s.to_dict() for s in self.schemata.values()},
+            "types": {t.name: t.to_dict() for t in registry.types},
+        }
+
+    def __iter__(self) -> Iterator[Schema]:
+        """Iterate across all schemata."""
         return iter(self.schemata.values())

followthemoney/namespace.py

@@ -0,0 +1,119 @@
+"""
+*We like our abstractions like our offshore banks: leaky.*
+
+Entity ID namespaces are a security mechanism related to the Aleph search index.
+
+Aleph allows the user (via mappings or the API) to create arbitrary entity IDs.
+Entity IDs that are controlled by the user and not the system are unusual.
+However, this makes it possible to generate bulk data outside Aleph,
+and then load entities into the system as a continuous :ref:`streams`.
+
+The problem is that having user controlled entity IDs increases the chance
+of conflict in the search index.
+
+Namespacing works around this by making each entity ID consist of two parts:
+one controlled by the client, the other controlled by the system. The second
+part of the ID is called its `signature`::
+
+    entity_id.a40a29300ac6bb79dd2f911e77bbda7a3b502126
+
+The signature is generated as ``hmac(entity_id, dataset_id)``. This guarantees
+that the combined ID is specific to a dataset, without needing an (expensive)
+index look up of each ID first. It can also be generated on the client or
+the server without compromising isolation.
+"""
+import hmac
+from typing import Any, Optional, Tuple, Union
+
+from followthemoney.types import registry
+from followthemoney.proxy import E
+from followthemoney.util import key_bytes, get_entity_id
+
+
+class Namespace(object):
+    """Namespaces are used to partition entity IDs into different units,
+    which traditionally represent a dataset, collection or source.
+
+    See module docstring for details."""
+
+    SEP = "."
+
+    def __init__(self, name: Optional[str] = None) -> None:
+        self.bname = key_bytes(name) if name else b""
+        self.hmac = hmac.new(self.bname, digestmod="sha1")
+
+    @classmethod
+    def parse(cls, entity_id: str) -> Tuple[Optional[str], Optional[str]]:
+        """Split up an entity ID into the plain ID and the namespace
+        signature. If either part is missing, return None instead."""
+        clean_id = registry.entity.clean(entity_id)
+        if clean_id is None:
+            return (None, None)
+        try:
+            plain_id, checksum = clean_id.rsplit(cls.SEP, 1)
+            return (plain_id, checksum)
+        except ValueError:
+            return (clean_id, None)
+
+    @classmethod
+    def strip(cls, entity_id: str) -> Optional[str]:
+        plain_id, _ = cls.parse(entity_id)
+        return plain_id
+
+    def signature(self, entity_id: str) -> Optional[str]:
+        """Generate a namespace-specific signature."""
+        if not len(self.bname) or entity_id is None:
+            return None
+        digest = self.hmac.copy()
+        digest.update(key_bytes(entity_id))
+        return digest.hexdigest()
+
+    def sign(self, entity_id: str) -> Optional[str]:
+        """Apply a namespace signature to an entity ID, removing any
+        previous namespace marker."""
+        parsed_id, _ = self.parse(entity_id)
+        if not len(self.bname):
+            return parsed_id
+        if parsed_id is None:
+            return None
+        digest = self.signature(parsed_id)
+        if digest is None:
+            return None
+        return self.SEP.join((parsed_id, digest))
+
+    def verify(self, entity_id: str) -> bool:
+        """Check if the signature matches the current namespace."""
+        parsed_id, digest = self.parse(entity_id)
+        if digest is None or parsed_id is None:
+            return False
+        signature = self.signature(parsed_id)
+        if signature is None:
+            return False
+        return hmac.compare_digest(digest, signature)
+
+    def apply(self, proxy: E, shallow: bool = False) -> E:
+        """Rewrite an entity proxy so all IDs mentioned are limited to
+        the namespace."""
+        signed = proxy.clone()
+        signed.id = self.sign(proxy.id)
+        if not shallow:
+            for prop in proxy.iterprops():
+                if prop.type != registry.entity:
+                    continue
+                for value in signed.pop(prop):
+                    entity_id = get_entity_id(value)
+                    if entity_id is not None:
+                        signed.add(prop, self.sign(entity_id))
+        return signed
+
+    @classmethod
+    def make(cls, name: Union[str, "Namespace"]) -> "Namespace":
+        if isinstance(name, str):
+            return cls(name)
+        return name
+
+    def __eq__(self, other: Any) -> bool:
+        return bool(self.bname == other.bname)
+
+    def __repr__(self) -> str:
+        return "<Namespace(%r)>" % self.bname

followthemoney/offshore.py

@@ -0,0 +1,48 @@
+from followthemoney.proxy import E
+
+# Derived from: https://fsi.taxjustice.net/en/introduction/fsi-results
+OFFSHORE_COUNTRIES = set(
+    (
+        "ky",
+        "ch",
+        "sg",
+        "lu",
+        "vg",
+        "gg",
+        "pa",
+        "je",
+        "mt",
+        "bs",
+        "cy",
+        "gi",
+        "mo",
+        "bm",
+        "im",
+        "mh",
+        "mu",
+        "li",
+        "ai",
+        "kn",
+        "tc",
+        "vu",
+        "mc",
+        "sc",
+        "ag",
+        "dm",
+        "ms",
+        "lc",
+        "ck",
+    )
+)
+
+
+def offshore_from_jurisdiction(proxy: E) -> E:
+    """Tag organizations linked to a well-known offshore jurisdiction as
+    offshores automatically. Complete generalization, use only in experiments."""
+    if not proxy.schema.is_a("Organization"):
+        return proxy
+    countries = set(proxy.get("country", quiet=True))
+    countries.update(proxy.get("jurisdiction", quiet=True))
+    if len(countries.intersection(OFFSHORE_COUNTRIES)) > 0:
+        proxy.add("topics", "corp.offshore")
+    return proxy

followthemoney/ontology.py

@@ -0,0 +1,77 @@
+import sys
+from datetime import datetime
+from rdflib import Graph, URIRef, Literal
+from rdflib.namespace import OWL, DCTERMS, RDF, RDFS, XSD
+
+from followthemoney import model
+from followthemoney.property import Property
+from followthemoney.schema import Schema
+from followthemoney.types import registry
+from followthemoney.rdf import NS
+from followthemoney.util import PathLike
+
+
+class Ontology(object):
+    def __init__(self) -> None:
+        self.uri = URIRef(NS)
+        self.graph = Graph(identifier=self.uri)
+        self.graph.namespace_manager.bind("ftm", NS)
+        self.graph.namespace_manager.bind("owl", OWL)
+        self.graph.namespace_manager.bind("dct", DCTERMS)
+
+        self.graph.add((self.uri, RDF.type, OWL.Ontology))
+        self.graph.add((self.uri, RDFS.label, Literal("Follow The Money")))
+        modified = datetime.now().strftime("%Y-%m-%dT%H:%I:%M")
+        modified = Literal(modified, datatype=XSD.dateTime)
+        self.graph.add((self.uri, DCTERMS.modified, modified))
+
+        self.add_schemata()
+
+    def add_schemata(self) -> None:
+        for schema in sorted(model):
+            self.add_class(schema)
+
+    def add_class(self, schema: Schema) -> None:
+        self.graph.add((schema.uri, RDF.type, RDFS.Class))
+        self.graph.add((schema.uri, RDFS.isDefinedBy, self.uri))
+        for parent in schema.extends:
+            self.graph.add((schema.uri, RDFS.subClassOf, parent.uri))
+
+        self.graph.add((schema.uri, RDFS.label, Literal(schema.label)))
+        if schema.description is not None:
+            description = Literal(schema.description)
+            self.graph.add((schema.uri, RDFS.comment, description))
+
+        for _, prop in sorted(schema.properties.items()):
+            self.add_property(prop)
+
+    def add_property(self, prop: Property) -> None:
+        self.graph.add((prop.uri, RDF.type, RDF.Property))
+        self.graph.add((prop.uri, RDFS.isDefinedBy, self.uri))
+
+        self.graph.add((prop.uri, RDFS.label, Literal(prop.label)))
+        if prop.description is not None:
+            self.graph.add((prop.uri, RDFS.comment, Literal(prop.description)))
+
+        self.graph.add((prop.uri, RDFS.domain, prop.schema.uri))
+        if prop.range is not None:
+            range = model.get(prop.range)
+            if range is not None:
+                range_uri = range.uri
+                self.graph.add((prop.uri, RDFS.range, range_uri))
+        if prop.reverse is not None:
+            self.graph.add((prop.uri, OWL.inverseOf, prop.reverse.uri))
+        if prop.type == registry.date:
+            self.graph.add((prop.uri, RDFS.range, XSD.dateTime))
+
+    def write_namespace_docs(self, path: PathLike) -> None:
+        xml_fn = "%s/ftm.xml" % path
+        with open(xml_fn, "w") as xml_file:
+            xml_file.write(self.graph.serialize(format="xml"))
+
+
+if __name__ == "__main__":
+    path = sys.argv[1]
+    ontology = Ontology()
+    ontology.write_namespace_docs(path)
+    print("Namespace docs written to %s" % path)