followthemoney 3.8.5__py3-none-any.whl → 4.0.1__py3-none-any.whl

followthemoney/entity.py

@@ -0,0 +1,73 @@
+from typing import Any, Dict, List, Optional, Set, TypeVar
+
+from rigour.names import pick_name
+
+from followthemoney.proxy import EntityProxy
+from followthemoney.schema import Schema
+from followthemoney.statement.util import BASE_ID
+
+VE = TypeVar("VE", bound="ValueEntity")
+
+
+def _defined(*args: Optional[str]) -> List[str]:
+    return [arg for arg in args if arg is not None]
+
+
+class ValueEntity(EntityProxy):
+    """
+    This class has the extended attributes from `StatementEntity` but without
+    statements. Useful for streaming around. Starting from followthemoeny 4.0,
+    applications should use this entity class as the base class.
+    """
+
+    def __init__(
+        self,
+        schema: Schema,
+        data: Dict[str, Any],
+        key_prefix: Optional[str] = None,
+        cleaned: bool = True,
+    ):
+        super().__init__(schema, data, key_prefix=key_prefix, cleaned=cleaned)
+        self._caption: Optional[str] = data.get("caption")
+        self.datasets: Set[str] = set(data.get("datasets", []))
+        self.referents: Set[str] = set(data.get("referents", []))
+        self.first_seen: Optional[str] = data.get("first_seen")
+        self.last_seen: Optional[str] = data.get("last_seen")
+        self.last_change: Optional[str] = data.get("last_change")
+
+        # add data from statement dict if present.
+        # this updates the dataset and referents set
+        for stmt_data in data.pop("statements", []):
+            self.datasets.add(stmt_data["dataset"])
+            if stmt_data["entity_id"] != self.id:
+                self.referents.add(stmt_data["entity_id"])
+            if stmt_data["prop"] != BASE_ID:
+                self.add(stmt_data["prop"], stmt_data["value"])
+
+    def merge(self: "ValueEntity", other: "ValueEntity") -> "ValueEntity":
+        merged = super().merge(other)
+        merged._caption = pick_name(_defined(self._caption, other._caption))
+        merged.referents.update(other.referents)
+        merged.datasets.update(other.datasets)
+        self.first_seen = min(_defined(self.first_seen, other.first_seen), default=None)
+        self.last_seen = max(_defined(self.last_seen, other.last_seen), default=None)
+        changed = _defined(self.last_change, other.last_change)
+        self.last_change = max(changed, default=None)
+        return merged
+
+    def to_dict(self) -> Dict[str, Any]:
+        data: Dict[str, Any] = {
+            "id": self.id,
+            "caption": self._caption or self.caption,
+            "schema": self.schema.name,
+            "properties": self.properties,
+            "referents": list(self.referents),
+            "datasets": list(self.datasets),
+        }
+        if self.first_seen is not None:
+            data["first_seen"] = self.first_seen
+        if self.last_seen is not None:
+            data["last_seen"] = self.last_seen
+        if self.last_change is not None:
+            data["last_change"] = self.last_change
+        return data

followthemoney/exc.py

@@ -11,6 +11,12 @@ class FollowTheMoneyException(Exception):
     pass
 
 
+class MetadataException(FollowTheMoneyException):
+    """An exception raised by dataset metadata validation."""
+
+    pass
+
+
 class InvalidData(FollowTheMoneyException):
     """Schema validation errors will be caught by the API."""
 

followthemoney/export/rdf.py

@@ -1,23 +1,75 @@
 import logging
-from rdflib import Graph
-from typing import List, Optional, TextIO
+from prefixdate import Precision
+from rdflib import Graph, Namespace
+from rdflib.term import Identifier, URIRef, Literal
+from rdflib import RDF, SKOS, XSD
+from typing import Generator, List, Optional, TextIO, Tuple
 
 from followthemoney.export.common import Exporter
-from followthemoney.proxy import E
+from followthemoney.types import registry
+from followthemoney.proxy import EntityProxy
 
 log = logging.getLogger(__name__)
+Triple = Tuple[Identifier, Identifier, Identifier]
+NS = Namespace("https://schema.followthemoney.tech/#")
 
 
 class RDFExporter(Exporter):
+    """Export the entity as RDF N-Triples."""
+
+    TYPE_PREFIXES = {
+        registry.checksum: "hash:",
+        registry.country: "http://id.loc.gov/vocabulary/countries/",
+        registry.email: "mailto:",
+        registry.entity: "e:",
+        registry.gender: "gender:",
+        registry.ip: "ip:",
+        registry.identifier: "id:",
+        registry.language: "http://lexvo.org/id/iso639-3/",
+        registry.mimetype: "urn:mimetype:",
+        registry.phone: "tel:",
+        registry.topic: "ftm:topic:",
+    }
+
     def __init__(self, fh: TextIO, qualified: bool = True) -> None:
         super(RDFExporter, self).__init__()
         self.fh = fh
         self.qualified = qualified
 
-    def write(self, proxy: E, extra: Optional[List[str]] = None) -> None:
+    def entity_triples(self, proxy: EntityProxy) -> Generator[Triple, None, None]:
+        if proxy.id is None or proxy.schema is None:
+            return
+        entity_prefix = self.TYPE_PREFIXES[registry.entity]
+        uri = URIRef(f"{entity_prefix}{proxy.id}")
+        yield (uri, RDF.type, NS[proxy.schema.name])
+        if self.qualified:
+            caption = proxy.caption
+            if caption != proxy.schema.label:
+                yield (uri, SKOS.prefLabel, Literal(caption))
+        for prop, value in proxy.itervalues():
+            if prop.type in self.TYPE_PREFIXES:
+                prefix = self.TYPE_PREFIXES[prop.type]
+                if prop.type == registry.identifier and prop.format is not None:
+                    prefix = f"{prefix}{prop.format}:"
+                obj: Identifier = URIRef(f"{prefix}{value}")
+            elif prop.type == registry.date:
+                if len(value) < Precision.HOUR.value:
+                    obj = Literal(value, datatype=XSD.date)
+                else:
+                    obj = Literal(value, datatype=XSD.dateTime)
+            elif prop.type == registry.url:
+                obj = URIRef(value)
+            else:
+                obj = Literal(value)
+            if self.qualified:
+                yield (uri, NS[prop.qname], obj)
+            else:
+                yield (uri, URIRef(prop.name), obj)
+
+    def write(self, proxy: EntityProxy, extra: Optional[List[str]] = None) -> None:
         graph = Graph()
 
-        for triple in proxy.triples(qualified=self.qualified):
+        for triple in self.entity_triples(proxy):
             graph.add(triple)
         try:
             nt = graph.serialize(format="nt11").strip()

followthemoney/graph.py

@@ -196,8 +196,7 @@ class Graph(object):
     """
 
     def __init__(self, edge_types: Iterable[PropertyType] = registry.pivots) -> None:
-        types = registry.get_types(edge_types)
-        self.edge_types = [t for t in types if t.matchable]
+        self.edge_types = [t for t in edge_types if t.matchable]
         self.flush()
 
     def flush(self) -> None:

followthemoney/model.py

@@ -1,16 +1,19 @@
 import os
 import yaml
-from functools import lru_cache
-from typing import Any, Dict, Generator, Iterator, Optional, Set, TypedDict, Union
+from functools import cache
+from typing import TYPE_CHECKING, Any
+from typing import Dict, Generator, Iterator, Optional, Set, TypedDict, Union
 
 from followthemoney.types import registry
 from followthemoney.types.common import PropertyType, PropertyTypeToDict
 from followthemoney.schema import Schema, SchemaToDict
 from followthemoney.property import Property
-from followthemoney.mapping import QueryMapping
-from followthemoney.proxy import EntityProxy
 from followthemoney.exc import InvalidModel, InvalidData
 
+if TYPE_CHECKING:
+    from followthemoney.proxy import EntityProxy
+    from followthemoney.mapping import QueryMapping
+
 
 class ModelToDict(TypedDict):
     schemata: Dict[str, SchemaToDict]
@@ -22,6 +25,8 @@ class Model(object):
     provides some helper functions to find schemata, properties or to instantiate
     entity proxies based on the schema metadata."""
 
+    _instance: Optional["Model"] = None
+
     __slots__ = ("path", "schemata", "properties", "qnames")
 
     def __init__(self, path: str) -> None:
@@ -38,6 +43,15 @@ class Model(object):
                 self._load(os.path.join(path, filename))
         self.generate()
 
+    @classmethod
+    def instance(cls) -> "Model":
+        if cls._instance is None:
+            model_path = os.path.dirname(__file__)
+            model_path = os.path.join(model_path, "schema")
+            model_path = os.environ.get("FTM_MODEL_PATH", model_path)
+            cls._instance = cls(model_path)
+        return cls._instance
+
     def generate(self) -> None:
         """Loading the model is a weird process because the schemata reference
         each other in complex ways, so the generation process cannot be fully
@@ -89,20 +103,22 @@ class Model(object):
 
     def make_mapping(
         self, mapping: Dict[str, Any], key_prefix: Optional[str] = None
-    ) -> QueryMapping:
+    ) -> "QueryMapping":
         """Parse a mapping that applies (tabular) source data to the model."""
+        from followthemoney.mapping import QueryMapping
+
         return QueryMapping(self, mapping, key_prefix=key_prefix)
 
     def map_entities(
         self, mapping: Dict[str, Any], key_prefix: Optional[str] = None
-    ) -> Generator[EntityProxy, None, None]:
+    ) -> Generator["EntityProxy", None, None]:
         """Given a mapping, yield a series of entities from the data source."""
         gen = self.make_mapping(mapping, key_prefix=key_prefix)
         for record in gen.source.records:
             for entity in gen.map(record).values():
                 yield entity
 
-    @lru_cache(maxsize=None)
+    @cache
     def common_schema(
         self, left: Union[str, Schema], right: Union[str, Schema]
     ) -> Schema:
@@ -127,20 +143,31 @@ class Model(object):
         msg = "No common schema: %s and %s"
         raise InvalidData(msg % (left, right))
 
+    def matchable_schemata(self) -> Set[Schema]:
+        """Return a list of all schemata that are matchable."""
+        return set([s for s in self.schemata.values() if s.matchable])
+
     def make_entity(
         self, schema: Union[str, Schema], key_prefix: Optional[str] = None
-    ) -> EntityProxy:
+    ) -> "EntityProxy":
         """Instantiate an empty entity proxy of the given schema type."""
-        return EntityProxy(self, {"schema": schema}, key_prefix=key_prefix)
+        from followthemoney.proxy import EntityProxy
+
+        schema_ = self.get(schema)
+        if schema_ is None:
+            raise InvalidData("Schema does not exist: %s" % schema)
+        return EntityProxy(schema_, {}, key_prefix=key_prefix)
 
-    def get_proxy(self, data: Dict[str, Any], cleaned: bool = True) -> EntityProxy:
+    def get_proxy(self, data: Dict[str, Any], cleaned: bool = True) -> "EntityProxy":
         """Create an entity proxy to reflect the entity data in the given
         dictionary. If ``cleaned`` is disabled, all property values are
         fully re-validated and normalised. Use this if handling input data
         from an untrusted source."""
+        from followthemoney.proxy import EntityProxy
+
         if isinstance(data, EntityProxy):
             return data
-        return EntityProxy.from_dict(self, data, cleaned=cleaned)
+        return EntityProxy.from_dict(data, cleaned=cleaned)
 
     def to_dict(self) -> ModelToDict:
         """Return metadata for all schemata and properties, in a serializable form."""

followthemoney/names.py

@@ -0,0 +1,33 @@
+from rigour.names import NamePartTag, NameTypeTag
+
+from followthemoney.schema import Schema
+
+
+# Define the mapping of property names to name part tags.
+# This is used to tag the parts of the name with their type by using
+# `Name.tag_text` with the value of the property to mark name parts.
+PROP_PART_TAGS = (
+    ("firstName", NamePartTag.GIVEN),
+    ("lastName", NamePartTag.FAMILY),
+    ("secondName", NamePartTag.MIDDLE),
+    ("middleName", NamePartTag.MIDDLE),
+    ("fatherName", NamePartTag.PATRONYMIC),
+    ("motherName", NamePartTag.MATRONYMIC),
+    ("title", NamePartTag.HONORIFIC),
+    ("nameSuffix", NamePartTag.SUFFIX),
+    ("weakAlias", NamePartTag.NICK),
+)
+
+
+def schema_type_tag(schema: Schema) -> NameTypeTag:
+    """Return the name type tag for the given schema."""
+    if schema.is_a("Person"):
+        return NameTypeTag.PER
+    elif schema.is_a("Organization"):
+        return NameTypeTag.ORG
+    elif schema.is_a("LegalEntity"):
+        return NameTypeTag.ENT
+    elif schema.name in ("Vessel", "Asset", "Airplane", "Security"):
+        return NameTypeTag.OBJ
+    else:
+        return NameTypeTag.UNK

followthemoney/ontology.py

@@ -1,15 +1,16 @@
 import sys
 from datetime import datetime
-from rdflib import Graph, URIRef, Literal
+from rdflib import Graph, URIRef, Literal, Namespace
 from rdflib.namespace import OWL, DCTERMS, RDF, RDFS, XSD
 
 from followthemoney import model
 from followthemoney.property import Property
 from followthemoney.schema import Schema
 from followthemoney.types import registry
-from followthemoney.rdf import NS
 from followthemoney.util import PathLike
 
+NS = Namespace("https://schema.followthemoney.tech/#")
+
 
 class Ontology(object):
     def __init__(self) -> None:
@@ -32,37 +33,38 @@ class Ontology(object):
             self.add_class(schema)
 
     def add_class(self, schema: Schema) -> None:
-        self.graph.add((schema.uri, RDF.type, RDFS.Class))
-        self.graph.add((schema.uri, RDFS.isDefinedBy, self.uri))
+        suri = NS[schema.name]
+        self.graph.add((suri, RDF.type, RDFS.Class))
+        self.graph.add((suri, RDFS.isDefinedBy, self.uri))
         for parent in schema.extends:
-            self.graph.add((schema.uri, RDFS.subClassOf, parent.uri))
+            self.graph.add((suri, RDFS.subClassOf, NS[parent.name]))
 
-        self.graph.add((schema.uri, RDFS.label, Literal(schema.label)))
+        self.graph.add((suri, RDFS.label, Literal(schema.label)))
         if schema.description is not None:
             description = Literal(schema.description)
-            self.graph.add((schema.uri, RDFS.comment, description))
+            self.graph.add((suri, RDFS.comment, description))
 
         for _, prop in sorted(schema.properties.items()):
             self.add_property(prop)
 
     def add_property(self, prop: Property) -> None:
-        self.graph.add((prop.uri, RDF.type, RDF.Property))
-        self.graph.add((prop.uri, RDFS.isDefinedBy, self.uri))
+        puri = NS[prop.qname]
+        self.graph.add((puri, RDF.type, RDF.Property))
+        self.graph.add((puri, RDFS.isDefinedBy, self.uri))
 
-        self.graph.add((prop.uri, RDFS.label, Literal(prop.label)))
+        self.graph.add((puri, RDFS.label, Literal(prop.label)))
         if prop.description is not None:
-            self.graph.add((prop.uri, RDFS.comment, Literal(prop.description)))
+            self.graph.add((puri, RDFS.comment, Literal(prop.description)))
 
-        self.graph.add((prop.uri, RDFS.domain, prop.schema.uri))
+        self.graph.add((puri, RDFS.domain, NS[prop.schema.name]))
         if prop.range is not None:
             range = model.get(prop.range)
             if range is not None:
-                range_uri = range.uri
-                self.graph.add((prop.uri, RDFS.range, range_uri))
+                self.graph.add((puri, RDFS.range, NS[range.name]))
         if prop.reverse is not None:
-            self.graph.add((prop.uri, OWL.inverseOf, prop.reverse.uri))
+            self.graph.add((puri, OWL.inverseOf, NS[prop.reverse.qname]))
         if prop.type == registry.date:
-            self.graph.add((prop.uri, RDFS.range, XSD.dateTime))
+            self.graph.add((puri, RDFS.range, XSD.dateTime))
 
     def write_namespace_docs(self, path: PathLike) -> None:
         xml_fn = "%s/ftm.xml" % path

followthemoney/property.py

@@ -1,10 +1,9 @@
 from banal import is_mapping, as_bool
-from typing import TYPE_CHECKING, cast, Any, List, Optional, TypedDict
+from typing import TYPE_CHECKING, Any, List, Optional, TypedDict
 
 from followthemoney.exc import InvalidModel
 from followthemoney.types import registry
-from followthemoney.rdf import NS, URIRef
-from followthemoney.util import gettext, get_entity_id
+from followthemoney.util import gettext, get_entity_id, const
 
 if TYPE_CHECKING:
     from followthemoney.schema import Schema
@@ -26,7 +25,6 @@ class PropertyDict(TypedDict, total=False):
     deprecated: Optional[bool]
     maxLength: Optional[int]
     # stub: Optional[bool]
-    rdf: Optional[str]
     range: Optional[str]
     format: Optional[str]
 
@@ -66,7 +64,6 @@ class Property:
         "stub",
         "_reverse",
         "reverse",
-        "uri",
     )
 
     #: Invalid property names.
@@ -79,10 +76,10 @@ class Property:
         self.schema = schema
 
         #: Machine-readable name for this property.
-        self.name = name
+        self.name = const(name)
 
         #: Qualified property name, which also includes the schema name.
-        self.qname = "%s:%s" % (schema.name, self.name)
+        self.qname = const("%s:%s" % (schema.name, self.name))
         if self.name in self.RESERVED:
             raise InvalidModel("Reserved name: %s" % self.name)
 
@@ -97,12 +94,11 @@ class Property:
         #: This property should not be shown or mentioned in the user interface.
         self.hidden = as_bool(data.get("hidden"))
 
-        type_ = data.get("type", "string")
-        if type_ is None or type_ not in registry.named:
-            raise InvalidModel("Invalid type: %s" % type_)
-
+        type_ = data.get("type") or "string"
         #: The data type for this property.
-        self.type = registry[type_]
+        self.type = registry.get(type_)
+        if self.type is None:
+            raise InvalidModel("Invalid type: %s" % type_)
 
         #: Whether this property should be used for matching and cross-referencing.
         _matchable = data.get("matchable")
@@ -137,9 +133,6 @@ class Property:
         self._reverse = data.get("reverse")
         self.reverse: Optional["Property"] = None
 
-        #: RDF term for this property (i.e. the predicate URI).
-        self.uri = URIRef(cast(str, data.get("rdf", NS[self.qname])))
-
     def generate(self, model: "Model") -> None:
         """Setup method used when loading the model in order to build out the reverse
         links of the property."""
@@ -170,6 +163,10 @@ class Property:
             return 0.0
         return self.type.specificity(value)
 
+    def caption(self, value: str) -> str:
+        """Return a user-friendly caption for the given value."""
+        return self.type.caption(value, format=self.format)
+
     def validate(self, data: List[Any]) -> Optional[str]:
         """Validate that the data should be stored.