followthemoney 3.8.4__py3-none-any.whl → 4.0.0__py3-none-any.whl

followthemoney/dataset/util.py

@@ -0,0 +1,58 @@
+from normality import slugify
+from typing import Annotated, Any
+from pydantic import BeforeValidator
+
+from followthemoney.types import registry
+
+
+def dataset_name_check(value: str) -> str:
+    """Check that the given value is a valid dataset name. This doesn't convert
+    or clean invalid names, but raises an error if they are not compliant to
+    force the user to fix an invalid name"""
+    if slugify(value, sep="_") != value:
+        raise ValueError("Invalid %s: %r" % ("dataset name", value))
+    return value
+
+
+def type_check_date(value: Any) -> str:
+    """Check that the given value is a valid date string."""
+    cleaned = registry.date.clean(value)
+    if cleaned is None:
+        raise ValueError("Invalid date: %r" % value)
+    return cleaned
+
+
+PartialDate = Annotated[str, BeforeValidator(type_check_date)]
+
+
+def type_check_country(value: Any) -> str:
+    """Check that the given value is a valid country code."""
+    cleaned = registry.country.clean(value)
+    if cleaned is None:
+        raise ValueError("Invalid country code: %r" % value)
+    return cleaned
+
+
+CountryCode = Annotated[str, BeforeValidator(type_check_country)]
+
+
+class Named:
+    name: str
+
+    def __init__(self, name: str) -> None:
+        self.name = name
+
+    def __eq__(self, other: Any) -> bool:
+        try:
+            return not not self.name == other.name
+        except AttributeError:
+            return False
+
+    def __lt__(self, other: Any) -> bool:
+        return self.name.__lt__(other.name)
+
+    def __hash__(self) -> int:
+        return hash(self.name)
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__}({self.name!r})>"

followthemoney/entity.py

@@ -0,0 +1,73 @@
+from typing import Any, Dict, List, Optional, Set, TypeVar
+
+from rigour.names import pick_name
+
+from followthemoney.proxy import EntityProxy
+from followthemoney.schema import Schema
+from followthemoney.statement.util import BASE_ID
+
+VE = TypeVar("VE", bound="ValueEntity")
+
+
+def _defined(*args: Optional[str]) -> List[str]:
+    return [arg for arg in args if arg is not None]
+
+
+class ValueEntity(EntityProxy):
+    """
+    This class has the extended attributes from `StatementEntity` but without
+    statements. Useful for streaming around. Starting from followthemoeny 4.0,
+    applications should use this entity class as the base class.
+    """
+
+    def __init__(
+        self,
+        schema: Schema,
+        data: Dict[str, Any],
+        key_prefix: Optional[str] = None,
+        cleaned: bool = True,
+    ):
+        super().__init__(schema, data, key_prefix=key_prefix, cleaned=cleaned)
+        self._caption: Optional[str] = data.get("caption")
+        self.datasets: Set[str] = set(data.get("datasets", []))
+        self.referents: Set[str] = set(data.get("referents", []))
+        self.first_seen: Optional[str] = data.get("first_seen")
+        self.last_seen: Optional[str] = data.get("last_seen")
+        self.last_change: Optional[str] = data.get("last_change")
+
+        # add data from statement dict if present.
+        # this updates the dataset and referents set
+        for stmt_data in data.pop("statements", []):
+            self.datasets.add(stmt_data["dataset"])
+            if stmt_data["entity_id"] != self.id:
+                self.referents.add(stmt_data["entity_id"])
+            if stmt_data["prop"] != BASE_ID:
+                self.add(stmt_data["prop"], stmt_data["value"])
+
+    def merge(self: "ValueEntity", other: "ValueEntity") -> "ValueEntity":
+        merged = super().merge(other)
+        merged._caption = pick_name(_defined(self._caption, other._caption))
+        merged.referents.update(other.referents)
+        merged.datasets.update(other.datasets)
+        self.first_seen = min(_defined(self.first_seen, other.first_seen), default=None)
+        self.last_seen = max(_defined(self.last_seen, other.last_seen), default=None)
+        changed = _defined(self.last_change, other.last_change)
+        self.last_change = max(changed, default=None)
+        return merged
+
+    def to_dict(self) -> Dict[str, Any]:
+        data: Dict[str, Any] = {
+            "id": self.id,
+            "caption": self._caption or self.caption,
+            "schema": self.schema.name,
+            "properties": self.properties,
+            "referents": list(self.referents),
+            "datasets": list(self.datasets),
+        }
+        if self.first_seen is not None:
+            data["first_seen"] = self.first_seen
+        if self.last_seen is not None:
+            data["last_seen"] = self.last_seen
+        if self.last_change is not None:
+            data["last_change"] = self.last_change
+        return data

followthemoney/exc.py

@@ -11,6 +11,12 @@ class FollowTheMoneyException(Exception):
     pass
 
 
+class MetadataException(FollowTheMoneyException):
+    """An exception raised by dataset metadata validation."""
+
+    pass
+
+
 class InvalidData(FollowTheMoneyException):
     """Schema validation errors will be caught by the API."""
 

followthemoney/export/common.py

@@ -1,6 +1,6 @@
 from typing import Generator, List, Optional, Tuple
 from followthemoney.property import Property
-from followthemoney.proxy import E
+from followthemoney.proxy import EntityProxy
 from followthemoney.schema import Schema
 from followthemoney.types import registry
 
@@ -17,12 +17,12 @@ class Exporter(object):
             yield prop
 
     def exportable_fields(
-        self, proxy: E
+        self, proxy: EntityProxy
     ) -> Generator[Tuple[Property, List[str]], None, None]:
         for prop in self.exportable_properties(proxy.schema):
             yield prop, proxy.get(prop)
 
-    def write(self, proxy: E, extra: Optional[List[str]] = None) -> None:
+    def write(self, proxy: EntityProxy, extra: Optional[List[str]] = None) -> None:
         raise NotImplementedError
 
     def finalize(self) -> None:

followthemoney/export/csv.py

@@ -1,21 +1,19 @@
 import csv
-
-try:
-    from _csv import _writer as csv_writer
-except ImportError:
-    # Python 3.8/3.9 work-around:
-    from _csv import writer as csv_writer  # type: ignore
-
-from io import TextIOWrapper
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple
+from io import TextIOWrapper
+from typing import Any, Dict, List, Optional, Protocol, Tuple
 
-from followthemoney.proxy import E
+from followthemoney.proxy import EntityProxy
 from followthemoney.export.common import Exporter
 from followthemoney.schema import Schema
 from followthemoney.util import PathLike
 
-CSVWriter = csv_writer
+
+class CSVWriter(Protocol):
+    @property
+    def dialect(self) -> Any: ...
+    def writerow(self, row: Any) -> Any: ...
+    def writerows(self, rows: Any) -> None: ...
 
 
 class CSVMixin(object):
@@ -69,7 +67,7 @@ class CSVExporter(Exporter, CSVMixin):
             headers.append(prop.name)
         writer.writerow(headers)
 
-    def write(self, proxy: E, extra: Optional[List[str]] = None) -> None:
+    def write(self, proxy: EntityProxy, extra: Optional[List[str]] = None) -> None:
         writer = self._get_writer(proxy.schema)
         cells = [proxy.id]
         cells.extend(extra or [])

followthemoney/export/neo4j.py

@@ -150,7 +150,7 @@ class CypherGraphExporter(GraphExporter):
                 labels = list(node.schema.names)
             else:
                 labels = [node.type.name]
-            cypher = "MERGE (p { %(id)s }) " "SET p += { %(map)s } SET p :%(label)s;\n"
+            cypher = "MERGE (p { %(id)s }) SET p += { %(map)s } SET p :%(label)s;\n"
             self.fh.write(
                 cypher
                 % {

followthemoney/export/rdf.py

@@ -1,23 +1,75 @@
 import logging
-from rdflib import Graph
-from typing import List, Optional, TextIO
+from prefixdate import Precision
+from rdflib import Graph, Namespace
+from rdflib.term import Identifier, URIRef, Literal
+from rdflib import RDF, SKOS, XSD
+from typing import Generator, List, Optional, TextIO, Tuple
 
 from followthemoney.export.common import Exporter
-from followthemoney.proxy import E
+from followthemoney.types import registry
+from followthemoney.proxy import EntityProxy
 
 log = logging.getLogger(__name__)
+Triple = Tuple[Identifier, Identifier, Identifier]
+NS = Namespace("https://schema.followthemoney.tech/#")
 
 
 class RDFExporter(Exporter):
+    """Export the entity as RDF N-Triples."""
+
+    TYPE_PREFIXES = {
+        registry.checksum: "hash:",
+        registry.country: "http://id.loc.gov/vocabulary/countries/",
+        registry.email: "mailto:",
+        registry.entity: "e:",
+        registry.gender: "gender:",
+        registry.ip: "ip:",
+        registry.identifier: "id:",
+        registry.language: "http://lexvo.org/id/iso639-3/",
+        registry.mimetype: "urn:mimetype:",
+        registry.phone: "tel:",
+        registry.topic: "ftm:topic:",
+    }
+
     def __init__(self, fh: TextIO, qualified: bool = True) -> None:
         super(RDFExporter, self).__init__()
         self.fh = fh
         self.qualified = qualified
 
-    def write(self, proxy: E, extra: Optional[List[str]] = None) -> None:
+    def entity_triples(self, proxy: EntityProxy) -> Generator[Triple, None, None]:
+        if proxy.id is None or proxy.schema is None:
+            return
+        entity_prefix = self.TYPE_PREFIXES[registry.entity]
+        uri = URIRef(f"{entity_prefix}{proxy.id}")
+        yield (uri, RDF.type, NS[proxy.schema.name])
+        if self.qualified:
+            caption = proxy.caption
+            if caption != proxy.schema.label:
+                yield (uri, SKOS.prefLabel, Literal(caption))
+        for prop, value in proxy.itervalues():
+            if prop.type in self.TYPE_PREFIXES:
+                prefix = self.TYPE_PREFIXES[prop.type]
+                if prop.type == registry.identifier and prop.format is not None:
+                    prefix = f"{prefix}{prop.format}:"
+                obj: Identifier = URIRef(f"{prefix}{value}")
+            elif prop.type == registry.date:
+                if len(value) < Precision.HOUR.value:
+                    obj = Literal(value, datatype=XSD.date)
+                else:
+                    obj = Literal(value, datatype=XSD.dateTime)
+            elif prop.type == registry.url:
+                obj = URIRef(value)
+            else:
+                obj = Literal(value)
+            if self.qualified:
+                yield (uri, NS[prop.qname], obj)
+            else:
+                yield (uri, URIRef(prop.name), obj)
+
+    def write(self, proxy: EntityProxy, extra: Optional[List[str]] = None) -> None:
         graph = Graph()
 
-        for triple in proxy.triples(qualified=self.qualified):
+        for triple in self.entity_triples(proxy):
             graph.add(triple)
         try:
             nt = graph.serialize(format="nt11").strip()

followthemoney/graph.py

@@ -5,6 +5,7 @@ This module provides an abstract data object that represents a property
 graph. This is used by the exporter modules to convert data
 to a specific output format, like Cypher or NetworkX.
 """
+
 import logging
 from typing import Any, Dict, Generator, Iterable, List, Optional
 
@@ -69,6 +70,8 @@ class Node(object):
     def from_proxy(cls, proxy: EntityProxy) -> "Node":
         """For a given :class:`~followthemoney.proxy.EntityProxy`, return a node
         based on the entity."""
+        if proxy.id is None:
+            raise InvalidModel("Invalid entity proxy: %r" % proxy)
         return cls(registry.entity, proxy.id, proxy=proxy)
 
     def __str__(self) -> str:
@@ -193,8 +196,7 @@ class Graph(object):
     """
 
     def __init__(self, edge_types: Iterable[PropertyType] = registry.pivots) -> None:
-        types = registry.get_types(edge_types)
-        self.edge_types = [t for t in types if t.matchable]
+        self.edge_types = [t for t in edge_types if t.matchable]
         self.flush()
 
     def flush(self) -> None:
@@ -256,11 +258,11 @@ class Graph(object):
         """Add an :class:`~followthemoney.proxy.EntityProxy` to the graph and make
         it either a :class:`~followthemoney.graph.Node` or an
         :class:`~followthemoney.graph.Edge`."""
-        if proxy is None:
+        if proxy is None or proxy.id is None:
             return
         self.queue(proxy.id, proxy)
         if proxy.schema.edge:
-            for (source, target) in proxy.edgepairs():
+            for source, target in proxy.edgepairs():
                 self._add_edge(proxy, source, target)
         else:
             self._add_node(proxy)

followthemoney/mapping/csv.py

@@ -1,24 +1,12 @@
 import io
 import os
 import logging
-from banal.lists import ensure_list
 import requests
 from csv import DictReader
 from urllib.parse import urlparse
-from banal import keys_values
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Dict,
-    Generator,
-    ItemsView,
-    Iterable,
-    List,
-    Optional,
-    Set,
-    Tuple,
-    cast,
-)
+from banal import keys_values, ensure_list
+from typing import TYPE_CHECKING, cast
+from typing import Any, Dict, Generator, ItemsView, Iterable, List, Optional, Set, Tuple
 
 from followthemoney.mapping.source import Record, Source
 from followthemoney.util import sanitize_text
@@ -48,16 +36,16 @@ class CSVSource(Source):
 
     def _parse_filters(self, filters: ItemsView[str, Any]) -> FilterList:
         filters_set: FilterList = []
-        for (key, value) in filters:
+        for key, value in filters:
             values = set(cast(List[Optional[str]], ensure_list(value)))
             filters_set.append((key, values))
         return filters_set
 
     def check_filters(self, data: Record) -> bool:
-        for (k, v) in self.filters_set:
+        for k, v in self.filters_set:
             if data.get(k) not in v:
                 return False
-        for (k, v) in self.filters_not_set:
+        for k, v in self.filters_not_set:
             if data.get(k) in v:
                 return False
         return True

followthemoney/mapping/sql.py

@@ -3,8 +3,7 @@ import logging
 from uuid import uuid4
 from typing import TYPE_CHECKING, Any, Dict, Generator, List, Optional, Union, cast
 from banal import ensure_list, is_listish, keys_values
-from sqlalchemy import MetaData, func
-from sqlalchemy.future import select
+from sqlalchemy import MetaData, func, select
 from sqlalchemy.engine import Engine, create_engine
 from sqlalchemy.sql.elements import Label
 from sqlalchemy.pool import NullPool
@@ -68,7 +67,7 @@ class SQLSource(Source):
                 return table.refs[ref]
         raise InvalidMapping("Missing reference: %s" % ref)
 
-    def apply_filters(self, q: Select) -> Select:
+    def apply_filters(self, q: Select[Any]) -> Select[Any]:
         for col, val in self.filters:
             if is_listish(val):
                 q = q.where(self.get_column(col).in_(val))
@@ -88,7 +87,7 @@ class SQLSource(Source):
             q = q.where(left == right)
         return q
 
-    def compose_query(self) -> Select:
+    def compose_query(self) -> Select[Any]:
         columns = [self.get_column(r) for r in self.query.refs]
         q = select(*columns)
         q = q.select_from(*[t.alias for t in self.tables])

followthemoney/model.py

@@ -1,16 +1,19 @@
 import os
 import yaml
 from functools import lru_cache
-from typing import Any, Dict, Generator, Iterator, Optional, Set, TypedDict, Union
+from typing import TYPE_CHECKING, Any
+from typing import Dict, Generator, Iterator, Optional, Set, TypedDict, Union
 
 from followthemoney.types import registry
 from followthemoney.types.common import PropertyType, PropertyTypeToDict
 from followthemoney.schema import Schema, SchemaToDict
 from followthemoney.property import Property
-from followthemoney.mapping import QueryMapping
-from followthemoney.proxy import EntityProxy
 from followthemoney.exc import InvalidModel, InvalidData
 
+if TYPE_CHECKING:
+    from followthemoney.proxy import EntityProxy
+    from followthemoney.mapping import QueryMapping
+
 
 class ModelToDict(TypedDict):
     schemata: Dict[str, SchemaToDict]
@@ -22,6 +25,8 @@ class Model(object):
     provides some helper functions to find schemata, properties or to instantiate
     entity proxies based on the schema metadata."""
 
+    _instance: Optional["Model"] = None
+
     __slots__ = ("path", "schemata", "properties", "qnames")
 
     def __init__(self, path: str) -> None:
@@ -38,6 +43,15 @@ class Model(object):
                 self._load(os.path.join(path, filename))
         self.generate()
 
+    @classmethod
+    def instance(cls) -> "Model":
+        if cls._instance is None:
+            model_path = os.path.dirname(__file__)
+            model_path = os.path.join(model_path, "schema")
+            model_path = os.environ.get("FTM_MODEL_PATH", model_path)
+            cls._instance = cls(model_path)
+        return cls._instance
+
     def generate(self) -> None:
         """Loading the model is a weird process because the schemata reference
         each other in complex ways, so the generation process cannot be fully
@@ -89,13 +103,15 @@ class Model(object):
 
     def make_mapping(
         self, mapping: Dict[str, Any], key_prefix: Optional[str] = None
-    ) -> QueryMapping:
+    ) -> "QueryMapping":
         """Parse a mapping that applies (tabular) source data to the model."""
+        from followthemoney.mapping import QueryMapping
+
         return QueryMapping(self, mapping, key_prefix=key_prefix)
 
     def map_entities(
         self, mapping: Dict[str, Any], key_prefix: Optional[str] = None
-    ) -> Generator[EntityProxy, None, None]:
+    ) -> Generator["EntityProxy", None, None]:
         """Given a mapping, yield a series of entities from the data source."""
         gen = self.make_mapping(mapping, key_prefix=key_prefix)
         for record in gen.source.records:
@@ -127,20 +143,31 @@ class Model(object):
         msg = "No common schema: %s and %s"
         raise InvalidData(msg % (left, right))
 
+    def matchable_schemata(self) -> Set[Schema]:
+        """Return a list of all schemata that are matchable."""
+        return set([s for s in self.schemata.values() if s.matchable])
+
     def make_entity(
         self, schema: Union[str, Schema], key_prefix: Optional[str] = None
-    ) -> EntityProxy:
+    ) -> "EntityProxy":
         """Instantiate an empty entity proxy of the given schema type."""
-        return EntityProxy(self, {"schema": schema}, key_prefix=key_prefix)
+        from followthemoney.proxy import EntityProxy
+
+        schema_ = self.get(schema)
+        if schema_ is None:
+            raise InvalidData("Schema does not exist: %s" % schema)
+        return EntityProxy(schema_, {}, key_prefix=key_prefix)
 
-    def get_proxy(self, data: Dict[str, Any], cleaned: bool = True) -> EntityProxy:
+    def get_proxy(self, data: Dict[str, Any], cleaned: bool = True) -> "EntityProxy":
         """Create an entity proxy to reflect the entity data in the given
         dictionary. If ``cleaned`` is disabled, all property values are
         fully re-validated and normalised. Use this if handling input data
         from an untrusted source."""
+        from followthemoney.proxy import EntityProxy
+
         if isinstance(data, EntityProxy):
             return data
-        return EntityProxy.from_dict(self, data, cleaned=cleaned)
+        return EntityProxy.from_dict(data, cleaned=cleaned)
 
     def to_dict(self) -> ModelToDict:
         """Return metadata for all schemata and properties, in a serializable form."""

followthemoney/namespace.py

@@ -22,6 +22,7 @@ that the combined ID is specific to a dataset, without needing an (expensive)
 index look up of each ID first. It can also be generated on the client or
 the server without compromising isolation.
 """
+
 import hmac
 from typing import Any, Optional, Tuple, Union
 
@@ -95,7 +96,8 @@ class Namespace(object):
         """Rewrite an entity proxy so all IDs mentioned are limited to
         the namespace."""
         signed = proxy.clone()
-        signed.id = self.sign(proxy.id)
+        if proxy.id is not None:
+            signed.id = self.sign(proxy.id)
         if not shallow:
             for prop in proxy.iterprops():
                 if prop.type != registry.entity:

followthemoney/ontology.py

@@ -1,15 +1,16 @@
 import sys
 from datetime import datetime
-from rdflib import Graph, URIRef, Literal
+from rdflib import Graph, URIRef, Literal, Namespace
 from rdflib.namespace import OWL, DCTERMS, RDF, RDFS, XSD
 
 from followthemoney import model
 from followthemoney.property import Property
 from followthemoney.schema import Schema
 from followthemoney.types import registry
-from followthemoney.rdf import NS
 from followthemoney.util import PathLike
 
+NS = Namespace("https://schema.followthemoney.tech/#")
+
 
 class Ontology(object):
     def __init__(self) -> None:
@@ -32,37 +33,38 @@ class Ontology(object):
             self.add_class(schema)
 
     def add_class(self, schema: Schema) -> None:
-        self.graph.add((schema.uri, RDF.type, RDFS.Class))
-        self.graph.add((schema.uri, RDFS.isDefinedBy, self.uri))
+        suri = NS[schema.name]
+        self.graph.add((suri, RDF.type, RDFS.Class))
+        self.graph.add((suri, RDFS.isDefinedBy, self.uri))
         for parent in schema.extends:
-            self.graph.add((schema.uri, RDFS.subClassOf, parent.uri))
+            self.graph.add((suri, RDFS.subClassOf, NS[parent.name]))
 
-        self.graph.add((schema.uri, RDFS.label, Literal(schema.label)))
+        self.graph.add((suri, RDFS.label, Literal(schema.label)))
         if schema.description is not None:
             description = Literal(schema.description)
-            self.graph.add((schema.uri, RDFS.comment, description))
+            self.graph.add((suri, RDFS.comment, description))
 
         for _, prop in sorted(schema.properties.items()):
             self.add_property(prop)
 
     def add_property(self, prop: Property) -> None:
-        self.graph.add((prop.uri, RDF.type, RDF.Property))
-        self.graph.add((prop.uri, RDFS.isDefinedBy, self.uri))
+        puri = NS[prop.qname]
+        self.graph.add((puri, RDF.type, RDF.Property))
+        self.graph.add((puri, RDFS.isDefinedBy, self.uri))
 
-        self.graph.add((prop.uri, RDFS.label, Literal(prop.label)))
+        self.graph.add((puri, RDFS.label, Literal(prop.label)))
         if prop.description is not None:
-            self.graph.add((prop.uri, RDFS.comment, Literal(prop.description)))
+            self.graph.add((puri, RDFS.comment, Literal(prop.description)))
 
-        self.graph.add((prop.uri, RDFS.domain, prop.schema.uri))
+        self.graph.add((puri, RDFS.domain, NS[prop.schema.name]))
         if prop.range is not None:
             range = model.get(prop.range)
             if range is not None:
-                range_uri = range.uri
-                self.graph.add((prop.uri, RDFS.range, range_uri))
+                self.graph.add((puri, RDFS.range, NS[range.name]))
         if prop.reverse is not None:
-            self.graph.add((prop.uri, OWL.inverseOf, prop.reverse.uri))
+            self.graph.add((puri, OWL.inverseOf, NS[prop.reverse.qname]))
         if prop.type == registry.date:
-            self.graph.add((prop.uri, RDFS.range, XSD.dateTime))
+            self.graph.add((puri, RDFS.range, XSD.dateTime))
 
     def write_namespace_docs(self, path: PathLike) -> None:
         xml_fn = "%s/ftm.xml" % path