followthemoney 3.8.4__py3-none-any.whl → 4.0.0__py3-none-any.whl

followthemoney/schema.py

@@ -1,22 +1,12 @@
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Dict,
-    List,
-    Optional,
-    Set,
-    TypedDict,
-    Union,
-    cast,
-)
+from typing import TYPE_CHECKING, Any, cast
+from typing import Dict, List, Optional, Set, TypedDict, Union
 from banal import ensure_list, ensure_dict, as_bool
 from functools import lru_cache
 
 from followthemoney.property import Property, PropertySpec, PropertyToDict, ReverseSpec
 from followthemoney.types import registry
 from followthemoney.exc import InvalidData, InvalidModel
-from followthemoney.rdf import URIRef, NS
-from followthemoney.util import gettext
+from followthemoney.util import gettext, const
 
 if TYPE_CHECKING:
     from followthemoney.model import Model
@@ -47,7 +37,6 @@ class SchemaSpec(TypedDict, total=False):
     edge: EdgeSpec
     temporalExtent: TemporalExtentSpec
     description: Optional[str]
-    rdf: Optional[str]
     abstract: bool
     hidden: bool
     generated: bool
@@ -90,7 +79,6 @@ class Schema:
         "_plural",
         "_description",
         "_hash",
-        "uri",
         "abstract",
         "hidden",
         "generated",
@@ -118,15 +106,12 @@ class Schema:
 
     def __init__(self, model: "Model", name: str, data: SchemaSpec) -> None:
         #: Machine-readable name of the schema, used for identification.
-        self.name = name
+        self.name = const(name)
         self.model = model
         self._label = data.get("label", name)
         self._plural = data.get("plural", self.label)
         self._description = data.get("description")
-        self._hash = hash("<Schema(%r)>" % name)
-
-        #: RDF identifier for this schema when it is transformed to a triple term.
-        self.uri = URIRef(cast(str, data.get("rdf", NS[name])))
+        self._hash = hash("<Schema(%r)>" % self.name)
 
         #: Do not store or emit entities of this type, it is used only for
         #: inheritance.
@@ -152,17 +137,17 @@ class Schema:
         #: Mark a set of properties as important, i.e. they should be shown
         #: first, or in an abridged view of the entity. In Aleph, these properties
         #: are included in tabular entity listings.
-        self.featured = ensure_list(data.get("featured", []))
+        self.featured = [const(f) for f in data.get("featured", [])]
 
         #: Mark a set of properties as required. This is applied only when
         #: an entity is created by the user - bulk created entities will
         #: slip through even if it is technically invalid.
-        self.required = ensure_list(data.get("required", []))
+        self.required = [const(r) for r in data.get("required", [])]
 
         #: Mark a set of properties to be used for the entity's caption.
         #: They will be checked in order and the first existent value will
         #: be used.
-        self.caption = ensure_list(data.get("caption", []))
+        self.caption = [const(s) for s in data.get("caption", [])]
 
         # A transform of the entity into an edge for its representation in
         # the context of a property graph representation like Neo4J/Gephi.
@@ -173,7 +158,7 @@ class Schema:
         #: Flag to indicate if this schema should be represented by an edge (rather than
         #: a node) when the data is converted into a property graph.
         self.edge: bool = self.edge_source is not None and self.edge_target is not None
-        self.edge_caption = ensure_list(edge.get("caption", []))
+        self.edge_caption = [const(p) for p in edge.get("caption", [])]
         self._edge_label = edge.get("label", self._label)
 
         #: Flag to indicate if the edge should be presented as directed to the user,
@@ -183,16 +168,16 @@ class Schema:
         #: Specify which properties should be used to represent this schema in a
         #: timeline.
         temporal_extent = data.get("temporalExtent", {})
-        self._temporal_start = ensure_list(temporal_extent.get("start", []))
-        self._temporal_end = ensure_list(temporal_extent.get("end", []))
+        self._temporal_start = [const(s) for s in temporal_extent.get("start", [])]
+        self._temporal_end = [const(e) for e in temporal_extent.get("end", [])]
 
         #: Direct parent schemata of this schema.
-        self._extends = ensure_list(data.get("extends", []))
+        self._extends = [const(s) for s in data.get("extends", [])]
         self.extends: Set["Schema"] = set()
 
         #: All parents of this schema (including indirect parents and the schema
         #: itself).
-        self.schemata = set([self])
+        self.schemata: Set[Schema] = set([self])
 
         #: All names of :attr:`~schemata`.
         self.names = set([self.name])
@@ -205,8 +190,8 @@ class Schema:
         #: The full list of properties defined for the entity, including those
         #: inherited from parent schemata.
         self.properties: Dict[str, Property] = {}
-        for name, prop in data.get("properties", {}).items():
-            self.properties[name] = Property(self, name, prop)
+        for pname, prop in data.get("properties", {}).items():
+            self.properties[pname] = Property(self, pname, prop)
 
     def generate(self, model: "Model") -> None:
         """While loading the schema, this function will validate and
@@ -317,12 +302,18 @@ class Schema:
 
     @property
     def source_prop(self) -> Optional[Property]:
-        """The entity property to be used as an edge source."""
+        """The entity property to be used as an edge source when the schema is
+        considered as a relationship."""
+        if self.edge_source is None:
+            return None
         return self.get(self.edge_source)
 
     @property
     def target_prop(self) -> Optional[Property]:
-        """The entity property to be used as an edge target."""
+        """The entity property to be used as an edge target when the schema is transformed
+        into a relationship."""
+        if self.edge_target is None:
+            return None
         return self.get(self.edge_target)
 
     @property
@@ -404,13 +395,13 @@ class Schema:
             other = other.name
         return other in self.names
 
-    def get(self, name: Optional[str]) -> Optional[Property]:
+    def get(self, name: str) -> Optional[Property]:
         """Retrieve a property defined for this schema by its name."""
         if name is None:
             return None
         return self.properties.get(name)
 
-    def validate(self, data: Any) -> Optional[str]:
+    def validate(self, data: Dict[str, Any]) -> Optional[str]:
         """Validate a dictionary against the given schema.
         This will also drop keys which are not valid as properties.
         """
@@ -478,7 +469,7 @@ class Schema:
     def __eq__(self, other: Any) -> bool:
         """Compare two schemata (via hash)."""
         try:
-            return self._hash == hash(other)
+            return self._hash == other._hash  # type: ignore
         except AttributeError:
             return False
 
@@ -486,10 +477,7 @@ class Schema:
         return self.name.__lt__(other.name)
 
     def __hash__(self) -> int:
-        try:
-            return self._hash
-        except AttributeError:
-            return super().__hash__()
+        return self._hash
 
     def __repr__(self) -> str:
         return "<Schema(%r)>" % self.name

followthemoney/statement/__init__.py

@@ -0,0 +1,19 @@
+from followthemoney.statement.statement import Statement, StatementDict
+from followthemoney.statement.serialize import CSV, JSON, PACK, FORMATS
+from followthemoney.statement.serialize import write_statements
+from followthemoney.statement.serialize import read_statements, read_path_statements
+from followthemoney.statement.entity import SE, StatementEntity
+
+__all__ = [
+    "Statement",
+    "StatementDict",
+    "StatementEntity",
+    "SE",
+    "CSV",
+    "JSON",
+    "PACK",
+    "FORMATS",
+    "write_statements",
+    "read_statements",
+    "read_path_statements",
+]

followthemoney/statement/entity.py

@@ -0,0 +1,437 @@
+from hashlib import sha1
+from collections.abc import Mapping
+from typing import Any, Dict, List, Optional, Set, Type
+from typing import Generator, Iterable, Tuple, TypeVar
+
+from followthemoney.model import Model
+from followthemoney.exc import InvalidData
+from followthemoney.types.common import PropertyType
+from followthemoney.property import Property
+from followthemoney.util import gettext
+from followthemoney.proxy import P
+from followthemoney.types import registry
+from followthemoney.value import string_list, Values
+from followthemoney.proxy import EntityProxy
+from followthemoney.dataset import Dataset, DefaultDataset
+from followthemoney.statement.statement import Statement
+from followthemoney.statement.util import BASE_ID
+
+SE = TypeVar("SE", bound="StatementEntity")
+
+
+class StatementEntity(EntityProxy):
+    """An entity object that can link to a set of datasets that it is sourced from."""
+
+    __slots__ = (
+        "schema",
+        "id",
+        "_caption",
+        "extra_referents",
+        "dataset",
+        "last_change",
+        "_statements",
+    )
+
+    def __init__(self, dataset: Dataset, data: Dict[str, Any], cleaned: bool = True):
+        data = dict(data or {})
+        schema = Model.instance().get(data.pop("schema", None))
+        if schema is None:
+            raise InvalidData(gettext("No schema for entity."))
+        self.schema = schema
+
+        self._caption: Optional[str] = None
+        """A pre-computed label for this entity."""
+
+        self.extra_referents: Set[str] = set(data.pop("referents", []))
+        """The IDs of all entities which are included in this canonical entity."""
+
+        self.last_change: Optional[str] = data.get("last_change", None)
+        """The last time this entity was changed."""
+
+        self.dataset = dataset
+        """The default dataset for new statements."""
+
+        self.id: Optional[str] = data.pop("id", None)
+        self._statements: Dict[str, Set[Statement]] = {}
+
+        properties = data.pop("properties", None)
+        if isinstance(properties, Mapping):
+            for key, value in properties.items():
+                self.add(key, value, cleaned=cleaned, quiet=True)
+
+        for stmt_data in data.pop("statements", []):
+            stmt = Statement.from_dict(stmt_data)
+            if self.id is not None:
+                stmt.canonical_id = self.id
+            self.add_statement(stmt)
+
+    @property
+    def _properties(self) -> Dict[str, List[str]]:  # type: ignore
+        return {p: [s.value for s in v] for p, v in self._statements.items()}
+
+    def _iter_stmt(self) -> Generator[Statement, None, None]:
+        for stmts in self._statements.values():
+            for stmt in stmts:
+                if stmt.entity_id is None and self.id is not None:
+                    stmt.entity_id = self.id
+                    stmt.id = stmt.generate_key()
+                if stmt.id is None:
+                    stmt.id = stmt.generate_key()
+                yield stmt
+
+    @property
+    def statements(self) -> Generator[Statement, None, None]:
+        """Return all statements for this entity, with extra ID statement."""
+        ids: List[str] = []
+        last_seen: Set[str] = set()
+        first_seen: Set[str] = set()
+        for stmt in self._iter_stmt():
+            yield stmt
+            if stmt.id is not None:
+                ids.append(stmt.id)
+            if stmt.last_seen is not None:
+                last_seen.add(stmt.last_seen)
+            if stmt.first_seen is not None:
+                first_seen.add(stmt.first_seen)
+        if self.id is not None:
+            digest = sha1(self.schema.name.encode("utf-8"))
+            for id in sorted(ids):
+                digest.update(id.encode("utf-8"))
+            checksum = digest.hexdigest()
+            # This is to make the last_change value stable across
+            # serialisation:
+            first = self.last_change or min(first_seen, default=None)
+            yield Statement(
+                canonical_id=self.id,
+                entity_id=self.id,
+                prop=BASE_ID,
+                schema=self.schema.name,
+                value=checksum,
+                dataset=self.dataset.name,
+                first_seen=first,
+                last_seen=max(last_seen, default=None),
+            )
+
+    @property
+    def first_seen(self) -> Optional[str]:
+        seen = (s.first_seen for s in self._iter_stmt() if s.first_seen is not None)
+        return min(seen, default=None)
+
+    @property
+    def last_seen(self) -> Optional[str]:
+        seen = (s.last_seen for s in self._iter_stmt() if s.last_seen is not None)
+        return max(seen, default=None)
+
+    @property
+    def datasets(self) -> Set[str]:
+        datasets: Set[str] = set()
+        for stmt in self._iter_stmt():
+            datasets.add(stmt.dataset)
+        return datasets
+
+    @property
+    def referents(self) -> Set[str]:
+        referents: Set[str] = set(self.extra_referents)
+        for stmt in self._iter_stmt():
+            if stmt.entity_id is not None and stmt.entity_id != self.id:
+                referents.add(stmt.entity_id)
+        return referents
+
+    @property
+    def key_prefix(self) -> Optional[str]:
+        return self.dataset.name
+
+    @key_prefix.setter
+    def key_prefix(self, dataset: Optional[str]) -> None:
+        raise NotImplementedError()
+
+    def add_statement(self, stmt: Statement) -> None:
+        schema = self.schema
+        if not schema.is_a(stmt.schema):
+            try:
+                self.schema = schema.model.common_schema(schema, stmt.schema)
+            except InvalidData as exc:
+                raise InvalidData(f"{self.id}: {exc}") from exc
+
+        if stmt.prop == BASE_ID:
+            if stmt.first_seen is not None:
+                # The last_change attribute describes the latest checksum change
+                # of any emitted component of the entity, which is stored in the BASE
+                # field.
+                if self.last_change is None:
+                    self.last_change = stmt.first_seen
+                else:
+                    self.last_change = max(self.last_change, stmt.first_seen)
+        else:
+            self._statements.setdefault(stmt.prop, set())
+            self._statements[stmt.prop].add(stmt)
+
+    def get(self, prop: P, quiet: bool = False) -> List[str]:
+        prop_name = self._prop_name(prop, quiet=quiet)
+        if prop_name is None or prop_name not in self._statements:
+            return []
+        return list({s.value for s in self._statements[prop_name]})
+
+    def get_statements(self, prop: P, quiet: bool = False) -> List[Statement]:
+        prop_name = self._prop_name(prop, quiet=quiet)
+        if prop_name is None or prop_name not in self._statements:
+            return []
+        return list(self._statements[prop_name])
+
+    def set(
+        self,
+        prop: P,
+        values: Values,
+        cleaned: bool = False,
+        quiet: bool = False,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        lang: Optional[str] = None,
+        original_value: Optional[str] = None,
+        origin: Optional[str] = None,
+    ) -> None:
+        prop_name = self._prop_name(prop, quiet=quiet)
+        if prop_name is None:
+            return
+        self._statements.pop(prop_name, None)
+        return self.add(
+            prop,
+            values,
+            cleaned=cleaned,
+            quiet=quiet,
+            fuzzy=fuzzy,
+            format=format,
+            lang=lang,
+            original_value=original_value,
+            origin=origin,
+        )
+
+    def add(
+        self,
+        prop: P,
+        values: Values,
+        cleaned: bool = False,
+        quiet: bool = False,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        lang: Optional[str] = None,
+        original_value: Optional[str] = None,
+        origin: Optional[str] = None,
+    ) -> None:
+        prop_name = self._prop_name(prop, quiet=quiet)
+        if prop_name is None:
+            return None
+        prop = self.schema.properties[prop_name]
+        for value in string_list(values, sanitize=not cleaned):
+            self.unsafe_add(
+                prop,
+                value,
+                cleaned=cleaned,
+                fuzzy=fuzzy,
+                format=format,
+                quiet=quiet,
+                lang=lang,
+                original_value=original_value,
+                origin=origin,
+            )
+        return None
+
+    def unsafe_add(
+        self,
+        prop: Property,
+        value: Optional[str],
+        cleaned: bool = False,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        quiet: bool = False,
+        schema: Optional[str] = None,
+        dataset: Optional[str] = None,
+        seen: Optional[str] = None,
+        lang: Optional[str] = None,
+        original_value: Optional[str] = None,
+        origin: Optional[str] = None,
+    ) -> Optional[str]:
+        """Add a statement to the entity, possibly the value."""
+        if value is None or len(value) == 0:
+            return None
+
+        # Don't allow setting the reverse properties:
+        if prop.stub:
+            if quiet:
+                return None
+            msg = gettext("Stub property (%s): %s")
+            raise InvalidData(msg % (self.schema, prop))
+
+        if lang is not None:
+            lang = registry.language.clean_text(lang)
+
+        clean: Optional[str] = value
+        if not cleaned:
+            clean = prop.type.clean_text(value, proxy=self, fuzzy=fuzzy, format=format)
+
+        if clean is None:
+            return None
+
+        if original_value is None and clean != value:
+            original_value = value
+
+        if self.id is None:
+            raise InvalidData("Cannot add statement to entity without ID!")
+        stmt = Statement(
+            entity_id=self.id,
+            prop=prop.name,
+            schema=schema or self.schema.name,
+            value=clean,
+            dataset=dataset or self.dataset.name,
+            lang=lang,
+            original_value=original_value,
+            first_seen=seen,
+            origin=origin,
+        )
+        self.add_statement(stmt)
+        return clean
+
+    def pop(self, prop: P, quiet: bool = True) -> List[str]:
+        prop_name = self._prop_name(prop, quiet=quiet)
+        if prop_name is None or prop_name not in self._statements:
+            return []
+        return list({s.value for s in self._statements.pop(prop_name, [])})
+
+    def remove(self, prop: P, value: str, quiet: bool = True) -> None:
+        prop_name = self._prop_name(prop, quiet=quiet)
+        if prop_name is not None and prop_name in self._properties:
+            stmts = {s for s in self._statements[prop_name] if s.value != value}
+            self._statements[prop_name] = stmts
+
+    def itervalues(self) -> Generator[Tuple[Property, str], None, None]:
+        for name, statements in self._statements.items():
+            prop = self.schema.properties[name]
+            for value in set((s.value for s in statements)):
+                yield (prop, value)
+
+    def get_type_values(
+        self, type_: PropertyType, matchable: bool = False
+    ) -> List[str]:
+        combined: Set[str] = set()
+        for stmt in self.get_type_statements(type_, matchable=matchable):
+            combined.add(stmt.value)
+        return list(combined)
+
+    def get_type_statements(
+        self, type_: PropertyType, matchable: bool = False
+    ) -> List[Statement]:
+        combined = []
+        for prop_name, statements in self._statements.items():
+            prop = self.schema.properties[prop_name]
+            if matchable and not prop.matchable:
+                continue
+            if prop.type == type_:
+                for statement in statements:
+                    combined.append(statement)
+        return combined
+
+    @property
+    def properties(self) -> Dict[str, List[str]]:
+        return {p: list({s.value for s in vs}) for p, vs in self._statements.items()}
+
+    def iterprops(self) -> List[Property]:
+        return [self.schema.properties[p] for p in self._statements.keys()]
+
+    def clone(self: SE) -> SE:
+        data = {"schema": self.schema.name, "id": self.id}
+        cloned = type(self)(self.dataset, data)
+        for stmt in self._iter_stmt():
+            cloned.add_statement(stmt)
+        return cloned
+
+    def merge(self: SE, other: EntityProxy) -> SE:
+        try:
+            self.schema = self.schema.model.common_schema(self.schema, other.schema)
+        except InvalidData as e:
+            msg = "Cannot merge entities with id %s: %s"
+            raise InvalidData(msg % (self.id, e))
+
+        if not isinstance(other, StatementEntity):
+            for prop, values in other._properties.items():
+                self.add(prop, values, cleaned=True, quiet=True)
+            return self
+        for stmt in other._iter_stmt():
+            if self.id is not None:
+                stmt.canonical_id = self.id
+            self.add_statement(stmt)
+        self.extra_referents.update(other.extra_referents)
+        return self
+
+    def to_dict(self) -> Dict[str, Any]:
+        data: Dict[str, Any] = {
+            "id": self.id,
+            "caption": self.caption,
+            "schema": self.schema.name,
+            "properties": self.properties,
+            "referents": list(self.referents),
+            "datasets": list(self.datasets),
+        }
+        if self.first_seen is not None:
+            data["first_seen"] = self.first_seen
+        if self.last_seen is not None:
+            data["last_seen"] = self.last_seen
+        if self.last_change is not None:
+            data["last_change"] = self.last_change
+        return data
+
+    def to_statement_dict(self) -> Dict[str, Any]:
+        """Return a dictionary representation of the entity's statements."""
+        data: Dict[str, Any] = {
+            "id": self.id,
+            "caption": self.caption,
+            "schema": self.schema.name,
+            "statements": [stmt.to_dict() for stmt in self.statements],
+            "referents": list(self.referents),
+            "datasets": list(self.datasets),
+        }
+        if self.first_seen is not None:
+            data["first_seen"] = self.first_seen
+        if self.last_seen is not None:
+            data["last_seen"] = self.last_seen
+        if self.last_change is not None:
+            data["last_change"] = self.last_change
+        return data
+
+    def __len__(self) -> int:
+        return len(list(self._iter_stmt())) + 1
+
+    @classmethod
+    def from_dict(
+        cls: Type[SE],
+        data: Dict[str, Any],
+        cleaned: bool = True,
+        default_dataset: Optional[Dataset] = None,
+    ) -> SE:
+        # Exists only for backwards compatibility.
+        dataset = default_dataset or DefaultDataset
+        return cls(dataset, data, cleaned=cleaned)
+
+    @classmethod
+    def from_data(
+        cls: Type[SE],
+        dataset: Dataset,
+        data: Dict[str, Any],
+        cleaned: bool = True,
+    ) -> SE:
+        return cls(dataset, data, cleaned=cleaned)
+
+    @classmethod
+    def from_statements(
+        cls: Type[SE],
+        dataset: Dataset,
+        statements: Iterable[Statement],
+    ) -> SE:
+        obj: Optional[SE] = None
+        for stmt in statements:
+            if obj is None:
+                data = {"schema": stmt.schema, "id": stmt.canonical_id}
+                obj = cls(dataset, data)
+            obj.add_statement(stmt)
+        if obj is None:
+            raise ValueError("No statements given!")
+        return obj