followthemoney 3.8.5__py3-none-any.whl → 4.0.1__py3-none-any.whl

followthemoney/proxy.py

@@ -1,36 +1,25 @@
 import logging
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Dict,
-    Generator,
-    List,
-    Optional,
-    Set,
-    Tuple,
-    Union,
-    Type,
-    TypeVar,
-    cast,
-)
-import warnings
+from typing import TYPE_CHECKING, cast, Any
+from typing import Dict, Generator, List, Optional, Set, Tuple, Union, Type, TypeVar
 from itertools import product
 from banal import ensure_dict
+from rigour.names import pick_name
 
 from followthemoney.exc import InvalidData
 from followthemoney.types import registry
 from followthemoney.types.common import PropertyType
 from followthemoney.property import Property
-from followthemoney.rdf import SKOS, RDF, Literal, URIRef, Identifier
+from followthemoney.value import string_list, Values
 from followthemoney.util import sanitize_text, gettext
-from followthemoney.util import merge_context, value_list, make_entity_id
+from followthemoney.util import merge_context, make_entity_id
+from followthemoney.model import Model
+from followthemoney.schema import Schema
 
 if TYPE_CHECKING:
     from followthemoney.model import Model
 
 log = logging.getLogger(__name__)
 P = Union[Property, str]
-Triple = Tuple[Identifier, Identifier, Identifier]
 E = TypeVar("E", bound="EntityProxy")
 
 
@@ -45,7 +34,7 @@ class EntityProxy(object):
 
     def __init__(
         self,
-        model: "Model",
+        schema: Schema,
         data: Dict[str, Any],
         key_prefix: Optional[str] = None,
         cleaned: bool = True,
@@ -57,9 +46,6 @@ class EntityProxy(object):
 
         #: The schema definition for this entity, which implies the properties
         #: That can be set on it.
-        schema = model.get(data.pop("schema", None))
-        if schema is None:
-            raise InvalidData(gettext("No schema for entity."))
         self.schema = schema
 
         #: When using :meth:`~make_id` to generate a natural key for this entity,
@@ -162,7 +148,7 @@ class EntityProxy(object):
     def add(
         self,
         prop: P,
-        values: Any,
+        values: Values,
         cleaned: bool = False,
         quiet: bool = False,
         fuzzy: bool = False,
@@ -192,11 +178,9 @@ class EntityProxy(object):
             msg = gettext("Stub property (%s): %s")
             raise InvalidData(msg % (self.schema, prop))
 
-        for value in value_list(values):
-            if not cleaned:
-                format = format or prop.format
-                value = prop.type.clean(value, proxy=self, fuzzy=fuzzy, format=format)
-            self.unsafe_add(prop, value, cleaned=True)
+        value: Optional[str] = None
+        for value in string_list(values, sanitize=not cleaned):
+            self.unsafe_add(prop, value, cleaned=cleaned, fuzzy=fuzzy, format=format)
         return None
 
     def unsafe_add(
@@ -236,7 +220,7 @@ class EntityProxy(object):
     def set(
         self,
         prop: P,
-        values: Any,
+        values: Values,
         cleaned: bool = False,
         quiet: bool = False,
         fuzzy: bool = False,
@@ -377,34 +361,21 @@ class EntityProxy(object):
                 data[group] = values
         return data
 
-    def triples(self, qualified: bool = True) -> Generator[Triple, None, None]:
-        """Serialise the entity into a set of RDF triple statements. The
-        statements include the property values, an ``RDF#type`` definition
-        that refers to the entity schema, and a ``SKOS#prefLabel`` with the
-        entity caption."""
-        if self.id is None or self.schema is None:
-            return
-        uri = registry.entity.rdf(self.id)
-        yield (uri, RDF.type, self.schema.uri)
-        if qualified:
-            caption = self.caption
-            if caption != self.schema.label:
-                yield (uri, SKOS.prefLabel, Literal(caption))
-        for prop, value in self.itervalues():
-            value = prop.type.rdf(value)
-            if qualified:
-                yield (uri, prop.uri, value)
-            else:
-                yield (uri, URIRef(prop.name), value)
-
     @property
     def caption(self) -> str:
         """The user-facing label to be used for this entity. This checks a list
         of properties defined by the schema (caption) and returns the first
         available value. If no caption is available, return the schema label."""
-        for prop in self.schema.caption:
-            for value in self.get(prop):
-                return value
+        for prop_ in self.schema.caption:
+            prop = self.schema.properties[prop_]
+            values = self.get(prop)
+            if prop.type == registry.name and len(values) > 1:
+                name = pick_name(sorted(values))
+                if name is not None:
+                    return name
+            else:
+                for value in values:
+                    return value
         return self.schema.label
 
     @property
@@ -448,7 +419,7 @@ class EntityProxy(object):
 
     def clone(self: E) -> E:
         """Make a deep copy of the current entity proxy."""
-        return self.__class__.from_dict(self.schema.model, self.to_dict())
+        return self.__class__.from_dict(self.to_dict())
 
     def merge(self: E, other: E) -> E:
         """Merge another entity proxy into this one. This will try and find
@@ -467,30 +438,36 @@ class EntityProxy(object):
             self.add(prop, values, cleaned=True, quiet=True)
         return self
 
+    def __getstate__(self) -> Dict[str, Any]:
+        data = {slot: getattr(self, slot) for slot in self.__slots__}
+        data["schema"] = self.schema.name
+        return data
+
+    def __setstate__(self, data: Dict[str, Any]) -> None:
+        for slot in self.__slots__:
+            value = data.get(slot)
+            if slot == "schema":
+                value = Model.instance()[data["schema"]]
+            setattr(self, slot, value)
+
     def __str__(self) -> str:
         return self.caption
 
     def __repr__(self) -> str:
-        return "<E(%r,%r)>" % (self.id, str(self))
+        return "<E(%r,%s,%r)>" % (self.id, self.schema.name, str(self))
 
     def __len__(self) -> int:
         return self._size
 
     def __hash__(self) -> int:
         if not self.id:
-            warnings.warn(
-                "Hashing an EntityProxy without an ID results in undefined behaviour",
-                RuntimeWarning,
-            )
+            raise RuntimeError("Cannot hash entity without an ID")
         return hash(self.id)
 
     def __eq__(self, other: Any) -> bool:
         try:
             if self.id is None or other.id is None:
-                warnings.warn(
-                    "Comparing EntityProxys without IDs results in undefined behaviour",
-                    RuntimeWarning,
-                )
+                raise RuntimeError("Cannot compare entities without IDs.")
             return bool(self.id == other.id)
         except AttributeError:
             return False
@@ -498,11 +475,13 @@ class EntityProxy(object):
     @classmethod
     def from_dict(
         cls: Type[E],
-        model: "Model",
         data: Dict[str, Any],
         cleaned: bool = True,
     ) -> E:
         """Instantiate a proxy based on the given model and serialised dictionary.
 
         Use :meth:`followthemoney.model.Model.get_proxy` instead."""
-        return cls(model, data, cleaned=cleaned)
+        schema = Model.instance().get(data.get("schema", ""))
+        if schema is None:
+            raise InvalidData(gettext("No schema for entity."))
+        return cls(schema, data, cleaned=cleaned)

followthemoney/schema/Analyzable.yaml

@@ -32,9 +32,8 @@ Analyzable:
     ibanMentioned:
       label: "Detected IBANs"
       hidden: true
-      # type: identifier
-      # format: iban
-      type: iban
+      type: identifier
+      format: iban
     ipMentioned:
       label: "Detected IP addresses"
       hidden: true

followthemoney/schema/BankAccount.yaml

@@ -30,9 +30,8 @@ BankAccount:
       maxLength: 64
     iban:
       label: IBAN
-      # type: identifier
-      # format: iban
-      type: iban
+      type: identifier
+      format: iban
       maxLength: 64
     bic:
       label: Bank Identifier Code

followthemoney/schema/Company.yaml

@@ -56,12 +56,6 @@ Company:
     mbsCode:
       label: "MBS"
       type: identifier
-    ibcRuc:
-      # TODO: Remove this. It's a column name in the ICIJ-released OffshoreLeaks datasets
-      # but seems to just mean "company number".
-      deprecated: true
-      label: "ibcRUC"
-      type: identifier
     caemCode:
       label: "COD CAEM"
       description: "(RO) What kind of activity a legal entity is allowed to develop"

followthemoney/schema/Contract.yaml

@@ -68,4 +68,3 @@ Contract:
     language:
       label: "Language"
       type: language
-      rdf: http://purl.org/dc/terms/language

followthemoney/schema/CryptoWallet.yaml

@@ -31,7 +31,7 @@ CryptoWallet:
       type: date
     currencySymbol:
       label: Currency short code
-    mangingExchange:
+    managingExchange:
       label: Managing exchange
     holder:
       label: Wallet holder

followthemoney/schema/Document.yaml

@@ -30,12 +30,10 @@ Document:
     title:
       label: "Title"
       type: string
-      rdf: http://purl.org/dc/elements/1.1/title
       caption: true
     author:
       label: "Author"
       description: "The original author, not the uploader"
-      rdf: http://purl.org/dc/elements/1.1/creator
     generator:
       label: "Generator"
       description: "The program used to generate this file"
@@ -61,11 +59,9 @@ Document:
     mimeType:
       label: "MIME type"
       type: mimetype
-      rdf: http://purl.org/dc/terms/format
     language:
       label: "Language"
       type: language
-      rdf: http://purl.org/dc/terms/language
     translatedLanguage:
       label: "The language of the translated text"
       hidden: true
@@ -78,7 +74,6 @@ Document:
       label: "Date"
       description: "If not otherwise specified"
       type: date
-      rdf: http://purl.org/dc/elements/1.1/date
     authoredAt:
       label: "Authored on"
       type: date
@@ -95,7 +90,6 @@ Document:
         name: children
         label: "Child documents"
         hidden: true
-      rdf: http://purl.org/dc/terms/isPartOf
     ancestors:
       label: "Ancestors"
       type: entity

followthemoney/schema/Interval.yaml

@@ -42,6 +42,13 @@ Interval:
       label: "Source link"
       type: url
       matchable: false
+    proof:
+      label: Source document
+      reverse:
+        name: provenIntervals
+        label: "Derived relationships"
+      type: entity
+      range: Document
     publisher:
       label: "Publishing source"
     publisherUrl:

followthemoney/schema/LegalEntity.yaml

@@ -102,6 +102,12 @@ LegalEntity:
     bvdId:
       label: Bureau van Dijk ID
       type: identifier
+    uscCode:
+      # cf. https://en.wikipedia.org/wiki/Unified_Social_Credit_Identifier
+      label: "USCC"
+      description: "Unified Social Credit Identifier"
+      type: identifier
+      format: uscc
     icijId:
       label: ICIJ ID
       description: "ID according to International Consortium for Investigative Journalists"

followthemoney/schema/License.yaml

@@ -19,6 +19,8 @@ License:
   properties:
     area:
       label: "Area"
+      type: number
+      description: "Spatial area of the license or concession"
     commodities:
       label: "Commodities"
     reviewDate:

followthemoney/schema/Page.yaml

@@ -21,7 +21,6 @@ Page:
         name: pages
         label: "Pages"
         hidden: true
-      rdf: http://purl.org/dc/terms/isPartOf
     detectedLanguage:
       label: "Detected language"
       type: language

followthemoney/schema/Person.yaml

@@ -6,7 +6,6 @@ Person:
   description: >
     A natural person, as opposed to a corporation of some type.
   matchable: true
-  rdf: http://xmlns.com/foaf/0.1/Person
   featured:
     - name
     - nationality
@@ -26,14 +25,12 @@ Person:
   properties:
     title:
       label: Title
-      rdf: http://xmlns.com/foaf/0.1/title
     # The `firstName`, `lastName`, `secondName` etc. properties intentionally do not use
     # the `name` property type. Many FtM tools (including Aleph) use name properties to
     # compare/match entities, but matching entites just on e.g. a first name would lead to
     # too many false positives.
     firstName:
       label: First name
-      rdf: http://xmlns.com/foaf/0.1/givenName
     secondName:
       label: Second name
     middleName:
@@ -44,13 +41,11 @@ Person:
       label: Matronymic
     lastName:
       label: Last name
-      rdf: http://xmlns.com/foaf/0.1/lastName
     nameSuffix:
       label: Name suffix
     birthDate:
       label: Birth date
       type: date
-      rdf: http://xmlns.com/foaf/0.1/birthday
     birthPlace:
       label: Place of birth
     birthCountry:

followthemoney/schema/Sanction.yaml

@@ -49,6 +49,7 @@ Sanction:
       label: "Status"
     duration:
       label: "Duration"
+      type: number
     reason:
       label: "Reason"
       type: text

followthemoney/schema/Thing.yaml

@@ -14,7 +14,6 @@ Thing:
     name:
       label: Name
       type: name
-      rdf: http://www.w3.org/2004/02/skos/core#prefLabel
     summary: # a short one-liner kind of description
       label: Summary
       type: text
@@ -27,7 +26,6 @@ Thing:
     alias:
       label: Other name
       type: name
-      rdf: http://www.w3.org/2004/02/skos/core#altLabel
     previousName:
       label: Previous name
       type: name

followthemoney/schema/UserAccount.yaml

@@ -28,11 +28,14 @@ UserAccount:
       label: "Service"
       type: string
     email:
-      label: "E-Mail"
+      label: E-Mail
       type: email
-    number:
-      label: "Phone Number"
+      description: "Email address"
+    phone:
+      label: Phone
       type: phone
+      description: "Phone number"
+      maxLength: 32
     username:
       label: "Username"
       type: string

followthemoney/schema.py

@@ -1,22 +1,12 @@
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Dict,
-    List,
-    Optional,
-    Set,
-    TypedDict,
-    Union,
-    cast,
-)
+from typing import TYPE_CHECKING, Any, cast
+from typing import Dict, List, Optional, Set, TypedDict, Union
 from banal import ensure_list, ensure_dict, as_bool
-from functools import lru_cache
+from functools import cache
 
 from followthemoney.property import Property, PropertySpec, PropertyToDict, ReverseSpec
 from followthemoney.types import registry
 from followthemoney.exc import InvalidData, InvalidModel
-from followthemoney.rdf import URIRef, NS
-from followthemoney.util import gettext
+from followthemoney.util import gettext, const
 
 if TYPE_CHECKING:
     from followthemoney.model import Model
@@ -47,7 +37,6 @@ class SchemaSpec(TypedDict, total=False):
     edge: EdgeSpec
     temporalExtent: TemporalExtentSpec
     description: Optional[str]
-    rdf: Optional[str]
     abstract: bool
     hidden: bool
     generated: bool
@@ -90,7 +79,6 @@ class Schema:
         "_plural",
         "_description",
         "_hash",
-        "uri",
         "abstract",
         "hidden",
         "generated",
@@ -118,15 +106,12 @@ class Schema:
 
     def __init__(self, model: "Model", name: str, data: SchemaSpec) -> None:
         #: Machine-readable name of the schema, used for identification.
-        self.name = name
+        self.name = const(name)
         self.model = model
         self._label = data.get("label", name)
         self._plural = data.get("plural", self.label)
         self._description = data.get("description")
-        self._hash = hash("<Schema(%r)>" % name)
-
-        #: RDF identifier for this schema when it is transformed to a triple term.
-        self.uri = URIRef(cast(str, data.get("rdf", NS[name])))
+        self._hash = hash("<Schema(%r)>" % self.name)
 
         #: Do not store or emit entities of this type, it is used only for
         #: inheritance.
@@ -152,17 +137,17 @@ class Schema:
         #: Mark a set of properties as important, i.e. they should be shown
         #: first, or in an abridged view of the entity. In Aleph, these properties
         #: are included in tabular entity listings.
-        self.featured = ensure_list(data.get("featured", []))
+        self.featured = [const(f) for f in data.get("featured", [])]
 
         #: Mark a set of properties as required. This is applied only when
         #: an entity is created by the user - bulk created entities will
         #: slip through even if it is technically invalid.
-        self.required = ensure_list(data.get("required", []))
+        self.required = [const(r) for r in data.get("required", [])]
 
         #: Mark a set of properties to be used for the entity's caption.
         #: They will be checked in order and the first existent value will
         #: be used.
-        self.caption = ensure_list(data.get("caption", []))
+        self.caption = [const(s) for s in data.get("caption", [])]
 
         # A transform of the entity into an edge for its representation in
         # the context of a property graph representation like Neo4J/Gephi.
@@ -173,7 +158,7 @@ class Schema:
         #: Flag to indicate if this schema should be represented by an edge (rather than
         #: a node) when the data is converted into a property graph.
         self.edge: bool = self.edge_source is not None and self.edge_target is not None
-        self.edge_caption = ensure_list(edge.get("caption", []))
+        self.edge_caption = [const(p) for p in edge.get("caption", [])]
         self._edge_label = edge.get("label", self._label)
 
         #: Flag to indicate if the edge should be presented as directed to the user,
@@ -183,16 +168,16 @@ class Schema:
         #: Specify which properties should be used to represent this schema in a
         #: timeline.
         temporal_extent = data.get("temporalExtent", {})
-        self._temporal_start = ensure_list(temporal_extent.get("start", []))
-        self._temporal_end = ensure_list(temporal_extent.get("end", []))
+        self._temporal_start = [const(s) for s in temporal_extent.get("start", [])]
+        self._temporal_end = [const(e) for e in temporal_extent.get("end", [])]
 
         #: Direct parent schemata of this schema.
-        self._extends = ensure_list(data.get("extends", []))
+        self._extends = [const(s) for s in data.get("extends", [])]
         self.extends: Set["Schema"] = set()
 
         #: All parents of this schema (including indirect parents and the schema
         #: itself).
-        self.schemata = set([self])
+        self.schemata: Set[Schema] = set([self])
 
         #: All names of :attr:`~schemata`.
         self.names = set([self.name])
@@ -205,8 +190,8 @@ class Schema:
         #: The full list of properties defined for the entity, including those
         #: inherited from parent schemata.
         self.properties: Dict[str, Property] = {}
-        for name, prop in data.get("properties", {}).items():
-            self.properties[name] = Property(self, name, prop)
+        for pname, prop in data.get("properties", {}).items():
+            self.properties[pname] = Property(self, pname, prop)
 
     def generate(self, model: "Model") -> None:
         """While loading the schema, this function will validate and
@@ -317,12 +302,18 @@ class Schema:
 
     @property
     def source_prop(self) -> Optional[Property]:
-        """The entity property to be used as an edge source."""
+        """The entity property to be used as an edge source when the schema is
+        considered as a relationship."""
+        if self.edge_source is None:
+            return None
         return self.get(self.edge_source)
 
     @property
     def target_prop(self) -> Optional[Property]:
-        """The entity property to be used as an edge target."""
+        """The entity property to be used as an edge target when the schema is transformed
+        into a relationship."""
+        if self.edge_target is None:
+            return None
         return self.get(self.edge_target)
 
     @property
@@ -391,12 +382,12 @@ class Schema:
                         self._matchable_schemata.add(schema)
         return self._matchable_schemata
 
-    @lru_cache(maxsize=None)
+    @cache
     def can_match(self, other: "Schema") -> bool:
         """Check if an schema can match with another schema."""
         return other in self.matchable_schemata
 
-    @lru_cache(maxsize=None)
+    @cache
     def is_a(self, other: Union[str, "Schema"]) -> bool:
         """Check if the schema or one of its parents is the same as the given
         candidate ``other``."""
@@ -404,13 +395,13 @@ class Schema:
             other = other.name
         return other in self.names
 
-    def get(self, name: Optional[str]) -> Optional[Property]:
+    def get(self, name: str) -> Optional[Property]:
         """Retrieve a property defined for this schema by its name."""
         if name is None:
             return None
         return self.properties.get(name)
 
-    def validate(self, data: Any) -> Optional[str]:
+    def validate(self, data: Dict[str, Any]) -> Optional[str]:
         """Validate a dictionary against the given schema.
         This will also drop keys which are not valid as properties.
         """
@@ -478,7 +469,7 @@ class Schema:
     def __eq__(self, other: Any) -> bool:
         """Compare two schemata (via hash)."""
         try:
-            return self._hash == hash(other)
+            return self._hash == other._hash  # type: ignore
         except AttributeError:
             return False
 
@@ -486,10 +477,7 @@ class Schema:
         return self.name.__lt__(other.name)
 
     def __hash__(self) -> int:
-        try:
-            return self._hash
-        except AttributeError:
-            return super().__hash__()
+        return self._hash
 
     def __repr__(self) -> str:
         return "<Schema(%r)>" % self.name

followthemoney/statement/__init__.py

@@ -0,0 +1,19 @@
+from followthemoney.statement.statement import Statement, StatementDict
+from followthemoney.statement.serialize import CSV, JSON, PACK, FORMATS
+from followthemoney.statement.serialize import write_statements
+from followthemoney.statement.serialize import read_statements, read_path_statements
+from followthemoney.statement.entity import SE, StatementEntity
+
+__all__ = [
+    "Statement",
+    "StatementDict",
+    "StatementEntity",
+    "SE",
+    "CSV",
+    "JSON",
+    "PACK",
+    "FORMATS",
+    "write_statements",
+    "read_statements",
+    "read_path_statements",
+]