followthemoney 3.8.4__py3-none-any.whl → 4.0.0__py3-none-any.whl

followthemoney/property.py

@@ -1,10 +1,9 @@
 from banal import is_mapping, as_bool
-from typing import TYPE_CHECKING, cast, Any, List, Optional, TypedDict
+from typing import TYPE_CHECKING, Any, List, Optional, TypedDict
 
 from followthemoney.exc import InvalidModel
 from followthemoney.types import registry
-from followthemoney.rdf import NS, URIRef
-from followthemoney.util import gettext, get_entity_id
+from followthemoney.util import gettext, get_entity_id, const
 
 if TYPE_CHECKING:
     from followthemoney.schema import Schema
@@ -26,7 +25,6 @@ class PropertyDict(TypedDict, total=False):
     deprecated: Optional[bool]
     maxLength: Optional[int]
     # stub: Optional[bool]
-    rdf: Optional[str]
     range: Optional[str]
     format: Optional[str]
 
@@ -66,7 +64,6 @@ class Property:
         "stub",
         "_reverse",
         "reverse",
-        "uri",
     )
 
     #: Invalid property names.
@@ -79,10 +76,10 @@ class Property:
         self.schema = schema
 
         #: Machine-readable name for this property.
-        self.name = name
+        self.name = const(name)
 
         #: Qualified property name, which also includes the schema name.
-        self.qname = "%s:%s" % (schema.name, self.name)
+        self.qname = const("%s:%s" % (schema.name, self.name))
         if self.name in self.RESERVED:
             raise InvalidModel("Reserved name: %s" % self.name)
 
@@ -97,12 +94,11 @@ class Property:
         #: This property should not be shown or mentioned in the user interface.
         self.hidden = as_bool(data.get("hidden"))
 
-        type_ = data.get("type", "string")
-        if type_ is None or type_ not in registry.named:
-            raise InvalidModel("Invalid type: %s" % type_)
-
+        type_ = data.get("type") or "string"
         #: The data type for this property.
-        self.type = registry[type_]
+        self.type = registry.get(type_)
+        if self.type is None:
+            raise InvalidModel("Invalid type: %s" % type_)
 
         #: Whether this property should be used for matching and cross-referencing.
         _matchable = data.get("matchable")
@@ -137,9 +133,6 @@ class Property:
         self._reverse = data.get("reverse")
         self.reverse: Optional["Property"] = None
 
-        #: RDF term for this property (i.e. the predicate URI).
-        self.uri = URIRef(cast(str, data.get("rdf", NS[self.qname])))
-
     def generate(self, model: "Model") -> None:
         """Setup method used when loading the model in order to build out the reverse
         links of the property."""
@@ -170,6 +163,10 @@ class Property:
             return 0.0
         return self.type.specificity(value)
 
+    def caption(self, value: str) -> str:
+        """Return a user-friendly caption for the given value."""
+        return self.type.caption(value, format=self.format)
+
     def validate(self, data: List[Any]) -> Optional[str]:
         """Validate that the data should be stored.
 

followthemoney/proxy.py

@@ -1,36 +1,25 @@
 import logging
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Dict,
-    Generator,
-    List,
-    Optional,
-    Set,
-    Tuple,
-    Union,
-    Type,
-    TypeVar,
-    cast,
-)
-import warnings
+from typing import TYPE_CHECKING, cast, Any
+from typing import Dict, Generator, List, Optional, Set, Tuple, Union, Type, TypeVar
 from itertools import product
 from banal import ensure_dict
+from rigour.names import pick_name
 
 from followthemoney.exc import InvalidData
 from followthemoney.types import registry
 from followthemoney.types.common import PropertyType
 from followthemoney.property import Property
-from followthemoney.rdf import SKOS, RDF, Literal, URIRef, Identifier
+from followthemoney.value import string_list, Values
 from followthemoney.util import sanitize_text, gettext
-from followthemoney.util import merge_context, value_list, make_entity_id
+from followthemoney.util import merge_context, make_entity_id
+from followthemoney.model import Model
+from followthemoney.schema import Schema
 
 if TYPE_CHECKING:
     from followthemoney.model import Model
 
 log = logging.getLogger(__name__)
 P = Union[Property, str]
-Triple = Tuple[Identifier, Identifier, Identifier]
 E = TypeVar("E", bound="EntityProxy")
 
 
@@ -45,7 +34,7 @@ class EntityProxy(object):
 
     def __init__(
         self,
-        model: "Model",
+        schema: Schema,
         data: Dict[str, Any],
         key_prefix: Optional[str] = None,
         cleaned: bool = True,
@@ -57,9 +46,6 @@ class EntityProxy(object):
 
         #: The schema definition for this entity, which implies the properties
         #: That can be set on it.
-        schema = model.get(data.pop("schema", None))
-        if schema is None:
-            raise InvalidData(gettext("No schema for entity."))
         self.schema = schema
 
         #: When using :meth:`~make_id` to generate a natural key for this entity,
@@ -71,7 +57,7 @@ class EntityProxy(object):
         #: A unique identifier for this entity, usually a hashed natural key,
         #: a UUID, or a very simple slug. Can be signed using a
         #: :class:`~followthemoney.namespace.Namespace`.
-        self.id = data.pop("id", None)
+        self.id = str(data["id"]) if "id" in data else None
         if not cleaned:
             self.id = sanitize_text(self.id)
 
@@ -162,7 +148,7 @@ class EntityProxy(object):
     def add(
         self,
         prop: P,
-        values: Any,
+        values: Values,
         cleaned: bool = False,
         quiet: bool = False,
         fuzzy: bool = False,
@@ -192,11 +178,9 @@ class EntityProxy(object):
             msg = gettext("Stub property (%s): %s")
             raise InvalidData(msg % (self.schema, prop))
 
-        for value in value_list(values):
-            if not cleaned:
-                format = format or prop.format
-                value = prop.type.clean(value, proxy=self, fuzzy=fuzzy, format=format)
-            self.unsafe_add(prop, value, cleaned=True)
+        value: Optional[str] = None
+        for value in string_list(values, sanitize=not cleaned):
+            self.unsafe_add(prop, value, cleaned=cleaned, fuzzy=fuzzy, format=format)
         return None
 
     def unsafe_add(
@@ -236,7 +220,7 @@ class EntityProxy(object):
     def set(
         self,
         prop: P,
-        values: Any,
+        values: Values,
         cleaned: bool = False,
         quiet: bool = False,
         fuzzy: bool = False,
@@ -377,34 +361,21 @@ class EntityProxy(object):
                 data[group] = values
         return data
 
-    def triples(self, qualified: bool = True) -> Generator[Triple, None, None]:
-        """Serialise the entity into a set of RDF triple statements. The
-        statements include the property values, an ``RDF#type`` definition
-        that refers to the entity schema, and a ``SKOS#prefLabel`` with the
-        entity caption."""
-        if self.id is None or self.schema is None:
-            return
-        uri = registry.entity.rdf(self.id)
-        yield (uri, RDF.type, self.schema.uri)
-        if qualified:
-            caption = self.caption
-            if caption != self.schema.label:
-                yield (uri, SKOS.prefLabel, Literal(caption))
-        for prop, value in self.itervalues():
-            value = prop.type.rdf(value)
-            if qualified:
-                yield (uri, prop.uri, value)
-            else:
-                yield (uri, URIRef(prop.name), value)
-
     @property
     def caption(self) -> str:
         """The user-facing label to be used for this entity. This checks a list
         of properties defined by the schema (caption) and returns the first
         available value. If no caption is available, return the schema label."""
-        for prop in self.schema.caption:
-            for value in self.get(prop):
-                return value
+        for prop_ in self.schema.caption:
+            prop = self.schema.properties[prop_]
+            values = self.get(prop)
+            if prop.type == registry.name and len(values) > 1:
+                name = pick_name(sorted(values))
+                if name is not None:
+                    return name
+            else:
+                for value in values:
+                    return value
         return self.schema.label
 
     @property
@@ -448,7 +419,7 @@ class EntityProxy(object):
 
     def clone(self: E) -> E:
         """Make a deep copy of the current entity proxy."""
-        return self.__class__.from_dict(self.schema.model, self.to_dict())
+        return self.__class__.from_dict(self.to_dict())
 
     def merge(self: E, other: E) -> E:
         """Merge another entity proxy into this one. This will try and find
@@ -467,30 +438,36 @@ class EntityProxy(object):
             self.add(prop, values, cleaned=True, quiet=True)
         return self
 
+    def __getstate__(self) -> Dict[str, Any]:
+        data = {slot: getattr(self, slot) for slot in self.__slots__}
+        data["schema"] = self.schema.name
+        return data
+
+    def __setstate__(self, data: Dict[str, Any]) -> None:
+        for slot in self.__slots__:
+            value = data.get(slot)
+            if slot == "schema":
+                value = Model.instance()[data["schema"]]
+            setattr(self, slot, value)
+
     def __str__(self) -> str:
         return self.caption
 
     def __repr__(self) -> str:
-        return "<E(%r,%r)>" % (self.id, str(self))
+        return "<E(%r,%s,%r)>" % (self.id, self.schema.name, str(self))
 
     def __len__(self) -> int:
         return self._size
 
     def __hash__(self) -> int:
         if not self.id:
-            warnings.warn(
-                "Hashing an EntityProxy without an ID results in undefined behaviour",
-                RuntimeWarning,
-            )
+            raise RuntimeError("Cannot hash entity without an ID")
         return hash(self.id)
 
     def __eq__(self, other: Any) -> bool:
         try:
             if self.id is None or other.id is None:
-                warnings.warn(
-                    "Comparing EntityProxys without IDs results in undefined behaviour",
-                    RuntimeWarning,
-                )
+                raise RuntimeError("Cannot compare entities without IDs.")
             return bool(self.id == other.id)
         except AttributeError:
             return False
@@ -498,11 +475,13 @@ class EntityProxy(object):
     @classmethod
     def from_dict(
         cls: Type[E],
-        model: "Model",
         data: Dict[str, Any],
         cleaned: bool = True,
     ) -> E:
         """Instantiate a proxy based on the given model and serialised dictionary.
 
         Use :meth:`followthemoney.model.Model.get_proxy` instead."""
-        return cls(model, data, cleaned=cleaned)
+        schema = Model.instance().get(data.get("schema", ""))
+        if schema is None:
+            raise InvalidData(gettext("No schema for entity."))
+        return cls(schema, data, cleaned=cleaned)

followthemoney/schema/Analyzable.yaml

@@ -32,9 +32,8 @@ Analyzable:
     ibanMentioned:
       label: "Detected IBANs"
       hidden: true
-      # type: identifier
-      # format: iban
-      type: iban
+      type: identifier
+      format: iban
     ipMentioned:
       label: "Detected IP addresses"
       hidden: true

followthemoney/schema/BankAccount.yaml

@@ -30,9 +30,8 @@ BankAccount:
       maxLength: 64
     iban:
       label: IBAN
-      # type: identifier
-      # format: iban
-      type: iban
+      type: identifier
+      format: iban
       maxLength: 64
     bic:
       label: Bank Identifier Code

followthemoney/schema/Company.yaml

@@ -56,12 +56,6 @@ Company:
     mbsCode:
       label: "MBS"
       type: identifier
-    ibcRuc:
-      # TODO: Remove this. It's a column name in the ICIJ-released OffshoreLeaks datasets
-      # but seems to just mean "company number".
-      deprecated: true
-      label: "ibcRUC"
-      type: identifier
     caemCode:
       label: "COD CAEM"
       description: "(RO) What kind of activity a legal entity is allowed to develop"

followthemoney/schema/Contract.yaml

@@ -68,4 +68,3 @@ Contract:
     language:
       label: "Language"
       type: language
-      rdf: http://purl.org/dc/terms/language

followthemoney/schema/CryptoWallet.yaml

@@ -31,7 +31,7 @@ CryptoWallet:
       type: date
     currencySymbol:
       label: Currency short code
-    mangingExchange:
+    managingExchange:
       label: Managing exchange
     holder:
       label: Wallet holder

followthemoney/schema/Document.yaml

@@ -30,12 +30,10 @@ Document:
     title:
       label: "Title"
       type: string
-      rdf: http://purl.org/dc/elements/1.1/title
       caption: true
     author:
       label: "Author"
       description: "The original author, not the uploader"
-      rdf: http://purl.org/dc/elements/1.1/creator
     generator:
       label: "Generator"
       description: "The program used to generate this file"
@@ -61,11 +59,9 @@ Document:
     mimeType:
       label: "MIME type"
       type: mimetype
-      rdf: http://purl.org/dc/terms/format
     language:
       label: "Language"
       type: language
-      rdf: http://purl.org/dc/terms/language
     translatedLanguage:
       label: "The language of the translated text"
       hidden: true
@@ -78,7 +74,6 @@ Document:
       label: "Date"
       description: "If not otherwise specified"
       type: date
-      rdf: http://purl.org/dc/elements/1.1/date
     authoredAt:
       label: "Authored on"
       type: date
@@ -95,7 +90,6 @@ Document:
         name: children
         label: "Child documents"
         hidden: true
-      rdf: http://purl.org/dc/terms/isPartOf
     ancestors:
       label: "Ancestors"
       type: entity

followthemoney/schema/Interval.yaml

@@ -42,6 +42,13 @@ Interval:
       label: "Source link"
       type: url
       matchable: false
+    proof:
+      label: Source document
+      reverse:
+        name: provenIntervals
+        label: "Derived relationships"
+      type: entity
+      range: Document
     publisher:
       label: "Publishing source"
     publisherUrl:

followthemoney/schema/LegalEntity.yaml

@@ -102,6 +102,12 @@ LegalEntity:
     bvdId:
       label: Bureau van Dijk ID
       type: identifier
+    uscCode:
+      # cf. https://en.wikipedia.org/wiki/Unified_Social_Credit_Identifier
+      label: "USCC"
+      description: "Unified Social Credit Identifier"
+      type: identifier
+      format: uscc
     icijId:
       label: ICIJ ID
       description: "ID according to International Consortium for Investigative Journalists"

followthemoney/schema/License.yaml

@@ -19,6 +19,8 @@ License:
   properties:
     area:
       label: "Area"
+      type: number
+      description: "Spatial area of the license or concession"
     commodities:
       label: "Commodities"
     reviewDate:

followthemoney/schema/Page.yaml

@@ -21,7 +21,6 @@ Page:
         name: pages
         label: "Pages"
         hidden: true
-      rdf: http://purl.org/dc/terms/isPartOf
     detectedLanguage:
       label: "Detected language"
       type: language

followthemoney/schema/Person.yaml

@@ -6,7 +6,6 @@ Person:
   description: >
     A natural person, as opposed to a corporation of some type.
   matchable: true
-  rdf: http://xmlns.com/foaf/0.1/Person
   featured:
     - name
     - nationality
@@ -26,14 +25,12 @@ Person:
   properties:
     title:
       label: Title
-      rdf: http://xmlns.com/foaf/0.1/title
     # The `firstName`, `lastName`, `secondName` etc. properties intentionally do not use
     # the `name` property type. Many FtM tools (including Aleph) use name properties to
     # compare/match entities, but matching entites just on e.g. a first name would lead to
     # too many false positives.
     firstName:
       label: First name
-      rdf: http://xmlns.com/foaf/0.1/givenName
     secondName:
       label: Second name
     middleName:
@@ -44,13 +41,11 @@ Person:
       label: Matronymic
     lastName:
       label: Last name
-      rdf: http://xmlns.com/foaf/0.1/lastName
     nameSuffix:
       label: Name suffix
     birthDate:
       label: Birth date
       type: date
-      rdf: http://xmlns.com/foaf/0.1/birthday
     birthPlace:
       label: Place of birth
     birthCountry:

followthemoney/schema/Sanction.yaml

@@ -49,6 +49,7 @@ Sanction:
       label: "Status"
     duration:
       label: "Duration"
+      type: number
     reason:
       label: "Reason"
       type: text

followthemoney/schema/Thing.yaml

@@ -14,7 +14,6 @@ Thing:
     name:
       label: Name
       type: name
-      rdf: http://www.w3.org/2004/02/skos/core#prefLabel
     summary: # a short one-liner kind of description
       label: Summary
       type: text
@@ -27,7 +26,6 @@ Thing:
     alias:
       label: Other name
       type: name
-      rdf: http://www.w3.org/2004/02/skos/core#altLabel
     previousName:
       label: Previous name
       type: name

followthemoney/schema/UserAccount.yaml

@@ -28,11 +28,14 @@ UserAccount:
       label: "Service"
       type: string
     email:
-      label: "E-Mail"
+      label: E-Mail
       type: email
-    number:
-      label: "Phone Number"
+      description: "Email address"
+    phone:
+      label: Phone
       type: phone
+      description: "Phone number"
+      maxLength: 32
     username:
       label: "Username"
       type: string