followthemoney 3.8.5__py3-none-any.whl → 4.0.1__py3-none-any.whl

followthemoney/statement/statement.py

@@ -0,0 +1,256 @@
+import hashlib
+import warnings
+from sqlalchemy.engine import Row
+from typing import cast
+from typing import Any, Dict, Generator, Optional
+from typing_extensions import TypedDict, Self
+from rigour.time import datetime_iso, iso_datetime
+from rigour.boolean import bool_text
+
+from followthemoney.proxy import EntityProxy
+from followthemoney.statement.util import get_prop_type, BASE_ID
+
+
+class StatementDict(TypedDict):
+    id: Optional[str]
+    entity_id: str
+    canonical_id: str
+    prop: str
+    schema: str
+    value: str
+    dataset: str
+    lang: Optional[str]
+    original_value: Optional[str]
+    external: bool
+    first_seen: Optional[str]
+    last_seen: Optional[str]
+    origin: Optional[str]
+
+
+class Statement(object):
+    """A single statement about a property relevant to an entity.
+
+    For example, this could be used to say: "In dataset A, entity X has the
+    property `name` set to 'John Smith'. I first observed this at K, and last
+    saw it at L."
+
+    Null property values are not supported. This might need to change if we
+    want to support making property-less entities.
+    """
+
+    BASE = BASE_ID
+
+    __slots__ = [
+        "id",
+        "entity_id",
+        "canonical_id",
+        "prop",
+        "schema",
+        "value",
+        "dataset",
+        "lang",
+        "original_value",
+        "external",
+        "first_seen",
+        "last_seen",
+        "origin",
+    ]
+
+    def __init__(
+        self,
+        entity_id: str,
+        prop: str,
+        schema: str,
+        value: str,
+        dataset: str,
+        lang: Optional[str] = None,
+        original_value: Optional[str] = None,
+        first_seen: Optional[str] = None,
+        external: bool = False,
+        id: Optional[str] = None,
+        canonical_id: Optional[str] = None,
+        last_seen: Optional[str] = None,
+        origin: Optional[str] = None,
+    ):
+        self.entity_id = entity_id
+        self.canonical_id = canonical_id or entity_id
+        self.prop = prop
+        self.schema = schema
+        self.value = value
+        self.dataset = dataset
+        self.lang = lang
+        self.original_value = original_value
+        self.first_seen = first_seen
+        self.last_seen = last_seen or first_seen
+        self.external = external
+        self.origin = origin
+        if id is None:
+            id = self.generate_key()
+        self.id = id
+
+    @property
+    def prop_type(self) -> str:
+        """The type of the property, e.g. 'string', 'number', 'url'."""
+        return get_prop_type(self.schema, self.prop)
+
+    def to_dict(self) -> StatementDict:
+        return {
+            "canonical_id": self.canonical_id,
+            "entity_id": self.entity_id,
+            "prop": self.prop,
+            "schema": self.schema,
+            "value": self.value,
+            "dataset": self.dataset,
+            "lang": self.lang,
+            "original_value": self.original_value,
+            "first_seen": self.first_seen,
+            "last_seen": self.last_seen,
+            "external": self.external,
+            "origin": self.origin,
+            "id": self.id,
+        }
+
+    def to_csv_row(self) -> Dict[str, Optional[str]]:
+        data = cast(Dict[str, Optional[str]], self.to_dict())
+        data["external"] = bool_text(self.external)
+        data["prop_type"] = get_prop_type(self.schema, self.prop)
+        return data
+
+    def to_db_row(self) -> Dict[str, Any]:
+        data = cast(Dict[str, Any], self.to_dict())
+        data["first_seen"] = iso_datetime(self.first_seen)
+        data["last_seen"] = iso_datetime(self.last_seen)
+        data["prop_type"] = get_prop_type(self.schema, self.prop)
+        return data
+
+    def __hash__(self) -> int:
+        if self.id is None:
+            warnings.warn(
+                "Hashing a statement without an ID results in undefined behaviour",
+                RuntimeWarning,
+            )
+        return hash(self.id)
+
+    def __repr__(self) -> str:
+        return "<Statement(%r, %r, %r)>" % (self.entity_id, self.prop, self.value)
+
+    def __eq__(self, other: Any) -> bool:
+        return not self.id != other.id
+
+    def __lt__(self, other: Any) -> bool:
+        self_key = (self.prop != BASE_ID, self.id or "")
+        other_key = (other.prop != BASE_ID, other.id or "")
+        return self_key < other_key
+
+    def clone(self: Self) -> "Statement":
+        """Make a deep copy of the given statement."""
+        return Statement.from_dict(self.to_dict())
+
+    def generate_key(self) -> Optional[str]:
+        return self.make_key(
+            self.dataset,
+            self.entity_id,
+            self.prop,
+            self.value,
+            self.external,
+        )
+
+    @classmethod
+    def make_key(
+        cls,
+        dataset: str,
+        entity_id: str,
+        prop: str,
+        value: str,
+        external: Optional[bool],
+    ) -> Optional[str]:
+        """Hash the key properties of a statement record to make a unique ID."""
+        if prop is None or value is None:
+            return None
+        key = f"{dataset}.{entity_id}.{prop}.{value}"
+        if external:
+            # We consider the external flag in key composition to avoid race conditions
+            # where a certain entity might be emitted as external while it is already
+            # linked in to the graph via another route.
+            key = f"{key}.ext"
+        return hashlib.sha1(key.encode("utf-8")).hexdigest()
+
+    @classmethod
+    def from_dict(cls, data: StatementDict) -> "Statement":
+        return cls(
+            entity_id=data["entity_id"],
+            prop=data["prop"],
+            schema=data["schema"],
+            value=data["value"],
+            dataset=data["dataset"],
+            lang=data.get("lang", None),
+            original_value=data.get("original_value", None),
+            first_seen=data.get("first_seen", None),
+            external=data.get("external", False),
+            id=data.get("id", None),
+            canonical_id=data.get("canonical_id", None),
+            last_seen=data.get("last_seen", None),
+            origin=data.get("origin", None),
+        )
+
+    @classmethod
+    def from_db_row(cls, row: Row[Any]) -> "Statement":
+        return cls(
+            id=row.id,
+            canonical_id=row.canonical_id,
+            entity_id=row.entity_id,
+            prop=row.prop,
+            schema=row.schema,
+            value=row.value,
+            dataset=row.dataset,
+            lang=row.lang,
+            original_value=row.original_value,
+            first_seen=datetime_iso(row.first_seen),
+            external=row.external,
+            last_seen=datetime_iso(row.last_seen),
+            origin=row.origin,
+        )
+
+    @classmethod
+    def from_entity(
+        cls,
+        entity: "EntityProxy",
+        dataset: str,
+        first_seen: Optional[str] = None,
+        last_seen: Optional[str] = None,
+        external: bool = False,
+        origin: Optional[str] = None,
+    ) -> Generator["Statement", None, None]:
+        from followthemoney.statement.entity import StatementEntity
+
+        if entity.id is None:
+            raise ValueError("Cannot create statements for entity without ID!")
+
+        # If the entity is already a StatementEntity, we return its statements directly.
+        if isinstance(entity, StatementEntity):
+            yield from entity.statements
+            return
+
+        yield cls(
+            entity_id=entity.id,
+            prop=BASE_ID,
+            schema=entity.schema.name,
+            value=entity.id,
+            dataset=dataset,
+            external=external,
+            first_seen=first_seen,
+            last_seen=last_seen,
+            origin=origin,
+        )
+        for prop, value in entity.itervalues():
+            yield cls(
+                entity_id=entity.id,
+                prop=prop.name,
+                schema=entity.schema.name,
+                value=value,
+                dataset=dataset,
+                external=external,
+                first_seen=first_seen,
+                last_seen=last_seen,
+                origin=origin,
+            )

followthemoney/statement/util.py

@@ -0,0 +1,31 @@
+import sys
+from functools import cache
+from typing import Tuple
+
+from followthemoney.model import Model
+
+BASE_ID = "id"
+
+
+def pack_prop(schema: str, prop: str) -> str:
+    return f"{schema}:{prop}"
+
+
+@cache
+def get_prop_type(schema: str, prop: str) -> str:
+    if prop == BASE_ID:
+        return BASE_ID
+    schema_obj = Model.instance().get(schema)
+    if schema_obj is None:
+        raise TypeError("Schema not found: %s" % schema)
+    prop_obj = schema_obj.get(prop)
+    if prop_obj is None:
+        raise TypeError("Property not found: %s" % prop)
+    return prop_obj.type.name
+
+
+@cache
+def unpack_prop(id: str) -> Tuple[str, str, str]:
+    schema, prop = id.split(":", 1)
+    prop_type = get_prop_type(schema, prop)
+    return sys.intern(schema), prop_type, sys.intern(prop)

followthemoney/types/__init__.py

@@ -1,4 +1,6 @@
-from followthemoney.types.registry import Registry
+from banal import ensure_list
+from typing import Dict, Iterable, List, Set, cast
+
 from followthemoney.types.url import UrlType
 from followthemoney.types.name import NameType
 from followthemoney.types.email import EmailType
@@ -11,7 +13,6 @@ from followthemoney.types.language import LanguageType
 from followthemoney.types.mimetype import MimeType
 from followthemoney.types.checksum import ChecksumType
 from followthemoney.types.identifier import IdentifierType
-from followthemoney.types.iban import IbanType
 from followthemoney.types.entity import EntityType
 from followthemoney.types.topic import TopicType
 from followthemoney.types.gender import GenderType
@@ -22,27 +23,69 @@ from followthemoney.types.string import StringType
 from followthemoney.types.number import NumberType
 from followthemoney.types.common import PropertyType
 
+
+class Registry(object):
+    """This registry keeps the processing helpers for all property types in the system. The
+    registry can be used to get a type, which can itself then clean, validate or format values
+    of that type."""
+
+    url = UrlType()
+    name = NameType()
+    email = EmailType()
+    ip = IpType()
+    address = AddressType()
+    date = DateType()
+    phone = PhoneType()
+    country = CountryType()
+    language = LanguageType()
+    mimetype = MimeType()
+    checksum = ChecksumType()
+    identifier = IdentifierType()
+    entity = EntityType()
+    topic = TopicType()
+    gender = GenderType()
+    json = JsonType()
+    text = TextType()
+    html = HTMLType()
+    string = StringType()
+    number = NumberType()
+
+    def __init__(self) -> None:
+        self.matchable: Set[PropertyType] = set()
+        self.types: Set[PropertyType] = set()
+        self.groups: Dict[str, PropertyType] = {}
+        self.pivots: Set[PropertyType] = set()
+        for name in dir(self):
+            type_ = getattr(self, name)
+            if not isinstance(type_, PropertyType):
+                continue
+            assert type_.name == name
+            self.types.add(type_)
+            if type_.matchable:
+                self.matchable.add(type_)
+            if type_.pivot:
+                self.pivots.add(type_)
+            if type_.group is not None:
+                self.groups[type_.group] = type_
+
+    def get(self, name: str) -> PropertyType:
+        """For a given property type name, get its type object. This can also
+        be used via getattr, e.g. ``registry.phone``."""
+        # Allow transparent re-checking.
+        if isinstance(name, PropertyType):
+            return name
+        return cast(PropertyType, getattr(self, name))
+
+    def get_types(self, names: Iterable[str]) -> List[PropertyType]:
+        """Get a list of all property type objects linked to a set of names."""
+        names = ensure_list(names)
+        types = [self.get(n) for n in names]
+        return [t for t in types if t is not None]
+
+    def __getitem__(self, name: str) -> PropertyType:
+        return cast(PropertyType, getattr(self, name))
+
+
 registry = Registry()
-registry.add(UrlType)
-registry.add(NameType)
-registry.add(EmailType)
-registry.add(IpType)
-registry.add(AddressType)
-registry.add(DateType)
-registry.add(PhoneType)
-registry.add(CountryType)
-registry.add(LanguageType)
-registry.add(MimeType)
-registry.add(ChecksumType)
-registry.add(IdentifierType)
-registry.add(IbanType)  # TODO: remove
-registry.add(EntityType)
-registry.add(TopicType)
-registry.add(GenderType)
-registry.add(JsonType)
-registry.add(TextType)
-registry.add(HTMLType)
-registry.add(StringType)
-registry.add(NumberType)
 
 __all__ = ["PropertyType", "registry"]

followthemoney/types/address.py

@@ -7,7 +7,7 @@ from rigour.text.distance import levenshtein_similarity
 
 from followthemoney.types.common import PropertyType
 from followthemoney.util import defer as _
-from followthemoney.util import dampen
+from followthemoney.util import dampen, const
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -21,8 +21,8 @@ class AddressType(PropertyType):
 
     LINE_BREAKS = re.compile(r"(\r\n|\n|<BR/>|<BR>|\t|ESQ\.,|ESQ,|;)")
     COMMATA = re.compile(r"(,\s?[,\.])")
-    name = "address"
-    group = "addresses"
+    name = const("address")
+    group = const("addresses")
     label = _("Address")
     plural = _("Addresses")
     matchable = True

followthemoney/types/checksum.py

@@ -1,6 +1,5 @@
-from followthemoney.rdf import URIRef, Identifier
 from followthemoney.types.common import PropertyType
-from followthemoney.util import defer as _
+from followthemoney.util import const, defer as _
 
 
 class ChecksumType(PropertyType):
@@ -13,13 +12,10 @@ class ChecksumType(PropertyType):
     of this type are scrubbed when submitted via the normal API. Checksums can only
     be defined by uploading a document to be ingested."""
 
-    name = "checksum"
-    group = "checksums"
+    name = const("checksum")
+    group = const("checksums")
     label = _("Checksum")
     plural = _("Checksums")
     matchable = True
     pivot = True
     max_length = 40
-
-    def rdf(self, value: str) -> Identifier:
-        return URIRef(f"hash:{value}")

followthemoney/types/common.py

@@ -5,8 +5,8 @@ from banal import ensure_list
 from normality import stringify
 from typing import Any, Dict, Optional, Sequence, Callable, TYPE_CHECKING, TypedDict
 
-from followthemoney.rdf import Literal, Identifier
-from followthemoney.util import get_locale
+from followthemoney.value import Value
+from followthemoney.util import get_locale, const
 from followthemoney.util import gettext, sanitize_text
 
 if TYPE_CHECKING:
@@ -29,7 +29,7 @@ class PropertyTypeToDict(TypedDict, total=False):
 class PropertyType(object):
     """Base class for all property types."""
 
-    name: str = "any"
+    name: str = const("any")
     """A machine-facing, variable safe name for the given type."""
 
     group: Optional[str] = None
@@ -87,7 +87,7 @@ class PropertyType(object):
 
     def clean(
         self,
-        raw: Any,
+        raw: Value,
         fuzzy: bool = False,
         format: Optional[str] = None,
         proxy: Optional["EntityProxy"] = None,
@@ -165,11 +165,6 @@ class PropertyType(object):
         be related to (e.g. using a country prefix on a phone number or IBAN)."""
         return None
 
-    def rdf(self, value: str) -> Identifier:
-        """Return an RDF term to represent the given value - either a string
-        literal, or a URI reference."""
-        return Literal(value)
-
     def pick(self, values: Sequence[str]) -> Optional[str]:
         """Pick the best value to show to the user."""
         raise NotImplementedError
@@ -178,7 +173,7 @@ class PropertyType(object):
         """Return an ID suitable to identify this entity as a typed node in a
         graph representation of some FtM data. It's usually the same as the the
         RDF form."""
-        return str(self.rdf(value))
+        return f"{self.name}:{value}"
 
     def node_id_safe(self, value: Optional[str]) -> Optional[str]:
         """Wrapper for node_id to handle None values."""
@@ -186,7 +181,7 @@ class PropertyType(object):
             return None
         return self.node_id(value)
 
-    def caption(self, value: str) -> Optional[str]:
+    def caption(self, value: str, format: Optional[str] = None) -> str:
         """Return a label for the given property value. This is often the same as the
         value, but for types like countries or languages, it would return the label,
         while other values like phone numbers can be formatted to be nicer to read."""
@@ -253,19 +248,19 @@ class EnumType(PropertyType):
 
     def clean_text(
         self,
-        code: str,
+        text: str,
         fuzzy: bool = False,
         format: Optional[str] = None,
         proxy: Optional["EntityProxy"] = None,
     ) -> Optional[str]:
         """All code values are cleaned to be lowercase and trailing whitespace is
         removed."""
-        code = code.lower().strip()
+        code = text.lower().strip()
         if code not in self.codes:
             return None
         return code
 
-    def caption(self, value: str) -> str:
+    def caption(self, value: str, format: Optional[str] = None) -> str:
         """Given a code value, return the label that should be shown to a user."""
         return self.names.get(value, value)
 

followthemoney/types/country.py

@@ -3,9 +3,8 @@ from typing import Optional, TYPE_CHECKING
 from babel.core import Locale
 from rigour.territories import get_territory, get_ftm_countries
 
-from followthemoney.rdf import URIRef, Identifier
 from followthemoney.types.common import EnumType, EnumValues
-from followthemoney.util import defer as _
+from followthemoney.util import const, defer as _
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -17,8 +16,8 @@ class CountryType(EnumType):
     a number of unusual and controversial designations (e.g. the Soviet Union,
     Transnistria, Somaliland, Kosovo)."""
 
-    name = "country"
-    group = "countries"
+    name = const("country")
+    group = const("countries")
     label = _("Country")
     plural = _("Countries")
     matchable = True
@@ -52,6 +51,3 @@ class CountryType(EnumType):
 
     def country_hint(self, value: str) -> str:
         return value
-
-    def rdf(self, value: str) -> Identifier:
-        return URIRef(f"iso-3166:{value}")

followthemoney/types/date.py

@@ -4,9 +4,8 @@ from typing import Optional, TYPE_CHECKING
 from prefixdate import parse, parse_format, Precision
 
 from followthemoney.types.common import PropertyType
-from followthemoney.rdf import XSD, Literal, Identifier
 from followthemoney.util import defer as _
-from followthemoney.util import dampen
+from followthemoney.util import dampen, const
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -21,8 +20,8 @@ class DateType(PropertyType):
     The timezone is always expected to be UTC and cannot be specified otherwise. There is
     no support for calendar weeks (`2021-W7`) and date ranges (`2021-2024`)."""
 
-    name = "date"
-    group = "dates"
+    name = const("date")
+    group = const("dates")
     label = _("Date")
     plural = _("Dates")
     matchable = True
@@ -57,18 +56,29 @@ class DateType(PropertyType):
         prefix = os.path.commonprefix([left, right])
         return dampen(4, 10, prefix)
 
-    def rdf(self, value: str) -> Identifier:
-        if len(value) < Precision.HOUR.value:
-            return Literal(value, datatype=XSD.date)
-        return Literal(value, datatype=XSD.dateTime)
+    def to_datetime(self, value: str) -> Optional[datetime]:
+        """Convert a date string to a datetime object in UTC for handling in Python. This
+        will convert the unset fields beyond the prefix to the first possible value, e.g.
+        `2021-02` will become `2021-02-01T00:00:00Z`.
 
-    def node_id(self, value: str) -> str:
-        return f"date:{value}"
+        Args:
+            value (str): The date string to convert.
 
-    def to_datetime(self, value: str) -> Optional[datetime]:
+        Returns:
+            Optional[datetime]: The parsed datetime object in UTC, or None if parsing fails.
+        """
         return parse(value).dt
 
     def to_number(self, value: str) -> Optional[float]:
+        """Convert a date string to a number, which is the number of seconds since the epoch
+        (1970-01-01T00:00:00Z).
+
+        Args:
+            value (str): The date string to convert.
+
+        Returns:
+            Optional[float]: The timestamp as a float, or None if parsing fails.
+        """
         date = self.to_datetime(value)
         if date is None:
             return None

followthemoney/types/email.py

@@ -4,7 +4,6 @@ from typing import Optional, TYPE_CHECKING
 from urllib.parse import urlparse
 from normality.cleaning import strip_quotes
 
-from followthemoney.rdf import URIRef, Identifier
 from followthemoney.types.common import PropertyType
 from followthemoney.util import sanitize_text, defer as _
 
@@ -80,6 +79,3 @@ class EmailType(PropertyType):
 
     # def country_hint(self, value)
     # TODO: do we want to use TLDs as country evidence?
-
-    def rdf(self, value: str) -> Identifier:
-        return URIRef("mailto:%s" % value.lower())

followthemoney/types/entity.py

@@ -2,9 +2,9 @@ import re
 from typing import Any, Optional, TYPE_CHECKING
 
 from followthemoney.types.common import PropertyType
-from followthemoney.rdf import URIRef, Identifier
+from followthemoney.value import Value
 from followthemoney.util import ENTITY_ID_LEN, get_entity_id, sanitize_text
-from followthemoney.util import gettext, defer as _
+from followthemoney.util import const, gettext, defer as _
 from followthemoney.exc import InvalidData
 
 if TYPE_CHECKING:
@@ -22,8 +22,8 @@ class EntityType(PropertyType):
 
     REGEX_RAW = r"^[0-9a-zA-Z]([0-9a-zA-Z\.\-]*[0-9a-zA-Z])?$"
     REGEX = re.compile(REGEX_RAW)
-    name = "entity"
-    group = "entities"
+    name = const("entity")
+    group = const("entities")
     label = _("Entity")
     plural = _("Entities")
     matchable = True
@@ -31,7 +31,7 @@ class EntityType(PropertyType):
     max_length = ENTITY_ID_LEN
 
     def validate(
-        self, value: str, fuzzy: bool = False, format: Optional[str] = None
+        self, value: Value, fuzzy: bool = False, format: Optional[str] = None
     ) -> bool:
         text = sanitize_text(value)
         if text is None:
@@ -66,9 +66,3 @@ class EntityType(PropertyType):
         if self.REGEX.match(text) is not None:
             return text
         return None
-
-    def rdf(self, value: str) -> Identifier:
-        return URIRef(f"entity:{value}")
-
-    def caption(self, value: str) -> None:
-        return None