followthemoney 4.3.4__py3-none-any.whl → 4.5.0__py3-none-any.whl

followthemoney/__init__.py

@@ -9,7 +9,7 @@ from followthemoney.statement import Statement, StatementEntity, SE
 from followthemoney.dataset import Dataset, DefaultDataset, DS
 from followthemoney.util import set_model_locale
 
-__version__ = "4.3.4"
+__version__ = "4.5.0"
 
 # Data model singleton
 model = Model.instance()

followthemoney/compare.py

@@ -71,31 +71,18 @@ def _compare(scores: Scores, weights: Weights, n_std: int = 1) -> float:
     return 1.0 / (1.0 + math.exp(-prob))
 
 
-def entity_is_same(left: EntityProxy, right: EntityProxy) -> bool:
-    """Check if two entities are the same apart from their ID."""
-    if left.schema != right.schema:
-        return False
-
-    props = set(left.properties.keys()).union(right.properties.keys())
-    if 0 == len(props):
-        return False
-
-    for prop in props:
-        left_vals = sorted(left.get(prop))
-        right_vals = sorted(right.get(prop))
-        if left_vals != right_vals:
-            return False
-    return True
-
-
 def compare(
     left: EntityProxy,
     right: EntityProxy,
     weights: Weights = COMPARE_WEIGHTS,
 ) -> float:
     """Compare two entities and return a match score."""
-    if entity_is_same(left, right):
-        return 1.0
+    if left.checksum == right.checksum:
+        # Check if there is any data at all (ie any basis for making a decision),
+        # if so, return a perfect match. This avoids marking two empty entities
+        # as matching. Bit ambiguous, but practical.
+        if len(left.properties) > 0 and len(right.properties) > 0:
+            return 1.0
     scores = compare_scores(left, right)
     return _compare(scores, weights)
 

followthemoney/dataset/dataset.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 import yaml
 import logging
 from functools import cached_property
@@ -38,6 +39,8 @@ class DatasetModel(BaseModel):
     coverage: DataCoverage | None = None
     resources: List[DataResource] = []
     children: Set[str] = set()
+    deprecation: Optional[str] = None
+    deprecated: bool = False
 
     @field_validator("name", mode="after")
     @classmethod
@@ -57,6 +60,18 @@ class DatasetModel(BaseModel):
             data["children"] = children
         return data
 
+    @model_validator(mode="after")
+    def evaluate_data(self) -> "DatasetModel":
+        # derive deprecated from deprecation notice:
+        if self.deprecation is not None:
+            self.deprecation = self.deprecation.strip()
+            if not len(self.deprecation):
+                self.deprecation = None
+        self.deprecated = self.deprecation is not None or self.deprecated
+        if self.deprecated and (self.coverage is None or self.coverage.end is None):
+            raise ValueError("Deprecated dataset coverage must have an end date.")
+        return self
+
     def get_resource(self, name: str) -> DataResource:
         for res in self.resources:
             if res.name == name:
@@ -121,10 +136,13 @@ class Dataset:
     ) -> DS:
         from followthemoney.dataset.catalog import DataCatalog
 
+        path = Path(path)
         with open(path, "r") as fh:
             data = yaml.safe_load(fh)
             if catalog is None:
                 catalog = DataCatalog(cls, {})
+            if "name" not in data:
+                data["name"] = path.stem
             return catalog.make_dataset(data)
 
     @classmethod

followthemoney/entity.py

@@ -5,6 +5,7 @@ from rigour.names import pick_name
 from followthemoney.proxy import EntityProxy
 from followthemoney.schema import Schema
 from followthemoney.statement import BASE_ID, Statement
+from followthemoney.util import HASH_ENCODING
 
 VE = TypeVar("VE", bound="ValueEntity")
 
@@ -81,6 +82,19 @@ class ValueEntity(EntityProxy):
             merged.last_change = max(changed, default=None)
         return merged
 
+    @property
+    def checksum(self) -> str:
+        digest = self._checksum_digest()
+        for dataset in sorted(self.datasets):
+            digest.update(dataset.encode(HASH_ENCODING))
+            digest.update(b"\x1e")
+        for referent in sorted(self.referents):
+            digest.update(referent.encode(HASH_ENCODING))
+            digest.update(b"\x1e")
+        if self.last_change is not None:
+            digest.update(self.last_change.encode(HASH_ENCODING))
+        return digest.hexdigest()
+
     def to_dict(self) -> Dict[str, Any]:
         data = super().to_dict()
         data["referents"] = list(self.referents)

followthemoney/mapping/csv.py

@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, cast
 from typing import Any, Dict, Generator, ItemsView, Iterable, List, Optional, Set, Tuple
 
 from followthemoney.mapping.source import Record, Source
+from followthemoney.settings import USER_AGENT
 from followthemoney.util import sanitize_text
 from followthemoney.exc import InvalidMapping
 
@@ -64,7 +65,8 @@ class CSVSource(Source):
         parsed_url = urlparse(url)
         log.info("Loading: %s", url)
         if parsed_url.scheme in ["http", "https"]:
-            res = requests.get(url, stream=True)
+            headers = {"User-Agent": USER_AGENT}
+            res = requests.get(url, stream=True, headers=headers)
             if not res.ok:
                 raise InvalidMapping("Failed to open CSV: %s" % url)
             # if res.encoding is None:

followthemoney/model.py

@@ -3,12 +3,14 @@ import yaml
 from functools import cache
 from typing import TYPE_CHECKING, Any
 from typing import Dict, Generator, Iterator, Optional, Set, TypedDict, Union
+from rigour.env import ENCODING
 
 from followthemoney.types import registry
 from followthemoney.types.common import PropertyType, PropertyTypeToDict
 from followthemoney.schema import Schema, SchemaToDict
 from followthemoney.property import Property
 from followthemoney.exc import InvalidModel, InvalidData
+from followthemoney.settings import MODEL_PATH
 from followthemoney.util import const
 
 if TYPE_CHECKING:
@@ -47,10 +49,7 @@ class Model(object):
     @classmethod
     def instance(cls) -> "Model":
         if cls._instance is None:
-            model_path = os.path.dirname(__file__)
-            model_path = os.path.join(model_path, "schema")
-            model_path = os.environ.get("FTM_MODEL_PATH", model_path)
-            cls._instance = cls(model_path)
+            cls._instance = cls(MODEL_PATH)
         return cls._instance
 
     def generate(self) -> None:
@@ -68,7 +67,7 @@ class Model(object):
                     schema.properties[prop.name] = prop
 
     def _load(self, filepath: str) -> None:
-        with open(filepath, "r", encoding="utf-8") as fh:
+        with open(filepath, "r", encoding=ENCODING) as fh:
             data = yaml.safe_load(fh)
             if not isinstance(data, dict):
                 raise InvalidModel("Model file is not a mapping: %s" % filepath)

followthemoney/proxy.py

@@ -1,3 +1,4 @@
+import hashlib
 import logging
 from typing import TYPE_CHECKING, cast, Any
 from typing import Dict, Generator, List, Optional, Set, Tuple, Union, Type, TypeVar
@@ -10,13 +11,14 @@ from followthemoney.types import registry
 from followthemoney.types.common import PropertyType
 from followthemoney.property import Property
 from followthemoney.value import string_list, Values
-from followthemoney.util import sanitize_text, gettext
+from followthemoney.util import HASH_ENCODING, sanitize_text, gettext
 from followthemoney.util import merge_context, make_entity_id
 from followthemoney.model import Model
 from followthemoney.schema import Schema
 
 if TYPE_CHECKING:
     from followthemoney.model import Model
+    from hashlib import _Hash
 
 log = logging.getLogger(__name__)
 P = Union[Property, str]
@@ -437,6 +439,28 @@ class EntityProxy(object):
             self.add(prop, values, cleaned=True, quiet=True)
         return self
 
+    def _checksum_digest(self) -> "_Hash":
+        """Create a SHA1 digest of the entity's ID, schema and properties for
+        change detection. This is returned as a hashlib digest object so that
+        it can be subclassed."""
+        digest = hashlib.sha1()
+        if self.id is not None:
+            digest.update(self.id.encode(HASH_ENCODING))
+        digest.update(self.schema.name.encode(HASH_ENCODING))
+        for prop in sorted(self._properties.keys()):
+            digest.update(prop.encode(HASH_ENCODING))
+            for value in sorted(self._properties[prop]):
+                digest.update(value.encode(HASH_ENCODING))
+                digest.update(b"\x1e")
+            digest.update(b"\x1f")
+        return digest
+
+    @property
+    def checksum(self) -> str:
+        """A SHA1 checksum hexdigest representing the current state of the
+        entity proxy. This can be used for change detection."""
+        return self._checksum_digest().hexdigest()
+
     def __getstate__(self) -> Dict[str, Any]:
         data = {slot: getattr(self, slot) for slot in self.__slots__}
         data["schema"] = self.schema.name
@@ -460,13 +484,13 @@ class EntityProxy(object):
 
     def __hash__(self) -> int:
         if self.id is None:
-            raise RuntimeError("Cannot hash entity without an ID")
+            raise RuntimeError("Unhashable entity proxy without ID.")
         return hash(self.id)
 
     def __eq__(self, other: Any) -> bool:
         try:
             if self.id is None or other.id is None:
-                raise RuntimeError("Cannot compare entities without IDs.")
+                raise RuntimeError("Cannot compare entity proxies without IDs.")
             return bool(self.id == other.id)
         except AttributeError:
             return False

followthemoney/schema/Company.yaml

@@ -19,6 +19,7 @@ Company:
   caption:
     - name
     - alias
+    - abbreviation
     - weakAlias
     - previousName
     - registrationNumber

followthemoney/schema/CryptoWallet.yaml

@@ -26,6 +26,10 @@ CryptoWallet:
       maxLength: 128
     privateKey:
       label: Private key
+    accountId:
+      label: Account ID
+      description: Platform-specific user/account identifier
+      type: identifier
     creationDate:
       label: Creation date
       type: date

followthemoney/schema/Image.yaml

@@ -1,4 +1,7 @@
 Image:
+  # This schema defines an image file entity within the FollowTheMoney data model.
+  # If a `checksum` property is present, consider loading it from an Aleph archive
+  # or FtM data lake. Otherwise, use `sourceUrl` to fetch the image directly.
   extends:
     - Document
   label: Image
@@ -23,3 +26,7 @@ Image:
         label: "Images"
       type: entity
       range: Person
+    credit:
+      label: "Credit"
+      description: "The credit or attribution for the image."
+      type: string

followthemoney/schema/LegalEntity.yaml

@@ -18,6 +18,7 @@ LegalEntity:
   caption:
     - name
     - alias
+    - abbreviation
     - weakAlias
     - previousName
     - email
@@ -29,6 +30,12 @@ LegalEntity:
     end:
       - dissolutionDate
   properties:
+    abbreviation:
+      label: Abbreviation
+      type: name
+      description: "Abbreviated name or acronym"
+      # TODO: is un-matchable wise? The idea is to handle it like `weakAlias` rather than `alias`.
+      matchable: false
     email:
       label: E-Mail
       type: email

followthemoney/schema/Organization.yaml

@@ -18,6 +18,7 @@ Organization:
   caption:
     - name
     - alias
+    - abbreviation
     - weakAlias
     - previousName
     - registrationNumber

followthemoney/schema/Person.yaml

@@ -15,8 +15,9 @@ Person:
   caption:
     - name
     - alias
-    - weakAlias
     - previousName
+    - weakAlias
+    - abbreviation
     - lastName
     - email
     - phone

followthemoney/schema/PublicBody.yaml

@@ -14,6 +14,7 @@ PublicBody:
   caption:
     - name
     - alias
+    - abbreviation
     - weakAlias
     - previousName
   required:

followthemoney/settings.py

@@ -0,0 +1,19 @@
+import os
+import requests
+from typing import List
+from rigour.env import env_opt, env_str
+
+
+def get_env_list(name: str, default: List[str] = []) -> List[str]:
+    value = env_opt(name)
+    if value is not None:
+        values = value.split(":")
+        if len(values):
+            return values
+    return default
+
+
+MODEL_PATH = os.path.join(os.path.dirname(__file__), "schema")
+MODEL_PATH = env_str("FTM_MODEL_PATH", MODEL_PATH)
+
+USER_AGENT = env_str("FTM_USER_AGENT", requests.utils.default_user_agent())

followthemoney/statement/entity.py

@@ -1,6 +1,6 @@
 from hashlib import sha1
 from collections.abc import Mapping
-from typing import Any, Dict, List, Optional, Set, Type
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Type
 from typing import Generator, Iterable, Tuple, TypeVar
 from rigour.langs import LangStr
 from rigour.names.pick import pick_lang_name
@@ -10,7 +10,7 @@ from followthemoney.exc import InvalidData
 from followthemoney.schema import Schema
 from followthemoney.types.common import PropertyType
 from followthemoney.property import Property
-from followthemoney.util import gettext
+from followthemoney.util import HASH_ENCODING, gettext
 from followthemoney.proxy import P
 from followthemoney.types import registry
 from followthemoney.value import string_list, Values
@@ -21,6 +21,9 @@ from followthemoney.statement.util import BASE_ID
 
 SE = TypeVar("SE", bound="StatementEntity")
 
+if TYPE_CHECKING:
+    from hashlib import _Hash
+
 
 class StatementEntity(EntityProxy):
     """An entity object that can link to a set of datasets that it is sourced from."""
@@ -35,7 +38,12 @@ class StatementEntity(EntityProxy):
         "_statements",
     )
 
-    def __init__(self, dataset: Dataset, data: Dict[str, Any], cleaned: bool = True):
+    def __init__(
+        self,
+        dataset: Dataset,
+        data: Dict[str, Any],
+        cleaned: bool = True,
+    ) -> None:
         data = dict(data or {})
         schema = Model.instance().get(data.pop("schema", None))
         if schema is None:
@@ -76,8 +84,7 @@ class StatementEntity(EntityProxy):
         for stmts in self._statements.values():
             for stmt in stmts:
                 if stmt.entity_id is None and self.id is not None:
-                    stmt.entity_id = self.id
-                    stmt.id = stmt.generate_key()
+                    stmt = stmt.clone(entity_id=self.id)
                 if stmt.id is None:
                     stmt.id = stmt.generate_key()
                 yield stmt
@@ -97,9 +104,9 @@ class StatementEntity(EntityProxy):
             if stmt.first_seen is not None:
                 first_seen.add(stmt.first_seen)
         if self.id is not None:
-            digest = sha1(self.schema.name.encode("utf-8"))
+            digest = sha1(self.schema.name.encode(HASH_ENCODING))
             for id in sorted(ids):
-                digest.update(id.encode("utf-8"))
+                digest.update(id.encode(HASH_ENCODING))
             checksum = digest.hexdigest()
             # This is to make the last_change value stable across
             # serialisation:
@@ -449,6 +456,23 @@ class StatementEntity(EntityProxy):
         data["statements"] = [stmt.to_dict() for stmt in self.statements]
         return data
 
+    def _checksum_digest(self) -> "_Hash":
+        """Create a SHA1 digest of the entity's ID, schema and properties for
+        change detection. This is returned as a hashlib digest object so that
+        it can be subclassed."""
+        digest = sha1()
+        if self.id is not None:
+            digest.update(self.id.encode(HASH_ENCODING))
+        statement_ids: List[str] = []
+        for stmts in self._statements.values():
+            for stmt in stmts:
+                if stmt.id is not None:
+                    statement_ids.append(stmt.id)
+        for stmt_id in sorted(statement_ids):
+            digest.update(stmt_id.encode(HASH_ENCODING))
+            digest.update(b"\x1e")
+        return digest
+
     def __len__(self) -> int:
         return len(list(self._iter_stmt())) + 1
 

followthemoney/statement/serialize.py

@@ -5,9 +5,10 @@ import logging
 from io import TextIOWrapper
 from pathlib import Path
 from types import TracebackType
-from typing import cast
+from typing import Dict, Tuple, cast
 from typing import BinaryIO, Generator, Iterable, List, Optional, TextIO, Type
 from rigour.boolean import text_bool
+from rigour.env import ENCODING
 
 from followthemoney.statement.statement import Statement, StatementDict
 from followthemoney.statement.util import unpack_prop
@@ -60,7 +61,7 @@ def read_json_statements(
 
 
 def read_csv_statements(fh: BinaryIO) -> Generator[Statement, None, None]:
-    wrapped = TextIOWrapper(fh, encoding="utf-8")
+    wrapped = TextIOWrapper(fh, encoding=ENCODING)
     for row in csv.DictReader(wrapped, dialect=csv.unix_dialect):
         data = cast(StatementDict, row)
         data["external"] = text_bool(row.get("external")) or False
@@ -72,7 +73,7 @@ def read_csv_statements(fh: BinaryIO) -> Generator[Statement, None, None]:
 
 
 def read_pack_statements(fh: BinaryIO) -> Generator[Statement, None, None]:
-    wrapped = TextIOWrapper(fh, encoding="utf-8")
+    wrapped = TextIOWrapper(fh, encoding=ENCODING)
     yield from read_pack_statements_decoded(wrapped)
 
 
@@ -129,10 +130,10 @@ def read_path_statements(path: Path, format: str) -> Generator[Statement, None,
 
 def get_statement_writer(fh: BinaryIO, format: str) -> "StatementWriter":
     if format == CSV:
-        wrapped = TextIOWrapper(fh, encoding="utf-8")
+        wrapped = TextIOWrapper(fh, encoding=ENCODING)
         return CSVStatementWriter(wrapped)
     elif format == PACK:
-        wrapped = TextIOWrapper(fh, encoding="utf-8")
+        wrapped = TextIOWrapper(fh, encoding=ENCODING)
         return PackStatementWriter(wrapped)
     elif format == JSON:
         return JSONStatementWriter(fh)
@@ -222,12 +223,14 @@ class PackStatementWriter(StatementWriter):
             "id",
         ]
         self.writer.writerow(columns)
-        self._batch: List[List[Optional[str]]] = []
+        self._batch: Dict[str, Tuple[Optional[str], ...]] = {}
 
     def write(self, stmt: Statement) -> None:
         # HACK: This is very similar to the CSV writer, but at the very inner
         # loop of the application, so we're duplicating code here.
-        row = [
+        if stmt.id is None:
+            raise RuntimeError("Cannot write pack statement without ID")
+        row = (
             stmt.entity_id,
             f"{stmt.schema}:{stmt.prop}",
             stmt.value,
@@ -239,13 +242,15 @@ class PackStatementWriter(StatementWriter):
             stmt.first_seen,
             stmt.last_seen,
             stmt.id,
-        ]
-        self._batch.append(row)
+        )
+        self._batch[stmt.id] = row
         if len(self._batch) >= CSV_BATCH:
-            self.writer.writerows(self._batch)
-            self._batch.clear()
+            self.flush()
+
+    def flush(self) -> None:
+        self.writer.writerows(self._batch.values())
+        self._batch.clear()
 
     def close(self) -> None:
-        if len(self._batch) > 0:
-            self.writer.writerows(self._batch)
+        self.flush()
         self.fh.close()

followthemoney/statement/statement.py

@@ -1,14 +1,22 @@
 import hashlib
 import warnings
 from sqlalchemy.engine import Row
-from typing import cast
-from typing import Any, Dict, Generator, Optional
+from typing import Union, cast
+from typing import Any, Dict, Generator, Optional, TypeGuard
 from typing_extensions import TypedDict, Self
 from rigour.time import datetime_iso, iso_datetime
 from rigour.boolean import bool_text
 
 from followthemoney.proxy import EntityProxy
-from followthemoney.statement.util import get_prop_type, BASE_ID
+from followthemoney.statement.util import get_prop_type, BASE_ID, NON_LANG_TYPE_NAMES
+from followthemoney.util import HASH_ENCODING
+
+
+UNSET = object()
+
+
+def is_not_unset(value: str | None | object) -> TypeGuard[str | None]:
+    return value is not UNSET
 
 
 class StatementDict(TypedDict):
@@ -42,15 +50,16 @@ class Statement(object):
 
     __slots__ = [
         "id",
-        "entity_id",
+        "_entity_id",
         "canonical_id",
-        "prop",
-        "schema",
-        "value",
-        "dataset",
-        "lang",
+        "_prop",
+        "_schema",
+        "_value",
+        "_dataset",
+        "_lang",
+        "prop_type",
         "original_value",
-        "external",
+        "_external",
         "first_seen",
         "last_seen",
         "origin",
@@ -72,55 +81,95 @@ class Statement(object):
         last_seen: Optional[str] = None,
         origin: Optional[str] = None,
     ):
-        self.entity_id = entity_id
+        self._entity_id = entity_id
         self.canonical_id = canonical_id or entity_id
-        self.prop = prop
-        self.schema = schema
-        self.value = value
-        self.dataset = dataset
-        self.lang = lang
+        self._prop = prop
+        self._schema = schema
+        self.prop_type = get_prop_type(schema, prop)
+        self._value = value
+        self._dataset = dataset
+
+        # Remove lang for non-linguistic property types. The goal here is to avoid
+        # duplicate statements because of language tags, but the language metadata
+        # may be relevant as context for how the original_value was parsed so it's
+        # a bit of information loss.
+        if lang is not None:
+            if self.prop_type in NON_LANG_TYPE_NAMES:
+                lang = None
+        self._lang = lang
+
         self.original_value = original_value
         self.first_seen = first_seen
         self.last_seen = last_seen or first_seen
-        self.external = external
+        self._external = external
         self.origin = origin
         if id is None:
             id = self.generate_key()
         self.id = id
 
     @property
-    def prop_type(self) -> str:
-        """The type of the property, e.g. 'string', 'number', 'url'."""
-        return get_prop_type(self.schema, self.prop)
+    def entity_id(self) -> str:
+        """The (original) ID of the entity this statement is about."""
+        return self._entity_id
+
+    @property
+    def dataset(self) -> str:
+        """The dataset this statement was observed in."""
+        return self._dataset
+
+    @property
+    def prop(self) -> str:
+        """The property name this statement is about."""
+        return self._prop
+
+    @property
+    def schema(self) -> str:
+        """The schema of the entity this statement is about."""
+        return self._schema
+
+    @property
+    def value(self) -> str:
+        """The value of the property captured by this statement."""
+        return self._value
+
+    @property
+    def lang(self) -> Optional[str]:
+        """The language of the property value, if applicable."""
+        return self._lang
+
+    @property
+    def external(self) -> bool:
+        """Whether this statement was observed in an external dataset."""
+        return self._external
 
     def to_dict(self) -> StatementDict:
         return {
             "canonical_id": self.canonical_id,
-            "entity_id": self.entity_id,
-            "prop": self.prop,
-            "schema": self.schema,
-            "value": self.value,
-            "dataset": self.dataset,
-            "lang": self.lang,
+            "entity_id": self._entity_id,
+            "prop": self._prop,
+            "schema": self._schema,
+            "value": self._value,
+            "dataset": self._dataset,
+            "lang": self._lang,
             "original_value": self.original_value,
             "first_seen": self.first_seen,
             "last_seen": self.last_seen,
-            "external": self.external,
+            "external": self._external,
             "origin": self.origin,
             "id": self.id,
         }
 
     def to_csv_row(self) -> Dict[str, Optional[str]]:
         data = cast(Dict[str, Optional[str]], self.to_dict())
-        data["external"] = bool_text(self.external)
-        data["prop_type"] = get_prop_type(self.schema, self.prop)
+        data["external"] = bool_text(self._external)
+        data["prop_type"] = self.prop_type
         return data
 
     def to_db_row(self) -> Dict[str, Any]:
         data = cast(Dict[str, Any], self.to_dict())
         data["first_seen"] = iso_datetime(self.first_seen)
         data["last_seen"] = iso_datetime(self.last_seen)
-        data["prop_type"] = get_prop_type(self.schema, self.prop)
+        data["prop_type"] = self.prop_type
         return data
 
     def __hash__(self) -> int:
@@ -132,27 +181,83 @@ class Statement(object):
         return hash(self.id)
 
     def __repr__(self) -> str:
-        return "<Statement(%r, %r, %r)>" % (self.entity_id, self.prop, self.value)
+        return "<Statement(%r, %r, %r)>" % (self._entity_id, self._prop, self._value)
 
     def __eq__(self, other: Any) -> bool:
         return not self.id != other.id
 
     def __lt__(self, other: Any) -> bool:
-        self_key = (self.prop != BASE_ID, self.id or "")
-        other_key = (other.prop != BASE_ID, other.id or "")
+        self_key = (self._prop != BASE_ID, self.id or "")
+        other_key = (other._prop != BASE_ID, other.id or "")
         return self_key < other_key
 
-    def clone(self: Self) -> "Statement":
+    def clone(
+        self: Self,
+        *,
+        entity_id: Optional[str] = None,
+        prop: Optional[str] = None,
+        schema: Optional[str] = None,
+        value: Optional[str] = None,
+        dataset: Optional[str] = None,
+        lang: Union[str, None, object] = UNSET,
+        original_value: Union[str, None, object] = UNSET,
+        first_seen: Union[str, None, object] = UNSET,
+        external: Optional[bool] = None,
+        canonical_id: Optional[str] = None,
+        last_seen: Union[str, None, object] = UNSET,
+        origin: Union[str, None, object] = UNSET,
+    ) -> "Statement":
         """Make a deep copy of the given statement."""
-        return Statement.from_dict(self.to_dict())
+        lang = lang if is_not_unset(lang) else self._lang
+        ov = original_value if is_not_unset(original_value) else self.original_value
+        first_seen = first_seen if is_not_unset(first_seen) else self.first_seen
+        last_seen = last_seen if is_not_unset(last_seen) else self.last_seen
+        origin = origin if is_not_unset(origin) else self.origin
+        if external is None:
+            external = self._external
+        if canonical_id is None and self._entity_id != self.canonical_id:
+            canonical_id = self.canonical_id
+
+        # Decide if the statement ID can be kept the same:
+        stmt_id = self.id
+        if entity_id is not None and entity_id != self.entity_id:
+            stmt_id = None
+        if prop is not None and prop != self._prop:
+            stmt_id = None
+        if schema is not None and schema != self._schema:
+            stmt_id = None
+        if value is not None and value != self._value:
+            stmt_id = None
+        if dataset is not None and dataset != self._dataset:
+            stmt_id = None
+        if external != self._external:
+            stmt_id = None
+        if lang != self._lang:
+            stmt_id = None
+        return Statement(
+            id=stmt_id,
+            entity_id=entity_id or self._entity_id,
+            prop=prop or self._prop,
+            schema=schema or self._schema,
+            value=value or self._value,
+            dataset=dataset or self._dataset,
+            lang=lang,
+            original_value=ov,
+            first_seen=first_seen,
+            external=external,
+            canonical_id=canonical_id,
+            last_seen=last_seen,
+            origin=origin,
+        )
 
     def generate_key(self) -> Optional[str]:
         return self.make_key(
-            self.dataset,
-            self.entity_id,
-            self.prop,
-            self.value,
-            self.external,
+            self._dataset,
+            self._entity_id,
+            self._prop,
+            self._value,
+            self._external,
+            lang=self._lang,
         )
 
     @classmethod
@@ -163,17 +268,21 @@ class Statement(object):
         prop: str,
         value: str,
         external: Optional[bool],
+        lang: Optional[str] = None,
     ) -> Optional[str]:
         """Hash the key properties of a statement record to make a unique ID."""
         if prop is None or value is None:
             return None
-        key = f"{dataset}.{entity_id}.{prop}.{value}"
+        if lang is None:
+            key = f"{dataset}.{entity_id}.{prop}.{value}"
+        else:
+            key = f"{dataset}.{entity_id}.{prop}.{value}@{lang}"
         if external:
             # We consider the external flag in key composition to avoid race conditions
             # where a certain entity might be emitted as external while it is already
             # linked in to the graph via another route.
             key = f"{key}.ext"
-        return hashlib.sha1(key.encode("utf-8")).hexdigest()
+        return hashlib.sha1(key.encode(HASH_ENCODING)).hexdigest()
 
     @classmethod
     def from_dict(cls, data: StatementDict) -> "Statement":

followthemoney/statement/util.py

@@ -2,10 +2,31 @@ from functools import cache
 from typing import Tuple
 
 from followthemoney.model import Model
+from followthemoney.types import registry
 from followthemoney.util import const
 
 BASE_ID = "id"
 
+# Some property types should not set the `lang` attribute on statements.
+# These are typically non-linguistic types, although there's an argument
+# that language metadata could be useful for dates and countries, where
+# text parsing is likely to have taken place.
+NON_LANG_TYPE_NAMES = {
+    registry.entity.name,
+    registry.date.name,
+    registry.checksum.name,
+    registry.email.name,
+    registry.phone.name,
+    registry.gender.name,
+    registry.mimetype.name,
+    registry.topic.name,
+    registry.url.name,
+    registry.country.name,
+    registry.language.name,
+    registry.ip.name,
+    BASE_ID,
+}
+
 
 def pack_prop(schema: str, prop: str) -> str:
     return f"{schema}:{prop}"

followthemoney/types/country.py

@@ -1,6 +1,7 @@
-from typing import Optional, TYPE_CHECKING
+from typing import Callable, Optional, TYPE_CHECKING, Sequence
 from babel.core import Locale
 from rigour.territories import get_ftm_countries, lookup_territory
+from rigour.territories import territories_intersect
 
 from followthemoney.types.common import EnumType, EnumValues
 from followthemoney.util import defer as _
@@ -25,6 +26,20 @@ class CountryType(EnumType):
     def _locale_names(self, locale: Locale) -> EnumValues:
         return {t.code: t.name for t in get_ftm_countries()}
 
+    def compare(self, left: str, right: str) -> float:
+        overlap = territories_intersect([left], [right])
+        return 1.0 if len(overlap) else 0.0
+
+    def compare_sets(
+        self,
+        left: Sequence[str],
+        right: Sequence[str],
+        func: Callable[[Sequence[float]], float] = max,
+    ) -> float:
+        """Compare two sets of values and select the highest-scored result."""
+        overlap = territories_intersect(left, right)
+        return 1.0 if len(overlap) else 0.0
+
     def clean_text(
         self,
         text: str,

followthemoney/types/date.py

@@ -27,6 +27,16 @@ class DateType(PropertyType):
     matchable = True
     max_length = 32
 
+    HISTORIC = "1001-01-01"
+    """A sentinel date value representing a very old date, used to indicate historic (and often imprecise) dates
+    that can be assumed to be long in the past."""
+
+    RELEVANCE_MIN = "1900-01-01"
+    """A cutoff date value representing the minimum relevant date for modern fincrime applications."""
+
+    RELEVANCE_MAX = "2100-12-31"
+    """A cutoff date value representing the maximum relevant date for modern fincrime applications."""
+
     def validate(
         self, value: str, fuzzy: bool = False, format: Optional[str] = None
     ) -> bool:

followthemoney/types/language.py

@@ -4,7 +4,7 @@ from rigour.langs import iso_639_alpha3
 
 from followthemoney.types.common import EnumType, EnumValues
 from followthemoney.util import defer as _, gettext
-from followthemoney.util import get_env_list
+from followthemoney.settings import get_env_list
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy

followthemoney/util.py

@@ -10,10 +10,11 @@ from threading import local
 from typing import cast, Dict, Any, List, Optional, TypeVar, Union
 from normality import stringify
 from normality.cleaning import remove_unsafe_chars
-from normality.encoding import DEFAULT_ENCODING
+from rigour.env import ENCODING
 from banal import is_mapping, unique_list, ensure_list
 
 MEGABYTE = 1024 * 1024
+HASH_ENCODING = "utf-8"
 DEFAULT_LOCALE = "en"
 ENTITY_ID_LEN = 200
 
@@ -55,16 +56,7 @@ def get_locale() -> Locale:
     return Locale.parse(state.locale)
 
 
-def get_env_list(name: str, default: List[str] = []) -> List[str]:
-    value = stringify(os.environ.get(name))
-    if value is not None:
-        values = value.split(":")
-        if len(values):
-            return values
-    return default
-
-
-def sanitize_text(value: Any, encoding: str = DEFAULT_ENCODING) -> Optional[str]:
+def sanitize_text(value: Any, encoding: str = ENCODING) -> Optional[str]:
     text = stringify(value, encoding_default=encoding)
     if text is None:
         return None
@@ -74,8 +66,8 @@ def sanitize_text(value: Any, encoding: str = DEFAULT_ENCODING) -> Optional[str]
         log.warning("Cannot NFC text: %s", ex)
         return None
     text = remove_unsafe_chars(text)
-    byte_text = text.encode(DEFAULT_ENCODING, "replace")
-    text = byte_text.decode(DEFAULT_ENCODING, "replace")
+    byte_text = text.encode("utf-8", "replace")
+    text = byte_text.decode("utf-8", "replace")
     if len(text) == 0:
         return None
     return text
@@ -88,7 +80,7 @@ def key_bytes(key: Any) -> bytes:
     text = stringify(key)
     if text is None:
         return b""
-    return text.encode("utf-8")
+    return text.encode(ENCODING)
 
 
 def join_text(*parts: Any, sep: str = " ") -> Optional[str]:

{followthemoney-4.3.4.dist-info → followthemoney-4.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: followthemoney
-Version: 4.3.4
+Version: 4.5.0
 Summary: A data model for anti corruption data modeling and analysis.
 Project-URL: Documentation, https://followthemoney.tech/
 Project-URL: Repository, https://github.com/opensanctions/followthemoney.git
@@ -48,9 +48,9 @@ Requires-Dist: prefixdate<1.0.0,>=0.5.0
 Requires-Dist: pydantic<3.0.0,>=2.11.0
 Requires-Dist: pytz>=2021.1
 Requires-Dist: pyyaml<7.0.0,>=5.0.0
-Requires-Dist: rdflib<7.5.0,>=6.2.0
+Requires-Dist: rdflib<7.6.0,>=6.2.0
 Requires-Dist: requests<3.0.0,>=2.21.0
-Requires-Dist: rigour<2.0.0,>=1.4.0
+Requires-Dist: rigour<2.0.0,>=1.6.0
 Requires-Dist: sqlalchemy[mypy]<3.0.0,>=2.0.0
 Provides-Extra: dev
 Requires-Dist: build; extra == 'dev'

{followthemoney-4.3.4.dist-info → followthemoney-4.5.0.dist-info}/RECORD

@@ -1,19 +1,20 @@
-followthemoney/__init__.py,sha256=UHPYwFuppho0TsOPG7vZw6KSsIDHX_Ar7gO1Vw7dLig,856
-followthemoney/compare.py,sha256=frgumsDv4Ru9UkNof62jDjKCxxpCgV1Rusfu8s20uGA,6327
-followthemoney/entity.py,sha256=YB6u7BMeQX5toAe7DndZBiPtzy0BQ5CKp3ix6kHxk3Y,3499
+followthemoney/__init__.py,sha256=3Mvq7FWl0cNbTovkJhys0iHU24UdFT44P2AqszoGs-8,856
+followthemoney/compare.py,sha256=6y6fqtbbfW7ee4_EVXPcKCIr75GjPqXvtHfvExJ-KK0,6119
+followthemoney/entity.py,sha256=biAjuuHlwsVT02imAsaWP0YtgdfU8skCntzBU3mgJpg,4052
 followthemoney/exc.py,sha256=GyMgwY4QVm87hLevDfV7gM1MJsDqfNCi_UQw7F_A8X8,858
 followthemoney/graph.py,sha256=7X1CGHGvmktS2LSZqld2iXWzG7B831eCNYyBqamqEJ8,10921
 followthemoney/helpers.py,sha256=KCdv1XAE7KQEXBiXp52Kvuck7wMaeNVBM3uaFemcvb4,7873
 followthemoney/messages.py,sha256=zUEa9CFecU8nRafIzhN6TKCh1kEihiIyIS1qr8PxY4g,806
-followthemoney/model.py,sha256=chAUGob5tXWS0o8f0X6mSFCCnI2HoHE5pXU9O5ukrpc,7447
+followthemoney/model.py,sha256=kwv3GT9vGqEMF-m4z94OZJ0phFzcv-K0Eme1ElOMEmU,7346
 followthemoney/names.py,sha256=LODQqExKEHdH4z6Mmbhlm0KeKRzGcptaSWzYXZ7lONI,1120
 followthemoney/namespace.py,sha256=utggu9IGA8bhgEYom3OUB1KxkAJR_TrMNbY5MUF_db8,4536
 followthemoney/ontology.py,sha256=WWY_PYQGl5Ket4zZBuZglzQxD2Bh9UqHok6GJNNX7GA,3001
 followthemoney/property.py,sha256=1w7p9aKLxRqFRnl3PlssqmvulSErl_0D5T2SefT3UFU,8675
-followthemoney/proxy.py,sha256=xZUsT4W9sLojaSD8j6P2JQyQuOQKBWDVovm5epxvtI0,19674
+followthemoney/proxy.py,sha256=ZD8jK88oj1aUTpF7s6r91g8tmTiEfubUKX7CGV5I9rE,20714
 followthemoney/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 followthemoney/schema.py,sha256=dwZg0eZF7yaxP9fJ5NQUKWadWZYTo9U-sVzzXZn_6do,18500
-followthemoney/util.py,sha256=LoCSp1iE6VwXjotCkBXFRppeQs55726GzOuNIu3CvRE,4409
+followthemoney/settings.py,sha256=wkwrkDPypsAICLo-d7l-zpKrKe_fTVYPllZO-RX3oGM,507
+followthemoney/util.py,sha256=JCZ35h_qrqxWO1FTxpa0eoOiko6DWTD9u1QKmH5ZaLU,4150
 followthemoney/value.py,sha256=BJ4Sj5Tg2kMrslR6FjQUr96d8Kt75U7ny9NgzVGT0ZE,2335
 followthemoney/cli/__init__.py,sha256=0mmz84uhXRp2qUn3syKnDXofU3MMAAe291s7htqX0Bg,187
 followthemoney/cli/aggregate.py,sha256=xQTFpU3cVVj7fplpX4OJVrRlTVpn6b9kBr_Vb87pKfg,2164
@@ -26,7 +27,7 @@ followthemoney/cli/util.py,sha256=C3nGMVY3-9JHSFLn3AGvTNcAdvGcgfFS-7jXIzKg6Ik,47
 followthemoney/dataset/__init__.py,sha256=rOKsI39dccDaYcSa7ASoNKkhmbFYUArxMCRqtrxy2iE,477
 followthemoney/dataset/catalog.py,sha256=bIpxr0jvJeutNSmCaXREQac7TyvZak2Y_QoCFdCM0d4,3001
 followthemoney/dataset/coverage.py,sha256=rBnKs7VngCtIuaDqrF5D0ygCHg8NAMkYbmtl7336PSI,724
-followthemoney/dataset/dataset.py,sha256=wWUzWsdzDW9qXLy8lS6Bpy08WMcaNU30oiMXU8jfo14,4724
+followthemoney/dataset/dataset.py,sha256=7lP3gz94AAaxX6J7OWlNmvPoWkoKvH7ISwoNqYsC4Go,5495
 followthemoney/dataset/publisher.py,sha256=nexZe9XexV8WI5Id999vf5OH_DPUmiKQ_GT3c59eF44,893
 followthemoney/dataset/resource.py,sha256=S_-tNjMwHQ8LcSOsZO_xhXD-vLK90wyxtIRBbyCJ0Xo,1164
 followthemoney/dataset/util.py,sha256=mfVTXdbNnWly6cXo4SjNzHuJK1c1uNBwULYOVg1gK5I,1617
@@ -38,7 +39,7 @@ followthemoney/export/graph.py,sha256=v0z1FgadyFk5aQ0A5q8E9R4fSO-Tpi5JU9YTDwnRKD
 followthemoney/export/neo4j.py,sha256=4Lih9lt3-5ATERhyMcfJfkiETG3tqj9vY4N9s7jiYmw,7049
 followthemoney/export/rdf.py,sha256=BOd4AIAVobwpmJ5GjyIqn9ZQHUwKQ-3fMdnD-Lcid0s,2978
 followthemoney/mapping/__init__.py,sha256=iwNqzzvrzJNbNDlOCaDLlBTUrNTlnYHIB5cvo_-9oN4,82
-followthemoney/mapping/csv.py,sha256=1eqQk1tn5JSEcr4rrv44XdT5biUk7J0E275uvUNoOrA,3125
+followthemoney/mapping/csv.py,sha256=bRaqFsr02DNBUZBj-GIBzJOaumnIRu8QbNlYWOMqjHU,3238
 followthemoney/mapping/entity.py,sha256=-x_VBHiVthIrZZ-PVKD3oBAq6LYcsyeYW-9TFv80k7M,5905
 followthemoney/mapping/property.py,sha256=41V16HJh6da7oKdSJWyRcyMkx2XFd6iDm9-4PH7Wihw,5036
 followthemoney/mapping/query.py,sha256=8M6bOlEX2p_bbVwEwTu_1slEtU0cfRJB7ajZp-F07CE,2622
@@ -54,12 +55,12 @@ followthemoney/schema/Audio.yaml,sha256=Eb1rZGUEOX7XDAj_1YIN28NCBzMvkopQBNwgHt_k
 followthemoney/schema/BankAccount.yaml,sha256=60v-VD296lW1Qq7fx--CzxfPNwfCcyMV6xIl8OrSy5g,1431
 followthemoney/schema/Call.yaml,sha256=kbVCnVxucBrEplxehXHThLSJAJjy_GhWan-IeZZjr0M,980
 followthemoney/schema/CallForTenders.yaml,sha256=2IWonTzfSbrkynMoEWqv5fekUeFM_xDKpKIbRe1XDbo,3227
-followthemoney/schema/Company.yaml,sha256=1P_JA2LIIQtzP_Y8FYM-a47VNSWneX8QjdmGD1DFpWQ,3527
+followthemoney/schema/Company.yaml,sha256=R2reqGWtCtURvIAvVB8lRQqGiGPXqjSNdkSWrXLY26w,3546
 followthemoney/schema/Contract.yaml,sha256=aSPB64T1h-0nuLDv6krasUvvoPZgo6sWUbv60c3vmzI,1541
 followthemoney/schema/ContractAward.yaml,sha256=b2spaZHYCaP1yR1RCsrI7mUjk-fAF7BUE3dc8Vl3cUQ,1689
 followthemoney/schema/CourtCase.yaml,sha256=lcovnY0Ne_xcggvkqfCW_RHvsRKo8kFTCPCyovAXRtI,599
 followthemoney/schema/CourtCaseParty.yaml,sha256=MpodN2251V_MYD2dBOHZ_qD7Uv6cLg8Gd_b-I8HZjPI,588
-followthemoney/schema/CryptoWallet.yaml,sha256=dhUpPf1eONsyzyORXoo_zFXgn0BuSLB4OKj2sHZ5wzA,1069
+followthemoney/schema/CryptoWallet.yaml,sha256=haae5h-b8qNBh9FQB2clBUqNYpdLoyyI_NEWMxe3tZ8,1192
 followthemoney/schema/Debt.yaml,sha256=gSGl1xKPaPWAYYEcX7MxezVn3Gu-CYBIzxGzMd9UTm4,657
 followthemoney/schema/Directorship.yaml,sha256=BMx2AQTLy5ta_lWPnYKj7LFjZWTwtu1hgWncISdKf28,773
 followthemoney/schema/Document.yaml,sha256=JxoYl_2o-ebVXh5MzRIKEHfV3q_E--wXGO2HG7zBaZg,2646
@@ -72,29 +73,29 @@ followthemoney/schema/Family.yaml,sha256=cgZ7ZM_2p_dzUGrLVj8QL86gpeDM-AoK8AbyHjh
 followthemoney/schema/Folder.yaml,sha256=W12iJHTYua7bF7oTpgiDbarEkfa0Q2gSmfLdl537HQY,180
 followthemoney/schema/HyperText.yaml,sha256=Wg5dWeLrVjbXiI-ao69tosJ7rI0DvNU8cCo8AhUXwh4,319
 followthemoney/schema/Identification.yaml,sha256=6txjZs6-3Kn94c3G4tDeDt9Jb4FW55-xjSnYVrvmiEA,853
-followthemoney/schema/Image.yaml,sha256=wuznboWECGiV96_GQiXq1-oKNoxO8zKisR4xyusnEn8,394
+followthemoney/schema/Image.yaml,sha256=5fjTy46LqzoybJlPs79RAaBuI-9_i6y77419KGiGOTM,753
 followthemoney/schema/Interest.yaml,sha256=VUrehmsN1WgtS1oAa5jn_JGtSkZGGYLGNahp-R5JhOQ,282
 followthemoney/schema/Interval.yaml,sha256=8YJQ51GI-GxvbjYs3uC593kQtCepWW_7ZiNnlbPm2aM,2084
-followthemoney/schema/LegalEntity.yaml,sha256=Yy28a9NYl6mgnNvNvApO2MAnfkkaZRfzl-GtRn56CFI,5101
+followthemoney/schema/LegalEntity.yaml,sha256=cG-o0slUnQNoJsrgQuk_CaFA_bschjalHHPl-zoKRPA,5352
 followthemoney/schema/License.yaml,sha256=bXESXY-JpSmc5sthZe4sssXhx50UoLPAMED9FvEUyRU,534
 followthemoney/schema/Membership.yaml,sha256=IPmaOX4Ai2r4sGcA5ig2WmLvWHb38akdxp4smEdDWOE,710
 followthemoney/schema/Mention.yaml,sha256=nBeulR_Jm4x75aJ7yNF0TAVhHJqXQaEzOutLIn_YU-4,1086
 followthemoney/schema/Message.yaml,sha256=PAxZ2NRFVvnOlp9Ohh5fJDEThjJ0jm3M2YCbJ9KtMuE,1565
 followthemoney/schema/Note.yaml,sha256=NohwtFupxIssZuEgQowiQWqKit4uQ-OatAu3yp9eJj4,411
 followthemoney/schema/Occupancy.yaml,sha256=WojlqzuWao84MJxRE9K6a-1D-Jtu78-0h6laODhdKw8,975
-followthemoney/schema/Organization.yaml,sha256=F_01GsdSsS2cUjJ732R6UfsiodylMITq23EiXdEuoDo,1136
+followthemoney/schema/Organization.yaml,sha256=Dl6cFDBVFaQfiZrqnaeZAVNvGUiAtj7F4v_ZLxVtPCM,1155
 followthemoney/schema/Ownership.yaml,sha256=tLWESE9VX0aUuhe6C1pToq2-auPVZBdE3xvBmTRfmPc,1057
 followthemoney/schema/Package.yaml,sha256=gPr-P3lcg7OOAav_KVa8baK4yK57JwfcXwxXheD96UQ,310
 followthemoney/schema/Page.yaml,sha256=YjYqaH2sOry0z4xh44CsX_eyuRClD6ZS0d2o2uQXFbo,1062
 followthemoney/schema/Pages.yaml,sha256=KKPGZ06Ehp5mWIGnYfHUBN9jT03bk8nakw0pB5bA_7E,450
 followthemoney/schema/Passport.yaml,sha256=rpuLC86sdXnHF-prFQM4mAqYzlSGWKvPE4Cphtn2KRw,805
 followthemoney/schema/Payment.yaml,sha256=WRBJuj9ljsxLBs-0g9Z9UD87uR1RTtuUiYnWOnKr1qA,1757
-followthemoney/schema/Person.yaml,sha256=485xdaX3YYfJNtMkKuwRS78dD2l2bMYjGCmo-NqTvOY,2845
+followthemoney/schema/Person.yaml,sha256=eFvXYhXbJ9RlhCeX0a0Fv6p_67JD9lMdBAJImZiSzMU,2864
 followthemoney/schema/PlainText.yaml,sha256=hfnVi-HmQeDbqDquSpkPJax9hNm86ioXGr4hzNzyPFE,278
 followthemoney/schema/Position.yaml,sha256=ZpxjWOLxwva_on32r9WD5ys0Ty3YxCju41mg9HG-pe0,1308
 followthemoney/schema/Project.yaml,sha256=2svtyGJopS0UrqPiuYGpBzj30V7k3LRDX4N1U56y4yY,462
 followthemoney/schema/ProjectParticipant.yaml,sha256=xNehEu90uqUfboNouezhZQ8ZQLxzWq1yyNO4kua-Lyc,727
-followthemoney/schema/PublicBody.yaml,sha256=uw4Ok36E8J5d7tyijHG9Sf_iNFOpUhGpo6UyKgTQp3Q,348
+followthemoney/schema/PublicBody.yaml,sha256=N_3-dFNsilHxgIjkq60049PKX46wP0PvP7y9uDpnlwo,367
 followthemoney/schema/RealEstate.yaml,sha256=NWFHXqEHskYQN-kvQESZpu74nztShqoYSZEjZAr-DHM,1363
 followthemoney/schema/Representation.yaml,sha256=sCvFnUDQaElq2cqSB0rILcMYb2gaMZqlzxlHxyX9IGg,792
 followthemoney/schema/Risk.yaml,sha256=2BRVBqb6wiLHxb_V50P-YMAOhjC64UVHDyh5PASpCIA,728
@@ -114,10 +115,10 @@ followthemoney/schema/Vessel.yaml,sha256=zWHUfSK8g6Pz58ZyCaK0AFJ4u_UHjEIUGC4c_7o
 followthemoney/schema/Video.yaml,sha256=LY3DYMWTHXiAhL0hxBCNCz50cp2sPbUlEhhig5Fbjos,327
 followthemoney/schema/Workbook.yaml,sha256=iikWPElz4klA7SkWH7eae6xqhbkMCIP_3zdeXzFEMU0,354
 followthemoney/statement/__init__.py,sha256=7m2VUCAuqNZXIY0WFJRFkw5UG14QuxATL4f_xbqKwhw,633
-followthemoney/statement/entity.py,sha256=oeudwhqfYLJKqbzxEydasMHqevkDASNyYN6s0yddW6I,18755
-followthemoney/statement/serialize.py,sha256=9eXzQ1biR2mSxWRID5C7xDdku4b4ZImHeRJ53yLZ0yo,7225
-followthemoney/statement/statement.py,sha256=Ae-EYuzS8S12BkaRqrvMuI1C7YwlRKa5C_pTBELyNMM,8029
-followthemoney/statement/util.py,sha256=QMYSwAcnh2fCM1LtH_-v8Z5GdwOZfUTT1UkQ_ZMQ470,797
+followthemoney/statement/entity.py,sha256=vznbPTMSWcezYt--bu2RretMresgwp-bUqCsv4w-U90,19568
+followthemoney/statement/serialize.py,sha256=PcG2Qf1jYcF_rF1YybYWBhX7NiX6WZ94u1WUv9Mh-Lw,7386
+followthemoney/statement/statement.py,sha256=bAwrrKyYRdJVxRGuXDQIOA7cdEngk1NKUaij_gAwSd4,11876
+followthemoney/statement/util.py,sha256=jHBwK3FIBynUJZRlBBOHayalAFrqpXf2f2JwkHi0zAU,1450
 followthemoney/translations/messages.pot,sha256=JhtY9NJ9wP_EAX4APxOqMyvKcX53oIC9kAxBsliJkf4,107703
 followthemoney/translations/ar/LC_MESSAGES/followthemoney.mo,sha256=uhb2crSNh8K2ts_QUeD2wvgWgzzpLJWRzXok-Uyx3Zk,38795
 followthemoney/translations/ar/LC_MESSAGES/followthemoney.po,sha256=DuIfvR5v0sPGwFbeg3y6_jCbeglvHWXQ2LDH6prfwLc,121326
@@ -145,15 +146,15 @@ followthemoney/types/__init__.py,sha256=rWwQeiuMh2BNIuvhpMfJ4bPADDvt9Axu1eedvNFi
 followthemoney/types/address.py,sha256=Gc-hqz00dRRkeANqkyPD2wtt7ksR9wMf4CX-U-5XvMo,2214
 followthemoney/types/checksum.py,sha256=_0ev2Wwtd4iX_bLz0Lu-xcJIxNfH_V9kBKKtuZhoAwg,802
 followthemoney/types/common.py,sha256=4ks7zPT8rknrGSd4JFc1zRkS-TL4SX-25_ZbjcVDos0,10081
-followthemoney/types/country.py,sha256=X3Z1j6rIiCITpLtpFXwjTIh9uJwI99_gmPMJx8Jsq2w,1512
-followthemoney/types/date.py,sha256=O3Xav9QNBqjy7LuUWiZrUdGrOvwwOdk6ea5qQEStIwQ,3084
+followthemoney/types/country.py,sha256=pwDiI_ipts9Oi2U7fHALYMJPCJHOqyI_2Liq7XI2XrA,2086
+followthemoney/types/date.py,sha256=SGk8q8qICIrA5Lf6wPbDU6v6oJYEiu1dZFZ18BO7i80,3551
 followthemoney/types/email.py,sha256=L3RTYrMABlNQF7hCynXGfzoj6YNEHW5JAY_BwuhoZdA,3375
 followthemoney/types/entity.py,sha256=56h6x8Ct7hWZIC3BjZHmRKGy9Ff2vuULNWH3xDRsKiU,2317
 followthemoney/types/gender.py,sha256=XY9us98Sk25O1xnHN-88tbv9pHy6Mn7SR8GRYi6v5gI,1683
 followthemoney/types/identifier.py,sha256=TYJwE7urjHFxEcDuiZMxGoCN6n34rAIdCt5_96Y7vI0,2198
 followthemoney/types/ip.py,sha256=rCXkRrh_jDeWAhswCgSe6Z4uhIW7yvLAxIEw4x1SM3A,1279
 followthemoney/types/json.py,sha256=Hefwns1-ziJf310MWvdfX5ICkOgj9cnnMJuqq1e6qKY,1676
-followthemoney/types/language.py,sha256=JDFCO9g9lvgKihhYTz6e7TbJd3V9RTGJlS8kDn6aSCY,2726
+followthemoney/types/language.py,sha256=ymEXaHAPIZGlGySUqzadB7tMB6mW1ASsl1G6EtqKdls,2730
 followthemoney/types/mimetype.py,sha256=oqVP8EfGckPAI3WAziHomp6oUN7KXdIPWzGZPsRtIA8,1242
 followthemoney/types/name.py,sha256=zd0aC4VGp1SYUI8Rj0-ZXlrpUI7ZcnJIljZqsEsV-CY,2363
 followthemoney/types/number.py,sha256=vpAyhmc7UQlIm8h7Z5k8k4cTk37ykRF-AgYA1r_g1QQ,3934
@@ -161,8 +162,8 @@ followthemoney/types/phone.py,sha256=_HanfxxTV7jp75gZO2evBc9HWwQTxEMQRaoVDcoXDIQ
 followthemoney/types/string.py,sha256=SEh3xqQCnm377PGvwfR6ao85pHJCNeCUWBKnvccrJ7I,1216
 followthemoney/types/topic.py,sha256=9FIH_WmwVOFg1CJRBF4KeE6vNTn-QQkzsKU5XaMqNJ0,4604
 followthemoney/types/url.py,sha256=sSHKtzvm4kc-VTvNCPIDykOG1hUoawhORj6Bklo0a2A,1434
-followthemoney-4.3.4.dist-info/METADATA,sha256=H3K0seI3SN6axQwxgjGK0zErE7ySmoYtOxDom-waNDU,6747
-followthemoney-4.3.4.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-followthemoney-4.3.4.dist-info/entry_points.txt,sha256=caoFTlf213jhg5sz3TNSofutjUTzaKtWATuSIdd9Cps,653
-followthemoney-4.3.4.dist-info/licenses/LICENSE,sha256=H6_EVXisnJC0-18CjXIaqrBSFq_VH3OnS7u3dccOv6g,1148
-followthemoney-4.3.4.dist-info/RECORD,,
+followthemoney-4.5.0.dist-info/METADATA,sha256=dteJlaqiJHM7Du9BTLMiOgl-DEjNp2ewkkFU8nMcCYA,6747
+followthemoney-4.5.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+followthemoney-4.5.0.dist-info/entry_points.txt,sha256=caoFTlf213jhg5sz3TNSofutjUTzaKtWATuSIdd9Cps,653
+followthemoney-4.5.0.dist-info/licenses/LICENSE,sha256=H6_EVXisnJC0-18CjXIaqrBSFq_VH3OnS7u3dccOv6g,1148
+followthemoney-4.5.0.dist-info/RECORD,,