followthemoney 4.3.0__py3-none-any.whl → 4.5.0__py3-none-any.whl

followthemoney/statement/serialize.py

@@ -5,9 +5,10 @@ import logging
 from io import TextIOWrapper
 from pathlib import Path
 from types import TracebackType
-from typing import cast
+from typing import Dict, Tuple, cast
 from typing import BinaryIO, Generator, Iterable, List, Optional, TextIO, Type
 from rigour.boolean import text_bool
+from rigour.env import ENCODING
 
 from followthemoney.statement.statement import Statement, StatementDict
 from followthemoney.statement.util import unpack_prop
@@ -60,7 +61,7 @@ def read_json_statements(
 
 
 def read_csv_statements(fh: BinaryIO) -> Generator[Statement, None, None]:
-    wrapped = TextIOWrapper(fh, encoding="utf-8")
+    wrapped = TextIOWrapper(fh, encoding=ENCODING)
     for row in csv.DictReader(wrapped, dialect=csv.unix_dialect):
         data = cast(StatementDict, row)
         data["external"] = text_bool(row.get("external")) or False
@@ -72,7 +73,7 @@ def read_csv_statements(fh: BinaryIO) -> Generator[Statement, None, None]:
 
 
 def read_pack_statements(fh: BinaryIO) -> Generator[Statement, None, None]:
-    wrapped = TextIOWrapper(fh, encoding="utf-8")
+    wrapped = TextIOWrapper(fh, encoding=ENCODING)
     yield from read_pack_statements_decoded(wrapped)
 
 
@@ -129,10 +130,10 @@ def read_path_statements(path: Path, format: str) -> Generator[Statement, None,
 
 def get_statement_writer(fh: BinaryIO, format: str) -> "StatementWriter":
     if format == CSV:
-        wrapped = TextIOWrapper(fh, encoding="utf-8")
+        wrapped = TextIOWrapper(fh, encoding=ENCODING)
         return CSVStatementWriter(wrapped)
     elif format == PACK:
-        wrapped = TextIOWrapper(fh, encoding="utf-8")
+        wrapped = TextIOWrapper(fh, encoding=ENCODING)
         return PackStatementWriter(wrapped)
     elif format == JSON:
         return JSONStatementWriter(fh)
@@ -222,12 +223,14 @@ class PackStatementWriter(StatementWriter):
             "id",
         ]
         self.writer.writerow(columns)
-        self._batch: List[List[Optional[str]]] = []
+        self._batch: Dict[str, Tuple[Optional[str], ...]] = {}
 
     def write(self, stmt: Statement) -> None:
         # HACK: This is very similar to the CSV writer, but at the very inner
         # loop of the application, so we're duplicating code here.
-        row = [
+        if stmt.id is None:
+            raise RuntimeError("Cannot write pack statement without ID")
+        row = (
             stmt.entity_id,
             f"{stmt.schema}:{stmt.prop}",
             stmt.value,
@@ -239,13 +242,15 @@ class PackStatementWriter(StatementWriter):
             stmt.first_seen,
             stmt.last_seen,
             stmt.id,
-        ]
-        self._batch.append(row)
+        )
+        self._batch[stmt.id] = row
         if len(self._batch) >= CSV_BATCH:
-            self.writer.writerows(self._batch)
-            self._batch.clear()
+            self.flush()
+
+    def flush(self) -> None:
+        self.writer.writerows(self._batch.values())
+        self._batch.clear()
 
     def close(self) -> None:
-        if len(self._batch) > 0:
-            self.writer.writerows(self._batch)
+        self.flush()
         self.fh.close()

followthemoney/statement/statement.py

@@ -1,14 +1,22 @@
 import hashlib
 import warnings
 from sqlalchemy.engine import Row
-from typing import cast
-from typing import Any, Dict, Generator, Optional
+from typing import Union, cast
+from typing import Any, Dict, Generator, Optional, TypeGuard
 from typing_extensions import TypedDict, Self
 from rigour.time import datetime_iso, iso_datetime
 from rigour.boolean import bool_text
 
 from followthemoney.proxy import EntityProxy
-from followthemoney.statement.util import get_prop_type, BASE_ID
+from followthemoney.statement.util import get_prop_type, BASE_ID, NON_LANG_TYPE_NAMES
+from followthemoney.util import HASH_ENCODING
+
+
+UNSET = object()
+
+
+def is_not_unset(value: str | None | object) -> TypeGuard[str | None]:
+    return value is not UNSET
 
 
 class StatementDict(TypedDict):
@@ -42,15 +50,16 @@ class Statement(object):
 
     __slots__ = [
         "id",
-        "entity_id",
+        "_entity_id",
         "canonical_id",
-        "prop",
-        "schema",
-        "value",
-        "dataset",
-        "lang",
+        "_prop",
+        "_schema",
+        "_value",
+        "_dataset",
+        "_lang",
+        "prop_type",
         "original_value",
-        "external",
+        "_external",
         "first_seen",
         "last_seen",
         "origin",
@@ -72,55 +81,95 @@ class Statement(object):
         last_seen: Optional[str] = None,
         origin: Optional[str] = None,
     ):
-        self.entity_id = entity_id
+        self._entity_id = entity_id
         self.canonical_id = canonical_id or entity_id
-        self.prop = prop
-        self.schema = schema
-        self.value = value
-        self.dataset = dataset
-        self.lang = lang
+        self._prop = prop
+        self._schema = schema
+        self.prop_type = get_prop_type(schema, prop)
+        self._value = value
+        self._dataset = dataset
+
+        # Remove lang for non-linguistic property types. The goal here is to avoid
+        # duplicate statements because of language tags, but the language metadata
+        # may be relevant as context for how the original_value was parsed so it's
+        # a bit of information loss.
+        if lang is not None:
+            if self.prop_type in NON_LANG_TYPE_NAMES:
+                lang = None
+        self._lang = lang
+
         self.original_value = original_value
         self.first_seen = first_seen
         self.last_seen = last_seen or first_seen
-        self.external = external
+        self._external = external
         self.origin = origin
         if id is None:
             id = self.generate_key()
         self.id = id
 
     @property
-    def prop_type(self) -> str:
-        """The type of the property, e.g. 'string', 'number', 'url'."""
-        return get_prop_type(self.schema, self.prop)
+    def entity_id(self) -> str:
+        """The (original) ID of the entity this statement is about."""
+        return self._entity_id
+
+    @property
+    def dataset(self) -> str:
+        """The dataset this statement was observed in."""
+        return self._dataset
+
+    @property
+    def prop(self) -> str:
+        """The property name this statement is about."""
+        return self._prop
+
+    @property
+    def schema(self) -> str:
+        """The schema of the entity this statement is about."""
+        return self._schema
+
+    @property
+    def value(self) -> str:
+        """The value of the property captured by this statement."""
+        return self._value
+
+    @property
+    def lang(self) -> Optional[str]:
+        """The language of the property value, if applicable."""
+        return self._lang
+
+    @property
+    def external(self) -> bool:
+        """Whether this statement was observed in an external dataset."""
+        return self._external
 
     def to_dict(self) -> StatementDict:
         return {
             "canonical_id": self.canonical_id,
-            "entity_id": self.entity_id,
-            "prop": self.prop,
-            "schema": self.schema,
-            "value": self.value,
-            "dataset": self.dataset,
-            "lang": self.lang,
+            "entity_id": self._entity_id,
+            "prop": self._prop,
+            "schema": self._schema,
+            "value": self._value,
+            "dataset": self._dataset,
+            "lang": self._lang,
             "original_value": self.original_value,
             "first_seen": self.first_seen,
             "last_seen": self.last_seen,
-            "external": self.external,
+            "external": self._external,
             "origin": self.origin,
             "id": self.id,
         }
 
     def to_csv_row(self) -> Dict[str, Optional[str]]:
         data = cast(Dict[str, Optional[str]], self.to_dict())
-        data["external"] = bool_text(self.external)
-        data["prop_type"] = get_prop_type(self.schema, self.prop)
+        data["external"] = bool_text(self._external)
+        data["prop_type"] = self.prop_type
         return data
 
     def to_db_row(self) -> Dict[str, Any]:
         data = cast(Dict[str, Any], self.to_dict())
         data["first_seen"] = iso_datetime(self.first_seen)
         data["last_seen"] = iso_datetime(self.last_seen)
-        data["prop_type"] = get_prop_type(self.schema, self.prop)
+        data["prop_type"] = self.prop_type
         return data
 
     def __hash__(self) -> int:
@@ -132,27 +181,83 @@ class Statement(object):
         return hash(self.id)
 
     def __repr__(self) -> str:
-        return "<Statement(%r, %r, %r)>" % (self.entity_id, self.prop, self.value)
+        return "<Statement(%r, %r, %r)>" % (self._entity_id, self._prop, self._value)
 
     def __eq__(self, other: Any) -> bool:
         return not self.id != other.id
 
     def __lt__(self, other: Any) -> bool:
-        self_key = (self.prop != BASE_ID, self.id or "")
-        other_key = (other.prop != BASE_ID, other.id or "")
+        self_key = (self._prop != BASE_ID, self.id or "")
+        other_key = (other._prop != BASE_ID, other.id or "")
         return self_key < other_key
 
-    def clone(self: Self) -> "Statement":
+    def clone(
+        self: Self,
+        *,
+        entity_id: Optional[str] = None,
+        prop: Optional[str] = None,
+        schema: Optional[str] = None,
+        value: Optional[str] = None,
+        dataset: Optional[str] = None,
+        lang: Union[str, None, object] = UNSET,
+        original_value: Union[str, None, object] = UNSET,
+        first_seen: Union[str, None, object] = UNSET,
+        external: Optional[bool] = None,
+        canonical_id: Optional[str] = None,
+        last_seen: Union[str, None, object] = UNSET,
+        origin: Union[str, None, object] = UNSET,
+    ) -> "Statement":
         """Make a deep copy of the given statement."""
-        return Statement.from_dict(self.to_dict())
+        lang = lang if is_not_unset(lang) else self._lang
+        ov = original_value if is_not_unset(original_value) else self.original_value
+        first_seen = first_seen if is_not_unset(first_seen) else self.first_seen
+        last_seen = last_seen if is_not_unset(last_seen) else self.last_seen
+        origin = origin if is_not_unset(origin) else self.origin
+        if external is None:
+            external = self._external
+        if canonical_id is None and self._entity_id != self.canonical_id:
+            canonical_id = self.canonical_id
+
+        # Decide if the statement ID can be kept the same:
+        stmt_id = self.id
+        if entity_id is not None and entity_id != self.entity_id:
+            stmt_id = None
+        if prop is not None and prop != self._prop:
+            stmt_id = None
+        if schema is not None and schema != self._schema:
+            stmt_id = None
+        if value is not None and value != self._value:
+            stmt_id = None
+        if dataset is not None and dataset != self._dataset:
+            stmt_id = None
+        if external != self._external:
+            stmt_id = None
+        if lang != self._lang:
+            stmt_id = None
+        return Statement(
+            id=stmt_id,
+            entity_id=entity_id or self._entity_id,
+            prop=prop or self._prop,
+            schema=schema or self._schema,
+            value=value or self._value,
+            dataset=dataset or self._dataset,
+            lang=lang,
+            original_value=ov,
+            first_seen=first_seen,
+            external=external,
+            canonical_id=canonical_id,
+            last_seen=last_seen,
+            origin=origin,
+        )
 
     def generate_key(self) -> Optional[str]:
         return self.make_key(
-            self.dataset,
-            self.entity_id,
-            self.prop,
-            self.value,
-            self.external,
+            self._dataset,
+            self._entity_id,
+            self._prop,
+            self._value,
+            self._external,
+            lang=self._lang,
         )
 
     @classmethod
@@ -163,17 +268,21 @@ class Statement(object):
         prop: str,
         value: str,
         external: Optional[bool],
+        lang: Optional[str] = None,
     ) -> Optional[str]:
         """Hash the key properties of a statement record to make a unique ID."""
         if prop is None or value is None:
             return None
-        key = f"{dataset}.{entity_id}.{prop}.{value}"
+        if lang is None:
+            key = f"{dataset}.{entity_id}.{prop}.{value}"
+        else:
+            key = f"{dataset}.{entity_id}.{prop}.{value}@{lang}"
         if external:
             # We consider the external flag in key composition to avoid race conditions
             # where a certain entity might be emitted as external while it is already
             # linked in to the graph via another route.
             key = f"{key}.ext"
-        return hashlib.sha1(key.encode("utf-8")).hexdigest()
+        return hashlib.sha1(key.encode(HASH_ENCODING)).hexdigest()
 
     @classmethod
     def from_dict(cls, data: StatementDict) -> "Statement":

followthemoney/statement/util.py

@@ -1,11 +1,32 @@
-import sys
 from functools import cache
 from typing import Tuple
 
 from followthemoney.model import Model
+from followthemoney.types import registry
+from followthemoney.util import const
 
 BASE_ID = "id"
 
+# Some property types should not set the `lang` attribute on statements.
+# These are typically non-linguistic types, although there's an argument
+# that language metadata could be useful for dates and countries, where
+# text parsing is likely to have taken place.
+NON_LANG_TYPE_NAMES = {
+    registry.entity.name,
+    registry.date.name,
+    registry.checksum.name,
+    registry.email.name,
+    registry.phone.name,
+    registry.gender.name,
+    registry.mimetype.name,
+    registry.topic.name,
+    registry.url.name,
+    registry.country.name,
+    registry.language.name,
+    registry.ip.name,
+    BASE_ID,
+}
+
 
 def pack_prop(schema: str, prop: str) -> str:
     return f"{schema}:{prop}"
@@ -28,4 +49,4 @@ def get_prop_type(schema: str, prop: str) -> str:
 def unpack_prop(id: str) -> Tuple[str, str, str]:
     schema, prop = id.split(":", 1)
     prop_type = get_prop_type(schema, prop)
-    return sys.intern(schema), prop_type, sys.intern(prop)
+    return const(schema), prop_type, const(prop)

followthemoney/types/address.py

@@ -6,7 +6,7 @@ from rigour.text.distance import levenshtein_similarity
 
 from followthemoney.types.common import PropertyType
 from followthemoney.util import defer as _
-from followthemoney.util import dampen, const
+from followthemoney.util import dampen
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -20,8 +20,8 @@ class AddressType(PropertyType):
 
     LINE_BREAKS = re.compile(r"(\r\n|\n|<BR/>|<BR>|\t|ESQ\.,|ESQ,|;)")
     COMMATA = re.compile(r"(,\s?[,\.])")
-    name = const("address")
-    group = const("addresses")
+    name = "address"
+    group = "addresses"
     label = _("Address")
     plural = _("Addresses")
     matchable = True

followthemoney/types/checksum.py

@@ -1,5 +1,5 @@
 from followthemoney.types.common import PropertyType
-from followthemoney.util import const, defer as _
+from followthemoney.util import defer as _
 
 
 class ChecksumType(PropertyType):
@@ -12,8 +12,8 @@ class ChecksumType(PropertyType):
     of this type are scrubbed when submitted via the normal API. Checksums can only
     be defined by uploading a document to be ingested."""
 
-    name = const("checksum")
-    group = const("checksums")
+    name = "checksum"
+    group = "checksums"
     label = _("Checksum")
     plural = _("Checksums")
     matchable = True

followthemoney/types/country.py

@@ -1,9 +1,10 @@
-from typing import Optional, TYPE_CHECKING
+from typing import Callable, Optional, TYPE_CHECKING, Sequence
 from babel.core import Locale
 from rigour.territories import get_ftm_countries, lookup_territory
+from rigour.territories import territories_intersect
 
 from followthemoney.types.common import EnumType, EnumValues
-from followthemoney.util import const, defer as _
+from followthemoney.util import defer as _
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -15,8 +16,8 @@ class CountryType(EnumType):
     a number of unusual and controversial designations (e.g. the Soviet Union,
     Transnistria, Somaliland, Kosovo)."""
 
-    name = const("country")
-    group = const("countries")
+    name = "country"
+    group = "countries"
     label = _("Country")
     plural = _("Countries")
     matchable = True
@@ -25,6 +26,20 @@ class CountryType(EnumType):
     def _locale_names(self, locale: Locale) -> EnumValues:
         return {t.code: t.name for t in get_ftm_countries()}
 
+    def compare(self, left: str, right: str) -> float:
+        overlap = territories_intersect([left], [right])
+        return 1.0 if len(overlap) else 0.0
+
+    def compare_sets(
+        self,
+        left: Sequence[str],
+        right: Sequence[str],
+        func: Callable[[Sequence[float]], float] = max,
+    ) -> float:
+        """Compare two sets of values and select the highest-scored result."""
+        overlap = territories_intersect(left, right)
+        return 1.0 if len(overlap) else 0.0
+
     def clean_text(
         self,
         text: str,

followthemoney/types/date.py

@@ -5,7 +5,7 @@ from prefixdate import parse, parse_format, Precision
 
 from followthemoney.types.common import PropertyType
 from followthemoney.util import defer as _
-from followthemoney.util import dampen, const
+from followthemoney.util import dampen
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -20,13 +20,23 @@ class DateType(PropertyType):
     The timezone is always expected to be UTC and cannot be specified otherwise. There is
     no support for calendar weeks (`2021-W7`) and date ranges (`2021-2024`)."""
 
-    name = const("date")
-    group = const("dates")
+    name = "date"
+    group = "dates"
     label = _("Date")
     plural = _("Dates")
     matchable = True
     max_length = 32
 
+    HISTORIC = "1001-01-01"
+    """A sentinel date value representing a very old date, used to indicate historic (and often imprecise) dates
+    that can be assumed to be long in the past."""
+
+    RELEVANCE_MIN = "1900-01-01"
+    """A cutoff date value representing the minimum relevant date for modern fincrime applications."""
+
+    RELEVANCE_MAX = "2100-12-31"
+    """A cutoff date value representing the maximum relevant date for modern fincrime applications."""
+
     def validate(
         self, value: str, fuzzy: bool = False, format: Optional[str] = None
     ) -> bool:

followthemoney/types/entity.py

@@ -4,7 +4,7 @@ from typing import Any, Optional, TYPE_CHECKING
 from followthemoney.types.common import PropertyType
 from followthemoney.value import Value
 from followthemoney.util import ENTITY_ID_LEN, get_entity_id, sanitize_text
-from followthemoney.util import const, gettext, defer as _
+from followthemoney.util import gettext, defer as _
 from followthemoney.exc import InvalidData
 
 if TYPE_CHECKING:
@@ -22,8 +22,8 @@ class EntityType(PropertyType):
 
     REGEX_RAW = r"^[0-9a-zA-Z]([0-9a-zA-Z\.\-]*[0-9a-zA-Z])?$"
     REGEX = re.compile(REGEX_RAW)
-    name = const("entity")
-    group = const("entities")
+    name = "entity"
+    group = "entities"
     label = _("Entity")
     plural = _("Entities")
     matchable = True

followthemoney/types/gender.py

@@ -2,7 +2,7 @@ from typing import Optional, TYPE_CHECKING
 from babel.core import Locale
 
 from followthemoney.types.common import EnumType, EnumValues
-from followthemoney.util import const, gettext, defer as _
+from followthemoney.util import gettext, defer as _
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -14,9 +14,9 @@ class GenderType(EnumType):
     government databases and represent it in a way that can be used by
     structured tools. I'm not sure this justifies the simplification."""
 
-    MALE = const("male")
-    FEMALE = const("female")
-    OTHER = const("other")
+    MALE = "male"
+    FEMALE = "female"
+    OTHER = "other"
 
     LOOKUP = {
         "m": MALE,
@@ -34,8 +34,8 @@ class GenderType(EnumType):
         "divers": OTHER,
     }
 
-    name = const("gender")
-    group = const("genders")
+    name = "gender"
+    group = "genders"
     label = _("Gender")
     plural = _("Genders")
     matchable = False

followthemoney/types/identifier.py

@@ -1,10 +1,10 @@
 import re
 from typing import Optional, TYPE_CHECKING
-from rigour.ids import get_identifier_format_names, get_identifier_format
+from rigour.ids import get_identifier_format
 
 from followthemoney.types.common import PropertyType
 from followthemoney.util import dampen, shortest, longest
-from followthemoney.util import const, defer as _
+from followthemoney.util import defer as _
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -20,8 +20,8 @@ class IdentifierType(PropertyType):
     Four- or five-digit industry classifiers create more noise than value."""
 
     COMPARE_CLEAN = re.compile(r"[\W_]+")
-    name = const("identifier")
-    group = const("identifiers")
+    name = "identifier"
+    group = "identifiers"
     label = _("Identifier")
     plural = _("Identifiers")
     matchable = True
@@ -35,8 +35,8 @@ class IdentifierType(PropertyType):
         format: Optional[str] = None,
         proxy: Optional["EntityProxy"] = None,
     ) -> Optional[str]:
-        if format in get_identifier_format_names():
-            format_ = get_identifier_format(format)
+        format_ = get_identifier_format(format)
+        if format_ is not None:
             return format_.normalize(text)
         return text
 
@@ -61,7 +61,7 @@ class IdentifierType(PropertyType):
         return f"id:{value}"
 
     def caption(self, value: str, format: Optional[str] = None) -> str:
-        if format in get_identifier_format_names():
-            format_ = get_identifier_format(format)
+        format_ = get_identifier_format(format)
+        if format_ is not None:
             return format_.format(value)
         return value

followthemoney/types/ip.py

@@ -2,7 +2,7 @@ from typing import Optional, TYPE_CHECKING
 from ipaddress import ip_address
 
 from followthemoney.types.common import PropertyType
-from followthemoney.util import const, defer as _
+from followthemoney.util import defer as _
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -13,8 +13,8 @@ class IpType(PropertyType):
     by the protocol versions 4 (e.g. `192.168.1.143`) and 6
     (e.g. `0:0:0:0:0:ffff:c0a8:18f`)."""
 
-    name = const("ip")
-    group = const("ips")
+    name = "ip"
+    group = "ips"
     label = _("IP Address")
     plural = _("IP Addresses")
     matchable = True

followthemoney/types/json.py

@@ -3,7 +3,7 @@ from typing import Any, Optional, Sequence, TYPE_CHECKING
 from banal import ensure_list
 
 from followthemoney.types.common import PropertyType
-from followthemoney.util import const, sanitize_text, defer as _
+from followthemoney.util import sanitize_text, defer as _
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -14,7 +14,7 @@ class JsonType(PropertyType):
     and some other edge cases. It's a really bad idea and we should try to get rid
     of JSON properties."""
 
-    name = const("json")
+    name = "json"
     group = None
     label = _("Nested data")
     plural = _("Nested data")