followthemoney 3.8.5__py3-none-any.whl → 4.0.1__py3-none-any.whl

followthemoney/types/gender.py

@@ -2,8 +2,7 @@ from typing import Optional, TYPE_CHECKING
 from babel.core import Locale
 
 from followthemoney.types.common import EnumType, EnumValues
-from followthemoney.rdf import URIRef, Identifier
-from followthemoney.util import gettext, defer as _
+from followthemoney.util import const, gettext, defer as _
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -15,9 +14,9 @@ class GenderType(EnumType):
     government databases and represent it in a way that can be used by
     structured tools. I'm not sure this justifies the simplification."""
 
-    MALE = "male"
-    FEMALE = "female"
-    OTHER = "other"
+    MALE = const("male")
+    FEMALE = const("female")
+    OTHER = const("other")
 
     LOOKUP = {
         "m": MALE,
@@ -35,8 +34,8 @@ class GenderType(EnumType):
         "divers": OTHER,
     }
 
-    name = "gender"
-    group = "genders"
+    name = const("gender")
+    group = const("genders")
     label = _("Gender")
     plural = _("Genders")
     matchable = False
@@ -61,6 +60,3 @@ class GenderType(EnumType):
         if code not in self.codes:
             return None
         return code
-
-    def rdf(self, value: str) -> Identifier:
-        return URIRef(f"gender:{value}")

followthemoney/types/identifier.py

@@ -4,7 +4,7 @@ from rigour.ids import get_identifier_format_names, get_identifier_format
 
 from followthemoney.types.common import PropertyType
 from followthemoney.util import dampen, shortest, longest
-from followthemoney.util import defer as _
+from followthemoney.util import const, defer as _
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -20,8 +20,8 @@ class IdentifierType(PropertyType):
     Four- or five-digit industry classifiers create more noise than value."""
 
     COMPARE_CLEAN = re.compile(r"[\W_]+")
-    name = "identifier"
-    group = "identifiers"
+    name = const("identifier")
+    group = const("identifiers")
     label = _("Identifier")
     plural = _("Identifiers")
     matchable = True
@@ -59,3 +59,9 @@ class IdentifierType(PropertyType):
 
     def node_id(self, value: str) -> str:
         return f"id:{value}"
+
+    def caption(self, value: str, format: Optional[str] = None) -> str:
+        if format in get_identifier_format_names():
+            format_ = get_identifier_format(format)
+            return format_.format(value)
+        return value

followthemoney/types/ip.py

@@ -2,8 +2,7 @@ from typing import Optional, TYPE_CHECKING
 from ipaddress import ip_address
 
 from followthemoney.types.common import PropertyType
-from followthemoney.rdf import URIRef, Identifier
-from followthemoney.util import defer as _
+from followthemoney.util import const, defer as _
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -14,10 +13,10 @@ class IpType(PropertyType):
     by the protocol versions 4 (e.g. `192.168.1.143`) and 6
     (e.g. `0:0:0:0:0:ffff:c0a8:18f`)."""
 
-    name = "ip"
-    group = "ips"
-    label = _("IP-Address")
-    plural = _("IP-Addresses")
+    name = const("ip")
+    group = const("ips")
+    label = _("IP Address")
+    plural = _("IP Addresses")
     matchable = True
     pivot = True
     max_length = 64
@@ -45,6 +44,3 @@ class IpType(PropertyType):
             return str(ip_address(text))
         except ValueError:
             return None
-
-    def rdf(self, value: str) -> Identifier:
-        return URIRef(f"ip:{value}")

followthemoney/types/json.py

@@ -3,7 +3,7 @@ from typing import Any, Optional, Sequence, TYPE_CHECKING
 from banal import ensure_list
 
 from followthemoney.types.common import PropertyType
-from followthemoney.util import sanitize_text, defer as _
+from followthemoney.util import const, sanitize_text, defer as _
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -14,7 +14,7 @@ class JsonType(PropertyType):
     and some other edge cases. It's a really bad idea and we should try to get rid
     of JSON properties."""
 
-    name = "json"
+    name = const("json")
     group = None
     label = _("Nested data")
     plural = _("Nested data")

followthemoney/types/language.py

@@ -3,9 +3,8 @@ from babel.core import Locale
 from rigour.langs import iso_639_alpha3
 
 from followthemoney.types.common import EnumType, EnumValues
-from followthemoney.rdf import URIRef, Identifier
 from followthemoney.util import defer as _, gettext
-from followthemoney.util import get_env_list
+from followthemoney.util import const, get_env_list
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -17,8 +16,8 @@ class LanguageType(EnumType):
     for additional languages once there is a specific need for them to be
     supported."""
 
-    name = "language"
-    group = "languages"
+    name = const("language")
+    group = const("languages")
     label = _("Language")
     plural = _("Languages")
     matchable = False
@@ -120,6 +119,3 @@ class LanguageType(EnumType):
         if code not in self.LANGUAGES:
             return None
         return code
-
-    def rdf(self, value: str) -> Identifier:
-        return URIRef(f"iso-639:{value}")

followthemoney/types/mimetype.py

@@ -3,8 +3,7 @@ from rigour.mime import normalize_mimetype, parse_mimetype
 from rigour.mime import DEFAULT
 
 from followthemoney.types.common import PropertyType
-from followthemoney.rdf import URIRef, Identifier
-from followthemoney.util import defer as _
+from followthemoney.util import const, defer as _
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -19,8 +18,8 @@ class MimeType(PropertyType):
     MIME type properties do not contain parameters as used in HTTP headers,
     like `charset=UTF-8`."""
 
-    name = "mimetype"
-    group = "mimetypes"
+    name = const("mimetype")
+    group = const("mimetypes")
     label = _("MIME-Type")
     plural = _("MIME-Types")
     matchable = False
@@ -37,8 +36,5 @@ class MimeType(PropertyType):
             return text
         return None
 
-    def rdf(self, value: str) -> Identifier:
-        return URIRef(f"urn:mimetype:{value}")
-
-    def caption(self, value: str) -> str:
+    def caption(self, value: str, format: Optional[str] = None) -> str:
         return parse_mimetype(value).label or value

followthemoney/types/name.py

@@ -2,13 +2,12 @@ from typing import TYPE_CHECKING, Optional, Sequence
 from normality import slugify
 from normality.cleaning import collapse_spaces, strip_quotes
 from rigour.env import MAX_NAME_LENGTH
-from rigour.names import pick_name
+from rigour.names import pick_name, tokenize_name
 from rigour.text.distance import levenshtein_similarity
-from fingerprints.cleanup import clean_name_light
 
 from followthemoney.types.common import PropertyType
 from followthemoney.util import dampen
-from followthemoney.util import defer as _
+from followthemoney.util import const, defer as _
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -22,8 +21,8 @@ class NameType(PropertyType):
     No validation rules apply, and things having multiple names must be considered
     a perfectly ordinary case."""
 
-    name = "name"
-    group = "names"
+    name = const("name")
+    group = const("names")
     label = _("Name")
     plural = _("Names")
     matchable = True
@@ -51,9 +50,9 @@ class NameType(PropertyType):
 
     def compare(self, left: str, right: str) -> float:
         """Compare two names for similarity."""
-        left_clean = clean_name_light(left)
-        right_clean = clean_name_light(right)
-        if left_clean is None or right_clean is None:
+        left_clean = " ".join(tokenize_name(left.lower()))
+        right_clean = " ".join(tokenize_name(right.lower()))
+        if not len(left_clean) or not len(right_clean):
             return 0.0
         return levenshtein_similarity(
             left_clean,

followthemoney/types/number.py

@@ -1,8 +1,8 @@
 import re
-from typing import Optional
+from typing import Optional, Tuple
 
 from followthemoney.types.common import PropertyType
-from followthemoney.util import defer as _
+from followthemoney.util import const, defer as _
 
 
 class NumberType(PropertyType):
@@ -13,8 +13,18 @@ class NumberType(PropertyType):
     In the future we might want to enable annotations for format, units, or
     even to introduce a separate property type for monetary values."""
 
-    CAST_RE = re.compile(r"[^0-9\-\.]")
-    name = "number"
+    DECIMAL = "."
+    SEPARATOR = ","
+    PRECISION = 2
+
+    _NUM_UNIT_RE = (
+        f"(\\s?\\-?\\s?\\d+(?:{re.escape(DECIMAL)}\\d+)?)\\s*([^\\s\\d][^\\s]*)?"
+    )
+    NUM_UNIT_RE = re.compile(_NUM_UNIT_RE, re.UNICODE)
+    _FLOAT_FMT = "{:" + SEPARATOR + "." + str(PRECISION) + "f}"
+    _INT_FMT = "{:" + SEPARATOR + "d}"
+
+    name = const("number")
     label = _("Number")
     plural = _("Numbers")
     matchable = False
@@ -22,9 +32,81 @@ class NumberType(PropertyType):
     def node_id(self, value: str) -> None:
         return None
 
+    def parse(
+        self, value: str, decimal: str = DECIMAL, separator: str = SEPARATOR
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """Parse a number into a numeric value and a unit. The numeric value is
+        aligned with the decimal and separator settings. The unit is stripped of
+        whitespace and returned as a string. If no unit is found, None is
+        returned. If no number is found, None is returned for both values.
+
+        Args:
+            value (str): The string to parse.
+            decimal (str): The character used as the decimal separator.
+            separator (str): The character used to separate thousands, lakhs, or crores.
+
+        Returns:
+            A tuple of (number, unit), where number is a string and unit is a string or None.
+        """
+        value = value.replace(separator, "")
+        if decimal != self.DECIMAL:
+            value = value.replace(decimal, self.DECIMAL)
+        match = self.NUM_UNIT_RE.match(value)
+        if not match:
+            return None, None
+        number, unit = match.groups()
+        if unit is not None:
+            unit = unit.strip()
+            if len(unit) == 0:
+                unit = None
+        # TODO: We could have a lookup table for common units, e.g. kg, m, etc. to
+        # convert them to a standard form.
+        number = number.replace(" ", "")
+        if number == "":
+            number = None
+        return number, unit
+
     def to_number(self, value: str) -> Optional[float]:
+        """Convert a number string to a float. The string is parsed and the unit is
+        discarded if present.
+
+        Args:
+            value (str): The string to convert.
+
+        Returns:
+            Optional[float]: The parsed float value, or None if parsing fails.
+        """
         try:
-            value = self.CAST_RE.sub("", value)
-            return float(value)
+            number, _ = self.parse(value)
+            if number is None:
+                return None
+            return float(number)
         except Exception:
             return None
+
+    def caption(self, value: str, format: Optional[str] = None) -> str:
+        """Return a caption for the number. This is used for display purposes.
+
+        Args:
+            value (str): The string to format.
+            format (Optional[str]): An optional format string to use for formatting the number.
+
+        Returns:
+            str: The formatted number string, possibly with a unit.
+        """
+        number, unit = self.parse(value)
+        if number is None:
+            return value
+        try:
+            fnumber = float(number)
+        except ValueError:
+            return value
+        if format is not None:
+            number = format.format(fnumber)
+        elif fnumber.is_integer():
+            number = self._INT_FMT.format(int(fnumber))
+        else:
+            number = self._FLOAT_FMT.format(fnumber)
+        if unit is not None:
+            return f"{number} {unit}"
+        return number

followthemoney/types/phone.py

@@ -5,9 +5,8 @@ from phonenumbers import PhoneNumber, PhoneNumberFormat
 from phonenumbers.phonenumberutil import region_code_for_number, NumberParseException
 
 from followthemoney.types.common import PropertyType
-from followthemoney.rdf import URIRef, Identifier
 from followthemoney.util import defer as _
-from followthemoney.util import dampen
+from followthemoney.util import const, dampen
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -30,8 +29,8 @@ class PhoneType(PropertyType):
     validation outcome from doing the two operations the other way around. Always
     define the country first."""
 
-    name = "phone"
-    group = "phones"
+    name = const("phone")
+    group = const("phones")
     label = _("Phone number")
     plural = _("Phone numbers")
     matchable = True
@@ -97,16 +96,10 @@ class PhoneType(PropertyType):
         # TODO: insert artificial intelligence here.
         return dampen(7, 11, value)
 
-    def rdf(self, value: str) -> Identifier:
-        node_id = self.node_id(value)
-        if node_id is not None:
-            return URIRef(node_id)
-        raise ValueError("Invalid phone number for serialisation: %s" % value)
-
     def node_id(self, value: str) -> Optional[str]:
         return f"tel:{value}"
 
-    def caption(self, value: str) -> str:
+    def caption(self, value: str, format: Optional[str] = None) -> str:
         try:
             number = parse_number(value)
             formatted = format_number(number, PhoneNumberFormat.INTERNATIONAL)

followthemoney/types/string.py

@@ -1,12 +1,12 @@
 from followthemoney.types.common import PropertyType
-from followthemoney.util import defer as _
+from followthemoney.util import const, defer as _
 from followthemoney.util import MEGABYTE
 
 
 class StringType(PropertyType):
     """A simple string property with no additional semantics."""
 
-    name = "string"
+    name = const("string")
     label = _("Label")
     plural = _("Labels")
     matchable = False
@@ -21,7 +21,7 @@ class TextType(StringType):
     string properties, it might make sense to treat properties of this type as
     full-text search material."""
 
-    name = "text"
+    name = const("text")
     label = _("Text")
     plural = _("Texts")
     total_size = 30 * MEGABYTE
@@ -36,7 +36,7 @@ class HTMLType(StringType):
     to perform server-side sanitisation, or to not render this property at all.
     """
 
-    name = "html"
+    name = const("html")
     label = _("HTML")
     plural = _("HTMLs")
     total_size = 30 * MEGABYTE

followthemoney/types/topic.py

@@ -1,8 +1,7 @@
 from babel.core import Locale
 
 from followthemoney.types.common import EnumType, EnumValues
-from followthemoney.rdf import URIRef, Identifier
-from followthemoney.util import gettext, defer as _
+from followthemoney.util import const, gettext, defer as _
 
 
 class TopicType(EnumType):
@@ -16,8 +15,8 @@ class TopicType(EnumType):
     enable queries such as _find all paths between a government procurement
     award and a politician_."""
 
-    name = "topic"
-    group = "topics"
+    name = const("topic")
+    group = const("topics")
     label = _("Topic")
     plural = _("Topics")
     matchable = False
@@ -90,6 +89,3 @@ class TopicType(EnumType):
 
     def _locale_names(self, locale: Locale) -> EnumValues:
         return {k: gettext(v) for (k, v) in self._TOPICS.items()}
-
-    def rdf(self, value: str) -> Identifier:
-        return URIRef(f"ftm:topic:{value}")

followthemoney/types/url.py

@@ -2,8 +2,7 @@ from typing import Optional, TYPE_CHECKING
 from rigour.urls import clean_url, compare_urls
 
 from followthemoney.types.common import PropertyType
-from followthemoney.rdf import URIRef, Identifier
-from followthemoney.util import dampen, defer as _
+from followthemoney.util import const, dampen, defer as _
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -17,8 +16,8 @@ class UrlType(PropertyType):
     SCHEMES = ("http", "https", "ftp", "mailto")
     DEFAULT_SCHEME = "http"
 
-    name = "url"
-    group = "urls"
+    name = const("url")
+    group = const("urls")
     label = _("URL")
     plural = _("URLs")
     matchable = True
@@ -37,13 +36,9 @@ class UrlType(PropertyType):
         return clean_url(text)
 
     def compare(self, left: str, right: str) -> float:
+        """Compare two URLs and return a float indicating how similar they are. This ignores
+        fragments and peforms hard URL normalisation."""
         return compare_urls(left, right)
 
     def _specificity(self, value: str) -> float:
         return dampen(10, 120, value)
-
-    def rdf(self, value: str) -> Identifier:
-        return URIRef(value)
-
-    def node_id(self, value: str) -> Optional[str]:
-        return f"url:{value}"

followthemoney/util.py

@@ -1,11 +1,12 @@
 import os
+import sys
 import logging
 from hashlib import sha1
 from babel import Locale
 from gettext import translation
 
 from threading import local
-from typing import cast, Dict, Any, List, Optional, TypeVar, Union, Sequence
+from typing import cast, Dict, Any, List, Optional, TypeVar, Union
 from normality import stringify
 from normality.cleaning import compose_nfc
 from normality.cleaning import remove_unsafe_chars
@@ -36,6 +37,11 @@ def defer(text: str) -> str:
     return text
 
 
+def const(text: str) -> str:
+    """Convert the given text to a runtime constant."""
+    return sys.intern(text.strip())
+
+
 def set_model_locale(locale: Locale) -> None:
     state.locale = locale
     state.translation = translation(
@@ -58,12 +64,14 @@ def get_env_list(name: str, default: List[str] = []) -> List[str]:
     return default
 
 
-def sanitize_text(text: Any, encoding: str = DEFAULT_ENCODING) -> Optional[str]:
-    text = stringify(text, encoding_default=encoding)
+def sanitize_text(value: Any, encoding: str = DEFAULT_ENCODING) -> Optional[str]:
+    text = stringify(value, encoding_default=encoding)
     if text is None:
         return None
     try:
         text = compose_nfc(text)
+        if text is None:
+            return None
     except (SystemError, Exception) as ex:
         log.warning("Cannot NFC text: %s", ex)
         return None
@@ -71,16 +79,7 @@ def sanitize_text(text: Any, encoding: str = DEFAULT_ENCODING) -> Optional[str]:
     if text is None:
         return None
     byte_text = text.encode(DEFAULT_ENCODING, "replace")
-    return cast(str, byte_text.decode(DEFAULT_ENCODING, "replace"))
-
-
-def value_list(value: Union[T, Sequence[T]]) -> List[T]:
-    if not isinstance(value, (str, bytes)):
-        try:
-            return [v for v in cast(Sequence[T], value)]
-        except TypeError:
-            pass
-    return [cast(T, value)]
+    return byte_text.decode(DEFAULT_ENCODING, "replace")
 
 
 def key_bytes(key: Any) -> bytes:

followthemoney/value.py

@@ -0,0 +1,67 @@
+from typing import Any, Iterable, List, Mapping, Union
+from datetime import datetime, date, timezone
+import typing
+from prefixdate import DatePrefix
+
+from followthemoney.util import sanitize_text
+
+if typing.TYPE_CHECKING:
+    from followthemoney.proxy import EntityProxy
+
+Value = Union[str, int, float, bool, date, datetime, DatePrefix, None, "EntityProxy"]
+Values = Union[Value, Iterable[Value]]
+
+
+def string_list(value: Any, sanitize: bool = False) -> List[str]:
+    """Convert a value - which may be a list or set - to a list of strings."""
+    # This function is called in the inner loop of placing values into entities,
+    # so it's unrolled to avoid the overhead of a comparatively heavy ops like
+    # `isinstance`.
+    if value is None:
+        return []
+    type_ = type(value)
+    if type_ is str:
+        if sanitize:
+            value = sanitize_text(value)
+            if value is None:
+                return []
+        return [value] if len(value) > 0 else []
+    if type_ is int:
+        return [str(value)]
+    if type_ is float:
+        return [f"{value:.2f}"]
+    if type_ is bool:
+        return ["true" if value else "false"]
+    if type_ is date:
+        return [value.isoformat()]
+    if type_ is datetime:
+        if value.tzinfo is not None:
+            value = value.astimezone(tz=timezone.utc)
+        return [value.isoformat()]
+    if type_ is set or type_ is list or type_ is tuple:
+        texts: List[str] = []
+        for inner in value:
+            texts.extend(string_list(inner, sanitize=sanitize))
+        return texts
+    if isinstance(value, DatePrefix):
+        return [value.text] if value.text else []
+    # EntityProxy
+    try:
+        return string_list(value.id, sanitize=sanitize)
+    except AttributeError:
+        pass
+    # Entity dict
+    if isinstance(value, Mapping):
+        return string_list(value.get("id"), sanitize=sanitize)
+    if isinstance(value, (str, bytes)):
+        # Handle sub-classes of str, bytes - always sanitize
+        text = sanitize_text(value)
+        if text is None:
+            return []
+        return [text]
+    if isinstance(value, Iterable):
+        stexts: List[str] = []
+        for inner in value:
+            stexts.extend(string_list(inner, sanitize=sanitize))
+        return stexts
+    raise TypeError("Cannot convert %r to string list" % value)