followthemoney 3.8.4__py3-none-any.whl → 4.0.0__py3-none-any.whl

followthemoney/types/common.py

@@ -5,8 +5,8 @@ from banal import ensure_list
 from normality import stringify
 from typing import Any, Dict, Optional, Sequence, Callable, TYPE_CHECKING, TypedDict
 
-from followthemoney.rdf import Literal, Identifier
-from followthemoney.util import get_locale
+from followthemoney.value import Value
+from followthemoney.util import get_locale, const
 from followthemoney.util import gettext, sanitize_text
 
 if TYPE_CHECKING:
@@ -29,7 +29,7 @@ class PropertyTypeToDict(TypedDict, total=False):
 class PropertyType(object):
     """Base class for all property types."""
 
-    name: str = "any"
+    name: str = const("any")
     """A machine-facing, variable safe name for the given type."""
 
     group: Optional[str] = None
@@ -87,7 +87,7 @@ class PropertyType(object):
 
     def clean(
         self,
-        raw: Any,
+        raw: Value,
         fuzzy: bool = False,
         format: Optional[str] = None,
         proxy: Optional["EntityProxy"] = None,
@@ -165,11 +165,6 @@ class PropertyType(object):
         be related to (e.g. using a country prefix on a phone number or IBAN)."""
         return None
 
-    def rdf(self, value: str) -> Identifier:
-        """Return an RDF term to represent the given value - either a string
-        literal, or a URI reference."""
-        return Literal(value)
-
     def pick(self, values: Sequence[str]) -> Optional[str]:
         """Pick the best value to show to the user."""
         raise NotImplementedError
@@ -178,7 +173,7 @@ class PropertyType(object):
         """Return an ID suitable to identify this entity as a typed node in a
         graph representation of some FtM data. It's usually the same as the the
         RDF form."""
-        return str(self.rdf(value))
+        return f"{self.name}:{value}"
 
     def node_id_safe(self, value: Optional[str]) -> Optional[str]:
         """Wrapper for node_id to handle None values."""
@@ -186,7 +181,7 @@ class PropertyType(object):
             return None
         return self.node_id(value)
 
-    def caption(self, value: str) -> Optional[str]:
+    def caption(self, value: str, format: Optional[str] = None) -> str:
         """Return a label for the given property value. This is often the same as the
         value, but for types like countries or languages, it would return the label,
         while other values like phone numbers can be formatted to be nicer to read."""
@@ -253,19 +248,19 @@ class EnumType(PropertyType):
 
     def clean_text(
         self,
-        code: str,
+        text: str,
         fuzzy: bool = False,
         format: Optional[str] = None,
         proxy: Optional["EntityProxy"] = None,
     ) -> Optional[str]:
         """All code values are cleaned to be lowercase and trailing whitespace is
         removed."""
-        code = code.lower().strip()
+        code = text.lower().strip()
         if code not in self.codes:
             return None
         return code
 
-    def caption(self, value: str) -> str:
+    def caption(self, value: str, format: Optional[str] = None) -> str:
         """Given a code value, return the label that should be shown to a user."""
         return self.names.get(value, value)
 

followthemoney/types/country.py

@@ -3,9 +3,8 @@ from typing import Optional, TYPE_CHECKING
 from babel.core import Locale
 from rigour.territories import get_territory, get_ftm_countries
 
-from followthemoney.rdf import URIRef, Identifier
 from followthemoney.types.common import EnumType, EnumValues
-from followthemoney.util import defer as _
+from followthemoney.util import const, defer as _
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -17,8 +16,8 @@ class CountryType(EnumType):
     a number of unusual and controversial designations (e.g. the Soviet Union,
     Transnistria, Somaliland, Kosovo)."""
 
-    name = "country"
-    group = "countries"
+    name = const("country")
+    group = const("countries")
     label = _("Country")
     plural = _("Countries")
     matchable = True
@@ -52,6 +51,3 @@ class CountryType(EnumType):
 
     def country_hint(self, value: str) -> str:
         return value
-
-    def rdf(self, value: str) -> Identifier:
-        return URIRef(f"iso-3166:{value}")

followthemoney/types/date.py

@@ -4,9 +4,8 @@ from typing import Optional, TYPE_CHECKING
 from prefixdate import parse, parse_format, Precision
 
 from followthemoney.types.common import PropertyType
-from followthemoney.rdf import XSD, Literal, Identifier
 from followthemoney.util import defer as _
-from followthemoney.util import dampen
+from followthemoney.util import dampen, const
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -21,8 +20,8 @@ class DateType(PropertyType):
     The timezone is always expected to be UTC and cannot be specified otherwise. There is
     no support for calendar weeks (`2021-W7`) and date ranges (`2021-2024`)."""
 
-    name = "date"
-    group = "dates"
+    name = const("date")
+    group = const("dates")
     label = _("Date")
     plural = _("Dates")
     matchable = True
@@ -57,18 +56,29 @@ class DateType(PropertyType):
         prefix = os.path.commonprefix([left, right])
         return dampen(4, 10, prefix)
 
-    def rdf(self, value: str) -> Identifier:
-        if len(value) < Precision.HOUR.value:
-            return Literal(value, datatype=XSD.date)
-        return Literal(value, datatype=XSD.dateTime)
+    def to_datetime(self, value: str) -> Optional[datetime]:
+        """Convert a date string to a datetime object in UTC for handling in Python. This
+        will convert the unset fields beyond the prefix to the first possible value, e.g.
+        `2021-02` will become `2021-02-01T00:00:00Z`.
 
-    def node_id(self, value: str) -> str:
-        return f"date:{value}"
+        Args:
+            value (str): The date string to convert.
 
-    def to_datetime(self, value: str) -> Optional[datetime]:
+        Returns:
+            Optional[datetime]: The parsed datetime object in UTC, or None if parsing fails.
+        """
         return parse(value).dt
 
     def to_number(self, value: str) -> Optional[float]:
+        """Convert a date string to a number, which is the number of seconds since the epoch
+        (1970-01-01T00:00:00Z).
+
+        Args:
+            value (str): The date string to convert.
+
+        Returns:
+            Optional[float]: The timestamp as a float, or None if parsing fails.
+        """
         date = self.to_datetime(value)
         if date is None:
             return None

followthemoney/types/email.py

@@ -4,7 +4,6 @@ from typing import Optional, TYPE_CHECKING
 from urllib.parse import urlparse
 from normality.cleaning import strip_quotes
 
-from followthemoney.rdf import URIRef, Identifier
 from followthemoney.types.common import PropertyType
 from followthemoney.util import sanitize_text, defer as _
 
@@ -80,6 +79,3 @@ class EmailType(PropertyType):
 
     # def country_hint(self, value)
     # TODO: do we want to use TLDs as country evidence?
-
-    def rdf(self, value: str) -> Identifier:
-        return URIRef("mailto:%s" % value.lower())

followthemoney/types/entity.py

@@ -2,9 +2,9 @@ import re
 from typing import Any, Optional, TYPE_CHECKING
 
 from followthemoney.types.common import PropertyType
-from followthemoney.rdf import URIRef, Identifier
+from followthemoney.value import Value
 from followthemoney.util import ENTITY_ID_LEN, get_entity_id, sanitize_text
-from followthemoney.util import gettext, defer as _
+from followthemoney.util import const, gettext, defer as _
 from followthemoney.exc import InvalidData
 
 if TYPE_CHECKING:
@@ -22,8 +22,8 @@ class EntityType(PropertyType):
 
     REGEX_RAW = r"^[0-9a-zA-Z]([0-9a-zA-Z\.\-]*[0-9a-zA-Z])?$"
     REGEX = re.compile(REGEX_RAW)
-    name = "entity"
-    group = "entities"
+    name = const("entity")
+    group = const("entities")
     label = _("Entity")
     plural = _("Entities")
     matchable = True
@@ -31,7 +31,7 @@ class EntityType(PropertyType):
     max_length = ENTITY_ID_LEN
 
     def validate(
-        self, value: str, fuzzy: bool = False, format: Optional[str] = None
+        self, value: Value, fuzzy: bool = False, format: Optional[str] = None
     ) -> bool:
         text = sanitize_text(value)
         if text is None:
@@ -66,9 +66,3 @@ class EntityType(PropertyType):
         if self.REGEX.match(text) is not None:
             return text
         return None
-
-    def rdf(self, value: str) -> Identifier:
-        return URIRef(f"entity:{value}")
-
-    def caption(self, value: str) -> None:
-        return None

followthemoney/types/gender.py

@@ -2,8 +2,7 @@ from typing import Optional, TYPE_CHECKING
 from babel.core import Locale
 
 from followthemoney.types.common import EnumType, EnumValues
-from followthemoney.rdf import URIRef, Identifier
-from followthemoney.util import gettext, defer as _
+from followthemoney.util import const, gettext, defer as _
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -15,9 +14,9 @@ class GenderType(EnumType):
     government databases and represent it in a way that can be used by
     structured tools. I'm not sure this justifies the simplification."""
 
-    MALE = "male"
-    FEMALE = "female"
-    OTHER = "other"
+    MALE = const("male")
+    FEMALE = const("female")
+    OTHER = const("other")
 
     LOOKUP = {
         "m": MALE,
@@ -35,8 +34,8 @@ class GenderType(EnumType):
         "divers": OTHER,
     }
 
-    name = "gender"
-    group = "genders"
+    name = const("gender")
+    group = const("genders")
     label = _("Gender")
     plural = _("Genders")
     matchable = False
@@ -61,6 +60,3 @@ class GenderType(EnumType):
         if code not in self.codes:
             return None
         return code
-
-    def rdf(self, value: str) -> Identifier:
-        return URIRef(f"gender:{value}")

followthemoney/types/identifier.py

@@ -4,7 +4,7 @@ from rigour.ids import get_identifier_format_names, get_identifier_format
 
 from followthemoney.types.common import PropertyType
 from followthemoney.util import dampen, shortest, longest
-from followthemoney.util import defer as _
+from followthemoney.util import const, defer as _
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -20,8 +20,8 @@ class IdentifierType(PropertyType):
     Four- or five-digit industry classifiers create more noise than value."""
 
     COMPARE_CLEAN = re.compile(r"[\W_]+")
-    name = "identifier"
-    group = "identifiers"
+    name = const("identifier")
+    group = const("identifiers")
     label = _("Identifier")
     plural = _("Identifiers")
     matchable = True
@@ -59,3 +59,9 @@ class IdentifierType(PropertyType):
 
     def node_id(self, value: str) -> str:
         return f"id:{value}"
+
+    def caption(self, value: str, format: Optional[str] = None) -> str:
+        if format in get_identifier_format_names():
+            format_ = get_identifier_format(format)
+            return format_.format(value)
+        return value

followthemoney/types/ip.py

@@ -2,8 +2,7 @@ from typing import Optional, TYPE_CHECKING
 from ipaddress import ip_address
 
 from followthemoney.types.common import PropertyType
-from followthemoney.rdf import URIRef, Identifier
-from followthemoney.util import defer as _
+from followthemoney.util import const, defer as _
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -14,10 +13,10 @@ class IpType(PropertyType):
     by the protocol versions 4 (e.g. `192.168.1.143`) and 6
     (e.g. `0:0:0:0:0:ffff:c0a8:18f`)."""
 
-    name = "ip"
-    group = "ips"
-    label = _("IP-Address")
-    plural = _("IP-Addresses")
+    name = const("ip")
+    group = const("ips")
+    label = _("IP Address")
+    plural = _("IP Addresses")
     matchable = True
     pivot = True
     max_length = 64
@@ -45,6 +44,3 @@ class IpType(PropertyType):
             return str(ip_address(text))
         except ValueError:
             return None
-
-    def rdf(self, value: str) -> Identifier:
-        return URIRef(f"ip:{value}")

followthemoney/types/json.py

@@ -3,7 +3,7 @@ from typing import Any, Optional, Sequence, TYPE_CHECKING
 from banal import ensure_list
 
 from followthemoney.types.common import PropertyType
-from followthemoney.util import sanitize_text, defer as _
+from followthemoney.util import const, sanitize_text, defer as _
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -14,7 +14,7 @@ class JsonType(PropertyType):
     and some other edge cases. It's a really bad idea and we should try to get rid
     of JSON properties."""
 
-    name = "json"
+    name = const("json")
     group = None
     label = _("Nested data")
     plural = _("Nested data")

followthemoney/types/language.py

@@ -3,9 +3,8 @@ from babel.core import Locale
 from rigour.langs import iso_639_alpha3
 
 from followthemoney.types.common import EnumType, EnumValues
-from followthemoney.rdf import URIRef, Identifier
 from followthemoney.util import defer as _, gettext
-from followthemoney.util import get_env_list
+from followthemoney.util import const, get_env_list
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -17,8 +16,8 @@ class LanguageType(EnumType):
     for additional languages once there is a specific need for them to be
     supported."""
 
-    name = "language"
-    group = "languages"
+    name = const("language")
+    group = const("languages")
     label = _("Language")
     plural = _("Languages")
     matchable = False
@@ -120,6 +119,3 @@ class LanguageType(EnumType):
         if code not in self.LANGUAGES:
             return None
         return code
-
-    def rdf(self, value: str) -> Identifier:
-        return URIRef(f"iso-639:{value}")

followthemoney/types/mimetype.py

@@ -3,8 +3,7 @@ from rigour.mime import normalize_mimetype, parse_mimetype
 from rigour.mime import DEFAULT
 
 from followthemoney.types.common import PropertyType
-from followthemoney.rdf import URIRef, Identifier
-from followthemoney.util import defer as _
+from followthemoney.util import const, defer as _
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -19,8 +18,8 @@ class MimeType(PropertyType):
     MIME type properties do not contain parameters as used in HTTP headers,
     like `charset=UTF-8`."""
 
-    name = "mimetype"
-    group = "mimetypes"
+    name = const("mimetype")
+    group = const("mimetypes")
     label = _("MIME-Type")
     plural = _("MIME-Types")
     matchable = False
@@ -37,8 +36,5 @@ class MimeType(PropertyType):
             return text
         return None
 
-    def rdf(self, value: str) -> Identifier:
-        return URIRef(f"urn:mimetype:{value}")
-
-    def caption(self, value: str) -> str:
+    def caption(self, value: str, format: Optional[str] = None) -> str:
         return parse_mimetype(value).label or value

followthemoney/types/name.py

@@ -2,13 +2,12 @@ from typing import TYPE_CHECKING, Optional, Sequence
 from normality import slugify
 from normality.cleaning import collapse_spaces, strip_quotes
 from rigour.env import MAX_NAME_LENGTH
-from rigour.names import pick_name
+from rigour.names import pick_name, tokenize_name
 from rigour.text.distance import levenshtein_similarity
-from fingerprints.cleanup import clean_name_light
 
 from followthemoney.types.common import PropertyType
 from followthemoney.util import dampen
-from followthemoney.util import defer as _
+from followthemoney.util import const, defer as _
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -22,8 +21,8 @@ class NameType(PropertyType):
     No validation rules apply, and things having multiple names must be considered
     a perfectly ordinary case."""
 
-    name = "name"
-    group = "names"
+    name = const("name")
+    group = const("names")
     label = _("Name")
     plural = _("Names")
     matchable = True
@@ -51,9 +50,9 @@ class NameType(PropertyType):
 
     def compare(self, left: str, right: str) -> float:
         """Compare two names for similarity."""
-        left_clean = clean_name_light(left)
-        right_clean = clean_name_light(right)
-        if left_clean is None or right_clean is None:
+        left_clean = " ".join(tokenize_name(left.lower()))
+        right_clean = " ".join(tokenize_name(right.lower()))
+        if not len(left_clean) or not len(right_clean):
             return 0.0
         return levenshtein_similarity(
             left_clean,

followthemoney/types/number.py

@@ -1,8 +1,8 @@
 import re
-from typing import Optional
+from typing import Optional, Tuple
 
 from followthemoney.types.common import PropertyType
-from followthemoney.util import defer as _
+from followthemoney.util import const, defer as _
 
 
 class NumberType(PropertyType):
@@ -13,8 +13,18 @@ class NumberType(PropertyType):
     In the future we might want to enable annotations for format, units, or
     even to introduce a separate property type for monetary values."""
 
-    CAST_RE = re.compile(r"[^0-9\-\.]")
-    name = "number"
+    DECIMAL = "."
+    SEPARATOR = ","
+    PRECISION = 2
+
+    _NUM_UNIT_RE = (
+        f"(\\s?\\-?\\s?\\d+(?:{re.escape(DECIMAL)}\\d+)?)\\s*([^\\s\\d][^\\s]*)?"
+    )
+    NUM_UNIT_RE = re.compile(_NUM_UNIT_RE, re.UNICODE)
+    _FLOAT_FMT = "{:" + SEPARATOR + "." + str(PRECISION) + "f}"
+    _INT_FMT = "{:" + SEPARATOR + "d}"
+
+    name = const("number")
     label = _("Number")
     plural = _("Numbers")
     matchable = False
@@ -22,9 +32,81 @@ class NumberType(PropertyType):
     def node_id(self, value: str) -> None:
         return None
 
+    def parse(
+        self, value: str, decimal: str = DECIMAL, separator: str = SEPARATOR
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """Parse a number into a numeric value and a unit. The numeric value is
+        aligned with the decimal and separator settings. The unit is stripped of
+        whitespace and returned as a string. If no unit is found, None is
+        returned. If no number is found, None is returned for both values.
+
+        Args:
+            value (str): The string to parse.
+            decimal (str): The character used as the decimal separator.
+            separator (str): The character used to separate thousands, lakhs, or crores.
+
+        Returns:
+            A tuple of (number, unit), where number is a string and unit is a string or None.
+        """
+        value = value.replace(separator, "")
+        if decimal != self.DECIMAL:
+            value = value.replace(decimal, self.DECIMAL)
+        match = self.NUM_UNIT_RE.match(value)
+        if not match:
+            return None, None
+        number, unit = match.groups()
+        if unit is not None:
+            unit = unit.strip()
+            if len(unit) == 0:
+                unit = None
+        # TODO: We could have a lookup table for common units, e.g. kg, m, etc. to
+        # convert them to a standard form.
+        number = number.replace(" ", "")
+        if number == "":
+            number = None
+        return number, unit
+
     def to_number(self, value: str) -> Optional[float]:
+        """Convert a number string to a float. The string is parsed and the unit is
+        discarded if present.
+
+        Args:
+            value (str): The string to convert.
+
+        Returns:
+            Optional[float]: The parsed float value, or None if parsing fails.
+        """
         try:
-            value = self.CAST_RE.sub("", value)
-            return float(value)
+            number, _ = self.parse(value)
+            if number is None:
+                return None
+            return float(number)
         except Exception:
             return None
+
+    def caption(self, value: str, format: Optional[str] = None) -> str:
+        """Return a caption for the number. This is used for display purposes.
+
+        Args:
+            value (str): The string to format.
+            format (Optional[str]): An optional format string to use for formatting the number.
+
+        Returns:
+            str: The formatted number string, possibly with a unit.
+        """
+        number, unit = self.parse(value)
+        if number is None:
+            return value
+        try:
+            fnumber = float(number)
+        except ValueError:
+            return value
+        if format is not None:
+            number = format.format(fnumber)
+        elif fnumber.is_integer():
+            number = self._INT_FMT.format(int(fnumber))
+        else:
+            number = self._FLOAT_FMT.format(fnumber)
+        if unit is not None:
+            return f"{number} {unit}"
+        return number

followthemoney/types/phone.py

@@ -5,9 +5,8 @@ from phonenumbers import PhoneNumber, PhoneNumberFormat
 from phonenumbers.phonenumberutil import region_code_for_number, NumberParseException
 
 from followthemoney.types.common import PropertyType
-from followthemoney.rdf import URIRef, Identifier
 from followthemoney.util import defer as _
-from followthemoney.util import dampen
+from followthemoney.util import const, dampen
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -30,8 +29,8 @@ class PhoneType(PropertyType):
     validation outcome from doing the two operations the other way around. Always
     define the country first."""
 
-    name = "phone"
-    group = "phones"
+    name = const("phone")
+    group = const("phones")
     label = _("Phone number")
     plural = _("Phone numbers")
     matchable = True
@@ -97,16 +96,10 @@ class PhoneType(PropertyType):
         # TODO: insert artificial intelligence here.
         return dampen(7, 11, value)
 
-    def rdf(self, value: str) -> Identifier:
-        node_id = self.node_id(value)
-        if node_id is not None:
-            return URIRef(node_id)
-        raise ValueError("Invalid phone number for serialisation: %s" % value)
-
     def node_id(self, value: str) -> Optional[str]:
         return f"tel:{value}"
 
-    def caption(self, value: str) -> str:
+    def caption(self, value: str, format: Optional[str] = None) -> str:
         try:
             number = parse_number(value)
             formatted = format_number(number, PhoneNumberFormat.INTERNATIONAL)

followthemoney/types/string.py

@@ -1,12 +1,12 @@
 from followthemoney.types.common import PropertyType
-from followthemoney.util import defer as _
+from followthemoney.util import const, defer as _
 from followthemoney.util import MEGABYTE
 
 
 class StringType(PropertyType):
     """A simple string property with no additional semantics."""
 
-    name = "string"
+    name = const("string")
     label = _("Label")
     plural = _("Labels")
     matchable = False
@@ -21,7 +21,7 @@ class TextType(StringType):
     string properties, it might make sense to treat properties of this type as
     full-text search material."""
 
-    name = "text"
+    name = const("text")
     label = _("Text")
     plural = _("Texts")
     total_size = 30 * MEGABYTE
@@ -36,7 +36,7 @@ class HTMLType(StringType):
     to perform server-side sanitisation, or to not render this property at all.
     """
 
-    name = "html"
+    name = const("html")
     label = _("HTML")
     plural = _("HTMLs")
     total_size = 30 * MEGABYTE

followthemoney/types/topic.py

@@ -1,8 +1,7 @@
 from babel.core import Locale
 
 from followthemoney.types.common import EnumType, EnumValues
-from followthemoney.rdf import URIRef, Identifier
-from followthemoney.util import gettext, defer as _
+from followthemoney.util import const, gettext, defer as _
 
 
 class TopicType(EnumType):
@@ -16,8 +15,8 @@ class TopicType(EnumType):
     enable queries such as _find all paths between a government procurement
     award and a politician_."""
 
-    name = "topic"
-    group = "topics"
+    name = const("topic")
+    group = const("topics")
     label = _("Topic")
     plural = _("Topics")
     matchable = False
@@ -90,6 +89,3 @@ class TopicType(EnumType):
 
     def _locale_names(self, locale: Locale) -> EnumValues:
         return {k: gettext(v) for (k, v) in self._TOPICS.items()}
-
-    def rdf(self, value: str) -> Identifier:
-        return URIRef(f"ftm:topic:{value}")