followthemoney 1.3.7__py3-none-any.whl → 3.8.1__py3-none-any.whl

followthemoney/types/ip.py

@@ -1,36 +1,50 @@
-from rdflib import URIRef
-from normality import stringify
+from typing import Optional, TYPE_CHECKING
 from ipaddress import ip_address
 
 from followthemoney.types.common import PropertyType
+from followthemoney.rdf import URIRef, Identifier
+from followthemoney.util import defer as _
+
+if TYPE_CHECKING:
+    from followthemoney.proxy import EntityProxy
 
 
 class IpType(PropertyType):
-    name = 'ip'
-    group = 'ips'
-    prefix = 'ip'
-    strong = False
+    """Internet protocol addresses. This supports both addresses used
+    by the protocol versions 4 (e.g. `192.168.1.143`) and 6
+    (e.g. `0:0:0:0:0:ffff:c0a8:18f`)."""
+
+    name = "ip"
+    group = "ips"
+    label = _("IP-Address")
+    plural = _("IP-Addresses")
+    matchable = True
+    pivot = True
+    max_length = 64
 
-    def validate(self, ip, **kwargs):
+    def validate(
+        self, value: str, fuzzy: bool = False, format: Optional[str] = None
+    ) -> bool:
         """Check to see if this is a valid ip address."""
         try:
-            ip_address(ip)
+            ip_address(value)
             return True
         except ValueError:
             return False
 
-    def clean(self, text, **kwargs):
+    def clean_text(
+        self,
+        text: str,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        proxy: Optional["EntityProxy"] = None,
+    ) -> Optional[str]:
         """Create a more clean, but still user-facing version of an
         instance of the type."""
-        text = stringify(text)
-        if text is not None:
-            try:
-                return str(ip_address(text))
-            except ValueError:
-                return None
-
-    def specificity(self, value):
-        return 1
-
-    def rdf(self, value):
-        return URIRef('ip:%s' % value)
+        try:
+            return str(ip_address(text))
+        except ValueError:
+            return None
+
+    def rdf(self, value: str) -> Identifier:
+        return URIRef(f"ip:{value}")

followthemoney/types/json.py

@@ -0,0 +1,58 @@
+import json
+from typing import Any, Optional, Sequence, TYPE_CHECKING
+from banal import ensure_list
+
+from followthemoney.types.common import PropertyType
+from followthemoney.util import sanitize_text, defer as _
+
+if TYPE_CHECKING:
+    from followthemoney.proxy import EntityProxy
+
+
+class JsonType(PropertyType):
+    """An encoded JSON object. This is used to store raw HTTP headers for documents
+    and some other edge cases. It's a really bad idea and we should try to get rid
+    of JSON properties."""
+
+    name = "json"
+    group = None
+    label = _("Nested data")
+    plural = _("Nested data")
+    matchable = False
+
+    def pack(self, obj: Any) -> Optional[str]:
+        """Encode a given value to JSON."""
+        # TODO: use a JSON encoder that handles more types?
+        if obj is None:
+            return None
+        return json.dumps(obj)
+
+    def unpack(self, obj: str) -> Any:
+        """Decode a given JSON object."""
+        try:
+            return json.loads(obj)
+        except Exception:
+            return obj
+
+    def clean(
+        self,
+        raw: Any,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        proxy: Optional["EntityProxy"] = None,
+    ) -> Optional[str]:
+        if not isinstance(raw, str):
+            return self.pack(raw)
+        else:
+            return sanitize_text(raw)
+
+    def join(self, values: Sequence[str]) -> str:
+        """Turn multiple values into a JSON array."""
+        values = [self.unpack(v) for v in ensure_list(values)]
+        data = self.pack(values)
+        if data is None:
+            return "[]"
+        return data
+
+    def node_id(self, value: str) -> None:
+        return None

followthemoney/types/language.py

@@ -1,37 +1,124 @@
-from rdflib import URIRef
-from normality import stringify
-
-from followthemoney.types.common import PropertyType
-from followthemoney.util import get_locale
-
-
-class LanguageType(PropertyType):
-    name = 'language'
-    group = 'languages'
-    prefix = 'lang'
-
-    def __init__(self, *args):
-        self._names = {}
-
-    @property
-    def names(self):
-        locale = get_locale()
-        if locale not in self._names:
-            self._names[locale] = {}
-            for code, label in locale.languages.items():
-                self._names[locale][code.lower()] = label
-        return self._names[locale]
-
-    def validate(self, text, **kwargs):
-        text = stringify(text)
-        if text is None:
-            return False
-        return text.lower() in self.names
-
-    def clean_text(self, text, **kwargs):
-        code = text.lower().strip()
-        if code in self.names:
-            return code
-
-    def rdf(self, value):
-        return URIRef('iso-639:%s' % value)
+from typing import Optional, TYPE_CHECKING
+from babel.core import Locale
+from rigour.langs import iso_639_alpha3
+
+from followthemoney.types.common import EnumType, EnumValues
+from followthemoney.rdf import URIRef, Identifier
+from followthemoney.util import defer as _, gettext
+from followthemoney.util import get_env_list
+
+if TYPE_CHECKING:
+    from followthemoney.proxy import EntityProxy
+
+
+class LanguageType(EnumType):
+    """A human written language. This list is arbitrarily limited for some
+    weird upstream technical reasons, but we'll happily accept pull requests
+    for additional languages once there is a specific need for them to be
+    supported."""
+
+    name = "language"
+    group = "languages"
+    label = _("Language")
+    plural = _("Languages")
+    matchable = False
+    max_length = 16
+
+    # Language whitelist
+    LANGUAGES = [
+        "eng",
+        "fra",
+        "deu",
+        "rus",
+        "spa",
+        "nld",
+        "ron",
+        "kat",
+        "ara",
+        "tur",
+        "ltz",
+        "ell",
+        "lit",
+        "ukr",
+        "zho",
+        "bel",
+        "bul",
+        "bos",
+        "jpn",
+        "ces",
+        "lav",
+        "por",
+        "pol",
+        "hye",
+        "hrv",
+        "hin",
+        "heb",
+        "uzb",
+        "mon",
+        "urd",
+        "sqi",
+        "kor",
+        "isl",
+        "ita",
+        "est",
+        "nor",
+        "fas",
+        "swa",
+        "slv",
+        "slk",
+        "aze",
+        "tgk",
+        "kaz",
+        "tuk",
+        "kir",
+        "hun",
+        "dan",
+        "afr",
+        "swe",
+        "srp",
+        "ind",
+        "kan",
+        "mkd",
+        "mlt",
+        "msa",
+        "fin",
+        "cat",
+        "nep",
+        "tgl",
+        "fil",
+        "mya",
+        "khm",
+        "cnr",
+    ]
+    LANGUAGES = get_env_list("FTM_LANGUAGES", LANGUAGES)
+    LANGUAGES = [lang.lower().strip() for lang in LANGUAGES]
+
+    def _locale_names(self, locale: Locale) -> EnumValues:
+        names = {
+            "ara": gettext("Arabic"),
+            "nor": gettext("Norwegian"),
+            "cnr": gettext("Montenegrin"),
+        }
+        for lang in self.LANGUAGES:
+            if lang not in names:
+                names[lang] = lang
+        for code, label in locale.languages.items():
+            code = iso_639_alpha3(code)
+            if code in self.LANGUAGES and names[code] == code:
+                names[code] = label
+        return names
+
+    def clean_text(
+        self,
+        text: str,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        proxy: Optional["EntityProxy"] = None,
+    ) -> Optional[str]:
+        code = iso_639_alpha3(text)
+        if code not in self.LANGUAGES:
+            return None
+        return code
+
+    def rdf(self, value: str) -> Identifier:
+        return URIRef(f"iso-639:{value}")

followthemoney/types/mimetype.py

@@ -0,0 +1,44 @@
+from typing import Optional, TYPE_CHECKING
+from rigour.mime import normalize_mimetype, parse_mimetype
+from rigour.mime import DEFAULT
+
+from followthemoney.types.common import PropertyType
+from followthemoney.rdf import URIRef, Identifier
+from followthemoney.util import defer as _
+
+if TYPE_CHECKING:
+    from followthemoney.proxy import EntityProxy
+
+
+class MimeType(PropertyType):
+    """A MIME media type are a specification of a content type on a network.
+    Each MIME type is assigned by IANA and consists of two parts: the type
+    and sub-type. Common examples are: `text/plain`, `application/json` and
+    `application/pdf`.
+
+    MIME type properties do not contain parameters as used in HTTP headers,
+    like `charset=UTF-8`."""
+
+    name = "mimetype"
+    group = "mimetypes"
+    label = _("MIME-Type")
+    plural = _("MIME-Types")
+    matchable = False
+
+    def clean_text(
+        self,
+        text: str,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        proxy: Optional["EntityProxy"] = None,
+    ) -> Optional[str]:
+        text = normalize_mimetype(text)
+        if text != DEFAULT:
+            return text
+        return None
+
+    def rdf(self, value: str) -> Identifier:
+        return URIRef(f"urn:mimetype:{value}")
+
+    def caption(self, value: str) -> str:
+        return parse_mimetype(value).label or value

followthemoney/types/name.py

@@ -1,24 +1,68 @@
-from Levenshtein import jaro_winkler
+from typing import TYPE_CHECKING, Optional, Sequence
+from normality import slugify
 from normality.cleaning import collapse_spaces, strip_quotes
+from rigour.env import MAX_NAME_LENGTH
+from rigour.names import pick_name
+from rigour.text.distance import levenshtein_similarity
+from fingerprints.cleanup import clean_name_light
 
 from followthemoney.types.common import PropertyType
 from followthemoney.util import dampen
+from followthemoney.util import defer as _
+
+if TYPE_CHECKING:
+    from followthemoney.proxy import EntityProxy
 
 
 class NameType(PropertyType):
-    name = 'name'
-    group = 'names'
-    prefix = 'n'
+    """A name used for a person or company. This is assumed to be as complete
+    a name as available - when a first name, family name or patronymic are given
+    separately, these are stored to string-type properties instead.
+
+    No validation rules apply, and things having multiple names must be considered
+    a perfectly ordinary case."""
+
+    name = "name"
+    group = "names"
+    label = _("Name")
+    plural = _("Names")
+    matchable = True
+    pivot = True
+    max_length = MAX_NAME_LENGTH
 
-    def clean_text(self, name, **kwargs):
+    def clean_text(
+        self,
+        text: str,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        proxy: Optional["EntityProxy"] = None,
+    ) -> Optional[str]:
         """Basic clean-up."""
-        name = strip_quotes(name)
-        name = collapse_spaces(name)
-        return name
+        name = strip_quotes(text)
+        return collapse_spaces(name)
 
-    def specificity(self, value):
+    def pick(self, values: Sequence[str]) -> Optional[str]:
+        """From a set of names, pick the most plausible user-facing one."""
+        return pick_name(list(values))
+
+    def _specificity(self, value: str) -> float:
         # TODO: insert artificial intelligence here.
-        return dampen(3, 50, value) * .8
+        return dampen(3, 50, value)
+
+    def compare(self, left: str, right: str) -> float:
+        """Compare two names for similarity."""
+        left_clean = clean_name_light(left)
+        right_clean = clean_name_light(right)
+        if left_clean is None or right_clean is None:
+            return 0.0
+        return levenshtein_similarity(
+            left_clean,
+            right_clean,
+            max_length=self.max_length,
+        )
 
-    def compare(self, left, right):
-        return jaro_winkler(left, right)
+    def node_id(self, value: str) -> Optional[str]:
+        slug = slugify(value)
+        if slug is None:
+            return None
+        return f"name:{slug}"

followthemoney/types/number.py

@@ -0,0 +1,30 @@
+import re
+from typing import Optional
+
+from followthemoney.types.common import PropertyType
+from followthemoney.util import defer as _
+
+
+class NumberType(PropertyType):
+    """A numeric value, like the size of a piece of land, or the value of a
+    contract. Since all property values in FtM are strings, this is also a
+    string and there is no specified format (e.g. `1,000.00` vs. `1.000,00`).
+
+    In the future we might want to enable annotations for format, units, or
+    even to introduce a separate property type for monetary values."""
+
+    CAST_RE = re.compile(r"[^0-9\-\.]")
+    name = "number"
+    label = _("Number")
+    plural = _("Numbers")
+    matchable = False
+
+    def node_id(self, value: str) -> None:
+        return None
+
+    def to_number(self, value: str) -> Optional[float]:
+        try:
+            value = self.CAST_RE.sub("", value)
+            return float(value)
+        except Exception:
+            return None

followthemoney/types/phone.py

@@ -1,31 +1,54 @@
-from rdflib import URIRef
-from banal import ensure_list
-from phonenumbers import geocoder
+from typing import Iterable, Optional, TYPE_CHECKING
 from phonenumbers import parse as parse_number
-from phonenumbers import is_possible_number, is_valid_number, format_number
-from phonenumbers import PhoneNumberFormat
-from phonenumbers.phonenumberutil import NumberParseException
+from phonenumbers import is_valid_number, format_number
+from phonenumbers import PhoneNumber, PhoneNumberFormat
+from phonenumbers.phonenumberutil import region_code_for_number, NumberParseException
 
 from followthemoney.types.common import PropertyType
+from followthemoney.rdf import URIRef, Identifier
+from followthemoney.util import defer as _
+from followthemoney.util import dampen
+
+if TYPE_CHECKING:
+    from followthemoney.proxy import EntityProxy
+
+
+# TODO: for json schema export
+# https://stackoverflow.com/questions/6478875/regular-expression-matching-e-164-formatted-phone-numbers
 
 
 class PhoneType(PropertyType):
-    name = 'phone'
-    group = 'phones'
-    prefix = 'tel'
-    strong = False
-
-    def _clean_countries(self, countries, country):
-        result = set([None])
-        countries = ensure_list(countries)
-        countries.extend(ensure_list(country))
-        for country in countries:
-            if isinstance(country, str):
-                country = country.strip().upper()
-                result.add(country)
-        return result
-
-    def clean_text(self, number, countries=None, country=None, **kwargs):
+    """A phone number in E.164 format. This means that phone numbers always
+    include an international country prefix (e.g. `+38760183628`). The
+    cleaning and validation functions for this try to be smart about by
+    accepting a list of countries as an argument in order to add the number
+    prefix.
+
+    When adding a property of this type to an entity, any country-type properties
+    defined for the entity are considered for validation. That means that adding a
+    phone number to an entity before adding a country can have a different
+    validation outcome from doing the two operations the other way around. Always
+    define the country first."""
+
+    name = "phone"
+    group = "phones"
+    label = _("Phone number")
+    plural = _("Phone numbers")
+    matchable = True
+    pivot = True
+    max_length = 64
+
+    def _clean_countries(
+        self, proxy: Optional["EntityProxy"]
+    ) -> Iterable[Optional[str]]:
+        yield None
+        if proxy is not None:
+            for country in proxy.countries:
+                yield country.upper()
+
+    def _parse_number(
+        self, number: str, proxy: Optional["EntityProxy"] = None
+    ) -> Iterable[PhoneNumber]:
         """Parse a phone number and return in international format.
 
         If no valid phone number can be detected, None is returned. If
@@ -34,24 +57,59 @@ class PhoneType(PropertyType):
 
         https://github.com/daviddrysdale/python-phonenumbers
         """
-        for code in self._clean_countries(countries, country):
+        for code in self._clean_countries(proxy):
             try:
-                num = parse_number(number, code)
-                if is_possible_number(num):
-                    if is_valid_number(num):
-                        return format_number(num, PhoneNumberFormat.E164)
+                yield parse_number(number, code)
             except NumberParseException:
                 pass
 
-    def specificity(self, value):
-        return 1
+    def validate(
+        self, value: str, fuzzy: bool = False, format: Optional[str] = None
+    ) -> bool:
+        for num in self._parse_number(value):
+            if is_valid_number(num):
+                return True
+        return False
+
+    def clean_text(
+        self,
+        text: str,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        proxy: Optional["EntityProxy"] = None,
+    ) -> Optional[str]:
+        for num in self._parse_number(text, proxy=proxy):
+            if is_valid_number(num):
+                return str(format_number(num, PhoneNumberFormat.E164))
+        return None
 
-    def country_hint(self, value):
+    def country_hint(self, value: str) -> Optional[str]:
         try:
             number = parse_number(value)
-            return geocoder.region_code_for_number(number).lower()
+            code = region_code_for_number(number)
+            if code is None:
+                return None
+            return str(code).lower()
         except NumberParseException:
-                pass
+            return None
+
+    def _specificity(self, value: str) -> float:
+        # TODO: insert artificial intelligence here.
+        return dampen(7, 11, value)
+
+    def rdf(self, value: str) -> Identifier:
+        node_id = self.node_id(value)
+        if node_id is not None:
+            return URIRef(node_id)
+        raise ValueError("Invalid phone number for serialisation: %s" % value)
+
+    def node_id(self, value: str) -> Optional[str]:
+        return f"tel:{value}"
 
-    def rdf(self, value):
-        return URIRef('tel:%s' % value)
+    def caption(self, value: str) -> str:
+        try:
+            number = parse_number(value)
+            formatted = format_number(number, PhoneNumberFormat.INTERNATIONAL)
+            return str(formatted)
+        except NumberParseException:
+            return value

followthemoney/types/registry.py

@@ -0,0 +1,52 @@
+from banal import ensure_list
+from typing import Iterable, Set, Dict, Type, Union, List, Optional
+
+from followthemoney.types.common import PropertyType
+
+
+class Registry(object):
+    """This registry keeps the processing helpers for all property types
+    in the system. They are instantiated as singletons when the system is first
+    loaded. The registry can be used to get a type, which can itself then
+    clean, validate or format values of that type."""
+
+    def __init__(self) -> None:
+        self.named: Dict[str, PropertyType] = {}
+        self.matchable: Set[PropertyType] = set()
+        self.types: Set[PropertyType] = set()
+        self.groups: Dict[str, PropertyType] = {}
+        self.pivots: Set[PropertyType] = set()
+
+    def add(self, clazz: Type[PropertyType]) -> None:
+        """Add a singleton class."""
+        type_ = clazz()
+        self.named[clazz.name] = type_
+        self.types.add(type_)
+        if type_.matchable:
+            self.matchable.add(type_)
+        if type_.pivot:
+            self.pivots.add(type_)
+        if type_.group is not None:
+            self.groups[type_.group] = type_
+
+    def get(self, name: Union[str, PropertyType]) -> Optional[PropertyType]:
+        """For a given property type name, get its type object. This can also
+        be used via getattr, e.g. ``registry.phone``."""
+        # Allow transparent re-checking.
+        if isinstance(name, PropertyType):
+            return name
+        return self.named.get(name)
+
+    def get_types(
+        self, names: Iterable[Union[str, PropertyType]]
+    ) -> List[PropertyType]:
+        """Get a list of all type names."""
+        names = ensure_list(names)
+        types = [self.get(n) for n in names]
+        return [t for t in types if t is not None]
+
+    def __getitem__(self, name: str) -> PropertyType:
+        return self.named[name]
+
+    def __getattr__(self, name: str) -> PropertyType:
+        return self.named[name]