followthemoney 1.3.7__py3-none-any.whl → 3.8.0__py3-none-any.whl

followthemoney/types/country.py

@@ -1,63 +1,96 @@
 import countrynames
-from rdflib import URIRef
-from normality import stringify
-
-from followthemoney.types.common import PropertyType
-from followthemoney.util import gettext, get_locale
-
-
-class CountryType(PropertyType):
-    name = 'country'
-    group = 'countries'
-    prefix = 'country'
-
-    def __init__(self, *args):
-        self._names = {}
-        self.codes = self.names.keys()
-
-    @property
-    def names(self):
-        locale = get_locale()
-        if locale not in self._names:
-            # extra territories that OCCRP is interested in.
-            self._names[locale] = {
-                'zz': gettext('Global'),
-                'eu': gettext('European Union'),
-                'xk': gettext('Kosovo'),
-                'yucs': gettext('Yugoslavia'),
-                'csxx': gettext('Serbia and Montenegro'),
-                'suhh': gettext('Soviet Union'),
-                'ge-ab': gettext('Abkhazia'),
-                'x-so': gettext('South Ossetia'),
-                'so-som': gettext('Somaliland'),
-                'gb-wls': gettext('Wales'),
-                'gb-sct': gettext('Scotland'),
-                'md-pmr': gettext('Transnistria')
-            }
-            for code, label in locale.territories.items():
-                self._names[locale][code.lower()] = label
-        return self._names[locale]
-
-    def validate(self, country, **kwargs):
-        country = stringify(country)
-        if country is None:
-            return False
-        return country.lower() in self.codes
-
-    def clean_text(self, country, guess=False, **kwargs):
+from typing import Optional, TYPE_CHECKING
+from babel.core import Locale
+
+from followthemoney.rdf import URIRef, Identifier
+from followthemoney.types.common import EnumType, EnumValues
+from followthemoney.util import gettext, defer as _
+
+if TYPE_CHECKING:
+    from followthemoney.proxy import EntityProxy
+
+
+class CountryType(EnumType):
+    """Properties to define countries and territories. This is completely
+    descriptive and needs to deal with data from many origins, so we support
+    a number of unusual and controversial designations (e.g. the Soviet Union,
+    Transnistria, Somaliland, Kosovo)."""
+
+    name = "country"
+    group = "countries"
+    label = _("Country")
+    plural = _("Countries")
+    matchable = True
+    max_length = 16
+
+    def _locale_names(self, locale: Locale) -> EnumValues:
+        # extra territories that OCCRP is interested in.
+        names = {
+            "zz": gettext("Global"),
+            "eu": gettext("European Union"),
+            "un": gettext("United Nations"),
+            "zr": gettext("Zaire"),
+            # Overwrite "Czechia" label:
+            "cz": gettext("Czech Republic"),
+            "xk": gettext("Kosovo"),
+            "dd": gettext("East Germany"),
+            "yucs": gettext("Yugoslavia"),
+            "csxx": gettext("Serbia and Montenegro"),
+            "cshh": gettext("Czechoslovakia"),
+            "suhh": gettext("Soviet Union"),
+            "ge-ab": gettext("Abkhazia (Occupied Georgia)"),
+            "x-so": gettext("South Ossetia (Occupied Georgia)"),
+            "ua-lpr": gettext("Luhansk (Occupied Ukraine)"),
+            "ua-dpr": gettext("Donetsk (Occupied Ukraine)"),
+            "ua-cri": gettext("Crimea (Occupied Ukraine)"),
+            "so-som": gettext("Somaliland"),
+            "cy-trnc": gettext("Northern Cyprus"),
+            "az-nk": gettext("Nagorno-Karabakh"),
+            "iq-kr": gettext("Kurdistan"),
+            "cn-xz": gettext("Tibet"),
+            "cq": gettext("Sark"),
+            "gb-wls": gettext("Wales"),
+            "gb-sct": gettext("Scotland"),
+            "gb-nir": gettext("Northern Ireland"),
+            "md-pmr": gettext("Transnistria (PMR)"),
+            "pk-km": gettext("Kashmir"),
+        }
+        for code, label in locale.territories.items():
+            code = code.lower()
+            if code in names:
+                continue
+            try:
+                int(code)
+            except ValueError:
+                names[code] = label
+        # Remove some ISO-3611 codes that are not countries:
+        names.pop("xa", None)
+        names.pop("xb", None)
+        names.pop("qo", None)
+        names.pop("ea", None)
+        names.pop("ez", None)
+        return names
+
+    def clean_text(
+        self,
+        text: str,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        proxy: Optional["EntityProxy"] = None,
+    ) -> Optional[str]:
         """Determine a two-letter country code based on an input.
 
         The input may be a country code, a country name, etc.
         """
-        code = country.lower().strip()
-        if code in self.codes:
-            return code
-        country = countrynames.to_code(country, fuzzy=guess)
-        if country is not None:
-            return country.lower()
-
-    def country_hint(self, value):
+        code = countrynames.to_code(text, fuzzy=fuzzy)
+        if code is not None:
+            lower = code.lower()
+            if lower in self.codes:
+                return lower
+        return None
+
+    def country_hint(self, value: str) -> str:
         return value
 
-    def rdf(self, value):
-        return URIRef('iso-3166-1:%s' % value)
+    def rdf(self, value: str) -> Identifier:
+        return URIRef(f"iso-3166-1:{value}")

followthemoney/types/date.py

@@ -1,95 +1,78 @@
-import re
 import os
-import pytz
-from rdflib import Literal
-from rdflib.namespace import XSD
-from normality import stringify
-from datetime import datetime, date
+from datetime import datetime, timezone
+from typing import Optional, TYPE_CHECKING
+from prefixdate import parse, parse_format, Precision
 
 from followthemoney.types.common import PropertyType
+from followthemoney.rdf import XSD, Literal, Identifier
+from followthemoney.util import defer as _
 from followthemoney.util import dampen
 
+if TYPE_CHECKING:
+    from followthemoney.proxy import EntityProxy
 
-class DateType(PropertyType):
-    # JS: '^([12]\\d{3}(-[01]?[1-9](-[0123]?[1-9])?)?)?$'
-    DATE_RE = re.compile('^([12]\d{3}(-[01]?[0-9](-[0123]?[0-9]([T ]([012]?\d(:\d{1,2}(:\d{1,2}(\.\d{6})?(Z|[-+]\d{2}(:?\d{2})?)?)?)?)?)?)?)?)?$')  # noqa
-    DATE_FULL = re.compile('\d{4}-\d{2}-\d{2}.*')
-    CUT_ZEROES = re.compile(r'((\-00.*)|(.00:00:00))$')
-    MONTH_FORMATS = re.compile(r'(%b|%B|%m|%c|%x)')
-    DAY_FORMATS = re.compile(r'(%d|%w|%c|%x)')
-    MAX_LENGTH = 19
 
-    name = 'date'
-    group = 'dates'
-    prefix = 'date'
+class DateType(PropertyType):
+    """A date or time stamp. This is based on ISO 8601, but meant to allow for different
+    degrees of precision by specifying a prefix. This means that `2021`, `2021-02`,
+    `2021-02-16`, `2021-02-16T21`, `2021-02-16T21:48` and `2021-02-16T21:48:52`
+    are all valid values, with an implied precision.
 
-    def validate(self, obj, **kwargs):
-        """Check if a thing is a valid date."""
-        obj = stringify(obj)
-        if obj is None:
-            return False
-        return self.DATE_RE.match(obj) is not None
+    The timezone is always expected to be UTC and cannot be specified otherwise. There is
+    no support for calendar weeks (`2021-W7`) and date ranges (`2021-2024`)."""
 
-    def _clean_datetime(self, obj):
-        """Python objects want to be text."""
-        if isinstance(obj, datetime):
-            # if it's not naive, put it on zulu time first:
-            if obj.tzinfo is not None:
-                obj = obj.astimezone(pytz.utc)
-            return obj.isoformat()[:self.MAX_LENGTH]
-        if isinstance(obj, date):
-            return obj.isoformat()
+    name = "date"
+    group = "dates"
+    label = _("Date")
+    plural = _("Dates")
+    matchable = True
+    max_length = 32
 
-    def _clean_text(self, text):
-        # limit to the date part of a presumed date string
-        # FIXME: this may get us rid of TZ info?
-        text = text[:self.MAX_LENGTH]
-        if not self.validate(text):
-            return None
-        text = text.replace(' ', 'T')
-        # fix up dates like 2017-1-5 into 2017-01-05
-        if not self.DATE_FULL.match(text):
-            parts = text.split('T', 1)
-            date = [p.zfill(2) for p in parts[0].split('-')]
-            parts[0] = '-'.join(date)
-            text = 'T'.join(parts)
-            text = text[:self.MAX_LENGTH]
-        # strip -00-00 from dates because it makes ES barf.
-        text = self.CUT_ZEROES.sub('', text)
-        return text
+    def validate(
+        self, value: str, fuzzy: bool = False, format: Optional[str] = None
+    ) -> bool:
+        """Check if a thing is a valid date."""
+        if format is not None:
+            prefix = parse_format(value, format)
+        else:
+            prefix = parse(value)
+        return prefix.precision != Precision.EMPTY
 
-    def clean(self, text, format=None, **kwargs):
+    def clean_text(
+        self,
+        text: str,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        proxy: Optional["EntityProxy"] = None,
+    ) -> Optional[str]:
         """The classic: date parsing, every which way."""
-        # handle date/datetime before converting to text.
-        date = self._clean_datetime(text)
-        if date is not None:
-            return date
-
-        text = stringify(text)
-        if text is None:
-            return
-
         if format is not None:
-            # parse with a specified format
-            try:
-                obj = datetime.strptime(text, format)
-                text = obj.date().isoformat()
-                if self.MONTH_FORMATS.search(format) is None:
-                    text = text[:4]
-                elif self.DAY_FORMATS.search(format) is None:
-                    text = text[:7]
-                return text
-            except Exception:
-                return None
-
-        return self._clean_text(text)
+            return parse_format(text, format).text
+        return parse(text).text
 
-    def specificity(self, value):
-        return dampen(5, self.MAX_LENGTH, value) * .5
+    def _specificity(self, value: str) -> float:
+        return dampen(5, 13, value)
 
-    def compare(self, left, right):
+    def compare(self, left: str, right: str) -> float:
         prefix = os.path.commonprefix([left, right])
         return dampen(4, 10, prefix)
 
-    def rdf(self, value):
+    def rdf(self, value: str) -> Identifier:
+        if len(value) < Precision.HOUR.value:
+            return Literal(value, datatype=XSD.date)
         return Literal(value, datatype=XSD.dateTime)
+
+    def node_id(self, value: str) -> str:
+        return f"date:{value}"
+
+    def to_datetime(self, value: str) -> Optional[datetime]:
+        return parse(value).dt
+
+    def to_number(self, value: str) -> Optional[float]:
+        date = self.to_datetime(value)
+        if date is None:
+            return None
+        # We make a best effort all over the app to ensure all times are in UTC.
+        if date.tzinfo is None:
+            date = date.replace(tzinfo=timezone.utc)
+        return date.timestamp()

followthemoney/types/email.py

@@ -1,54 +1,85 @@
 import re
-from rdflib import URIRef
-from normality import stringify
+import logging
+from typing import Optional, TYPE_CHECKING
+from urllib.parse import urlparse
 from normality.cleaning import strip_quotes
 
+from followthemoney.rdf import URIRef, Identifier
 from followthemoney.types.common import PropertyType
-from followthemoney.types.domain import DomainType
+from followthemoney.util import sanitize_text, defer as _
+
+log = logging.getLogger(__name__)
+
+if TYPE_CHECKING:
+    from followthemoney.proxy import EntityProxy
 
 
 class EmailType(PropertyType):
-    EMAIL_REGEX = re.compile(r"[^@]+@[^@]+\.[^@]+")
-    domains = DomainType()
-    name = 'email'
-    group = 'emails'
-    prefix = 'mail'
-    strong = True
-
-    def validate(self, email, **kwargs):
+    """Internet mail address (e.g. user@example.com). These are notoriously hard
+    to validate, but we use an irresponsibly simple rule and hope for the best."""
+
+    REGEX_RAW = r"^[^@\s]+@[^@\s]+\.\w+$"
+    REGEX = re.compile(REGEX_RAW)
+    name = "email"
+    group = "emails"
+    label = _("E-Mail Address")
+    plural = _("E-Mail Addresses")
+    matchable = True
+    pivot = True
+
+    # def _check_exists(self, domain):
+    #     """Actually try to resolve a domain name."""
+    #     try:
+    #         domain = domain.encode('idna').lower()
+    #         socket.getaddrinfo(domain, None)
+    #         return True
+    #     except:
+    #         return False
+
+    def validate(
+        self, value: str, fuzzy: bool = False, format: Optional[str] = None
+    ) -> bool:
         """Check to see if this is a valid email address."""
-        email = stringify(email)
-        if email is None:
-            return
-        if not self.EMAIL_REGEX.match(email):
+        # TODO: adopt email.utils.parseaddr
+        email = sanitize_text(value)
+        if email is None or not self.REGEX.match(email):
             return False
-        mailbox, domain = email.rsplit('@', 1)
-        return self.domains.validate(domain, **kwargs)
+        _, domain = email.rsplit("@", 1)
+        if len(domain) < 4 or "." not in domain:
+            return False
+        return True
 
-    def clean_text(self, email, **kwargs):
+    def clean_text(
+        self,
+        text: str,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        proxy: Optional["EntityProxy"] = None,
+    ) -> Optional[str]:
         """Parse and normalize an email address.
 
         Returns None if this is not an email address.
         """
-        if not self.EMAIL_REGEX.match(email):
+        email = strip_quotes(text)
+        if email is None or not self.REGEX.match(email):
             return None
-        email = strip_quotes(email)
-        mailbox, domain = email.rsplit('@', 1)
-        domain = self.domains.clean(domain, **kwargs)
-        if domain is None or mailbox is None:
-            return
-        return '@'.join((mailbox, domain))
-
-    def normalize(self, email, **kwargs):
-        """Normalize for comparison."""
-        emails = super(EmailType, self).normalize(email, **kwargs)
-        return [e.lower() for e in emails]
-
-    def specificity(self, value):
-        return 0 if value is None else 1
+        mailbox, domain = email.rsplit("@", 1)
+        # TODO: https://pypi.python.org/pypi/publicsuffix/
+        # handle URLs by extracting the domain name
+        domain = urlparse(domain).hostname or domain
+        domain = domain.lower()
+        domain = domain.rstrip(".")
+        # handle unicode
+        try:
+            domain = domain.encode("idna").decode("ascii")
+        except UnicodeError:
+            return None
+        if domain is not None and mailbox is not None:
+            return "@".join((mailbox, domain))
+        return None
 
     # def country_hint(self, value)
     # TODO: do we want to use TLDs as country evidence?
 
-    def rdf(self, value):
-        return URIRef('mailto:%s' % value)
+    def rdf(self, value: str) -> Identifier:
+        return URIRef("mailto:%s" % value.lower())

followthemoney/types/entity.py

@@ -1,21 +1,74 @@
-from rdflib import URIRef
-from banal import is_mapping
+import re
+from typing import Any, Optional, TYPE_CHECKING
 
 from followthemoney.types.common import PropertyType
+from followthemoney.rdf import URIRef, Identifier
+from followthemoney.util import ENTITY_ID_LEN, get_entity_id, sanitize_text
+from followthemoney.util import gettext, defer as _
+from followthemoney.exc import InvalidData
+
+if TYPE_CHECKING:
+    from followthemoney.proxy import EntityProxy
 
 
 class EntityType(PropertyType):
-    name = 'entity'
-    group = 'entities'
-    prefix = 'e'
+    """A reference to another entity via its ID. This is how entities in FtM
+    become a graph: by pointing at each other using references.
+
+    Entity IDs can either be `namespaced` or `plain`, depending on the context.
+    When setting properties of this type, you can pass in an entity proxy or
+    dict of the entity, the ID will then be extracted and stored.
+    """
+
+    REGEX_RAW = r"^[0-9a-zA-Z]([0-9a-zA-Z\.\-]*[0-9a-zA-Z])?$"
+    REGEX = re.compile(REGEX_RAW)
+    name = "entity"
+    group = "entities"
+    label = _("Entity")
+    plural = _("Entities")
+    matchable = True
+    pivot = True
+    max_length = ENTITY_ID_LEN
+
+    def validate(
+        self, value: str, fuzzy: bool = False, format: Optional[str] = None
+    ) -> bool:
+        text = sanitize_text(value)
+        if text is None:
+            return False
+        return self.REGEX.match(text) is not None
+
+    def clean(
+        self,
+        raw: Any,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        proxy: Optional["EntityProxy"] = None,
+    ) -> Optional[str]:
+        entity_id = get_entity_id(raw)
+        if entity_id is None:
+            return None
+        return self.clean_text(entity_id, fuzzy=fuzzy, format=format, proxy=proxy)
 
-    def clean(self, text, **kwargs):
-        if is_mapping(text):
-            text = text.get('id')
-        return super(EntityType, self).clean(text, **kwargs)
+    def clean_text(
+        self,
+        text: str,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        proxy: Optional["EntityProxy"] = None,
+    ) -> Optional[str]:
+        """Specific types can apply their own cleaning routines here (this is called
+        by ``clean`` after the value has been converted to a string and null values
+        have been filtered)."""
+        if proxy is not None and text == proxy.id:
+            msg = gettext("Self-relationship (%s): %s")
+            raise InvalidData(msg % (proxy.schema, text))
+        if self.REGEX.match(text) is not None:
+            return text
+        return None
 
-    def specificity(self, value):
-        return 1
+    def rdf(self, value: str) -> Identifier:
+        return URIRef(f"entity:{value}")
 
-    def rdf(self, value):
-        return URIRef('urn:entity:%s' % value)
+    def caption(self, value: str) -> None:
+        return None

followthemoney/types/gender.py

@@ -0,0 +1,66 @@
+from typing import Optional, TYPE_CHECKING
+from babel.core import Locale
+
+from followthemoney.types.common import EnumType, EnumValues
+from followthemoney.rdf import URIRef, Identifier
+from followthemoney.util import gettext, defer as _
+
+if TYPE_CHECKING:
+    from followthemoney.proxy import EntityProxy
+
+
+class GenderType(EnumType):
+    """A human gender. This is not meant to be a comprehensive model of
+    the social realities of gender but a way to capture data from (mostly)
+    government databases and represent it in a way that can be used by
+    structured tools. I'm not sure this justifies the simplification."""
+
+    MALE = "male"
+    FEMALE = "female"
+    OTHER = "other"
+
+    LOOKUP = {
+        "m": MALE,
+        "man": MALE,
+        "masculin": MALE,
+        "männlich": MALE,
+        "мужской": MALE,
+        "f": FEMALE,
+        "woman": FEMALE,
+        "féminin": FEMALE,
+        "weiblich": FEMALE,
+        "женский": FEMALE,
+        "o": OTHER,
+        "d": OTHER,
+        "divers": OTHER,
+    }
+
+    name = "gender"
+    group = "genders"
+    label = _("Gender")
+    plural = _("Genders")
+    matchable = False
+    max_length = 16
+
+    def _locale_names(self, locale: Locale) -> EnumValues:
+        return {
+            self.MALE: gettext("male"),
+            self.FEMALE: gettext("female"),
+            self.OTHER: gettext("other"),
+        }
+
+    def clean_text(
+        self,
+        text: str,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        proxy: Optional["EntityProxy"] = None,
+    ) -> Optional[str]:
+        code = text.lower().strip()
+        code = self.LOOKUP.get(code, code)
+        if code not in self.codes:
+            return None
+        return code
+
+    def rdf(self, value: str) -> Identifier:
+        return URIRef(f"gender:{value}")