followthemoney 1.3.7__py3-none-any.whl → 3.8.1__py3-none-any.whl

followthemoney/types/date.py

@@ -1,95 +1,78 @@
-import re
 import os
-import pytz
-from rdflib import Literal
-from rdflib.namespace import XSD
-from normality import stringify
-from datetime import datetime, date
+from datetime import datetime, timezone
+from typing import Optional, TYPE_CHECKING
+from prefixdate import parse, parse_format, Precision
 
 from followthemoney.types.common import PropertyType
+from followthemoney.rdf import XSD, Literal, Identifier
+from followthemoney.util import defer as _
 from followthemoney.util import dampen
 
+if TYPE_CHECKING:
+    from followthemoney.proxy import EntityProxy
 
-class DateType(PropertyType):
-    # JS: '^([12]\\d{3}(-[01]?[1-9](-[0123]?[1-9])?)?)?$'
-    DATE_RE = re.compile('^([12]\d{3}(-[01]?[0-9](-[0123]?[0-9]([T ]([012]?\d(:\d{1,2}(:\d{1,2}(\.\d{6})?(Z|[-+]\d{2}(:?\d{2})?)?)?)?)?)?)?)?)?$')  # noqa
-    DATE_FULL = re.compile('\d{4}-\d{2}-\d{2}.*')
-    CUT_ZEROES = re.compile(r'((\-00.*)|(.00:00:00))$')
-    MONTH_FORMATS = re.compile(r'(%b|%B|%m|%c|%x)')
-    DAY_FORMATS = re.compile(r'(%d|%w|%c|%x)')
-    MAX_LENGTH = 19
 
-    name = 'date'
-    group = 'dates'
-    prefix = 'date'
+class DateType(PropertyType):
+    """A date or time stamp. This is based on ISO 8601, but meant to allow for different
+    degrees of precision by specifying a prefix. This means that `2021`, `2021-02`,
+    `2021-02-16`, `2021-02-16T21`, `2021-02-16T21:48` and `2021-02-16T21:48:52`
+    are all valid values, with an implied precision.
 
-    def validate(self, obj, **kwargs):
-        """Check if a thing is a valid date."""
-        obj = stringify(obj)
-        if obj is None:
-            return False
-        return self.DATE_RE.match(obj) is not None
+    The timezone is always expected to be UTC and cannot be specified otherwise. There is
+    no support for calendar weeks (`2021-W7`) and date ranges (`2021-2024`)."""
 
-    def _clean_datetime(self, obj):
-        """Python objects want to be text."""
-        if isinstance(obj, datetime):
-            # if it's not naive, put it on zulu time first:
-            if obj.tzinfo is not None:
-                obj = obj.astimezone(pytz.utc)
-            return obj.isoformat()[:self.MAX_LENGTH]
-        if isinstance(obj, date):
-            return obj.isoformat()
+    name = "date"
+    group = "dates"
+    label = _("Date")
+    plural = _("Dates")
+    matchable = True
+    max_length = 32
 
-    def _clean_text(self, text):
-        # limit to the date part of a presumed date string
-        # FIXME: this may get us rid of TZ info?
-        text = text[:self.MAX_LENGTH]
-        if not self.validate(text):
-            return None
-        text = text.replace(' ', 'T')
-        # fix up dates like 2017-1-5 into 2017-01-05
-        if not self.DATE_FULL.match(text):
-            parts = text.split('T', 1)
-            date = [p.zfill(2) for p in parts[0].split('-')]
-            parts[0] = '-'.join(date)
-            text = 'T'.join(parts)
-            text = text[:self.MAX_LENGTH]
-        # strip -00-00 from dates because it makes ES barf.
-        text = self.CUT_ZEROES.sub('', text)
-        return text
+    def validate(
+        self, value: str, fuzzy: bool = False, format: Optional[str] = None
+    ) -> bool:
+        """Check if a thing is a valid date."""
+        if format is not None:
+            prefix = parse_format(value, format)
+        else:
+            prefix = parse(value)
+        return prefix.precision != Precision.EMPTY
 
-    def clean(self, text, format=None, **kwargs):
+    def clean_text(
+        self,
+        text: str,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        proxy: Optional["EntityProxy"] = None,
+    ) -> Optional[str]:
         """The classic: date parsing, every which way."""
-        # handle date/datetime before converting to text.
-        date = self._clean_datetime(text)
-        if date is not None:
-            return date
-
-        text = stringify(text)
-        if text is None:
-            return
-
         if format is not None:
-            # parse with a specified format
-            try:
-                obj = datetime.strptime(text, format)
-                text = obj.date().isoformat()
-                if self.MONTH_FORMATS.search(format) is None:
-                    text = text[:4]
-                elif self.DAY_FORMATS.search(format) is None:
-                    text = text[:7]
-                return text
-            except Exception:
-                return None
-
-        return self._clean_text(text)
+            return parse_format(text, format).text
+        return parse(text).text
 
-    def specificity(self, value):
-        return dampen(5, self.MAX_LENGTH, value) * .5
+    def _specificity(self, value: str) -> float:
+        return dampen(5, 13, value)
 
-    def compare(self, left, right):
+    def compare(self, left: str, right: str) -> float:
         prefix = os.path.commonprefix([left, right])
         return dampen(4, 10, prefix)
 
-    def rdf(self, value):
+    def rdf(self, value: str) -> Identifier:
+        if len(value) < Precision.HOUR.value:
+            return Literal(value, datatype=XSD.date)
         return Literal(value, datatype=XSD.dateTime)
+
+    def node_id(self, value: str) -> str:
+        return f"date:{value}"
+
+    def to_datetime(self, value: str) -> Optional[datetime]:
+        return parse(value).dt
+
+    def to_number(self, value: str) -> Optional[float]:
+        date = self.to_datetime(value)
+        if date is None:
+            return None
+        # We make a best effort all over the app to ensure all times are in UTC.
+        if date.tzinfo is None:
+            date = date.replace(tzinfo=timezone.utc)
+        return date.timestamp()

followthemoney/types/email.py

@@ -1,54 +1,85 @@
 import re
-from rdflib import URIRef
-from normality import stringify
+import logging
+from typing import Optional, TYPE_CHECKING
+from urllib.parse import urlparse
 from normality.cleaning import strip_quotes
 
+from followthemoney.rdf import URIRef, Identifier
 from followthemoney.types.common import PropertyType
-from followthemoney.types.domain import DomainType
+from followthemoney.util import sanitize_text, defer as _
+
+log = logging.getLogger(__name__)
+
+if TYPE_CHECKING:
+    from followthemoney.proxy import EntityProxy
 
 
 class EmailType(PropertyType):
-    EMAIL_REGEX = re.compile(r"[^@]+@[^@]+\.[^@]+")
-    domains = DomainType()
-    name = 'email'
-    group = 'emails'
-    prefix = 'mail'
-    strong = True
-
-    def validate(self, email, **kwargs):
+    """Internet mail address (e.g. user@example.com). These are notoriously hard
+    to validate, but we use an irresponsibly simple rule and hope for the best."""
+
+    REGEX_RAW = r"^[^@\s]+@[^@\s]+\.\w+$"
+    REGEX = re.compile(REGEX_RAW)
+    name = "email"
+    group = "emails"
+    label = _("E-Mail Address")
+    plural = _("E-Mail Addresses")
+    matchable = True
+    pivot = True
+
+    # def _check_exists(self, domain):
+    #     """Actually try to resolve a domain name."""
+    #     try:
+    #         domain = domain.encode('idna').lower()
+    #         socket.getaddrinfo(domain, None)
+    #         return True
+    #     except:
+    #         return False
+
+    def validate(
+        self, value: str, fuzzy: bool = False, format: Optional[str] = None
+    ) -> bool:
         """Check to see if this is a valid email address."""
-        email = stringify(email)
-        if email is None:
-            return
-        if not self.EMAIL_REGEX.match(email):
+        # TODO: adopt email.utils.parseaddr
+        email = sanitize_text(value)
+        if email is None or not self.REGEX.match(email):
             return False
-        mailbox, domain = email.rsplit('@', 1)
-        return self.domains.validate(domain, **kwargs)
+        _, domain = email.rsplit("@", 1)
+        if len(domain) < 4 or "." not in domain:
+            return False
+        return True
 
-    def clean_text(self, email, **kwargs):
+    def clean_text(
+        self,
+        text: str,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        proxy: Optional["EntityProxy"] = None,
+    ) -> Optional[str]:
         """Parse and normalize an email address.
 
         Returns None if this is not an email address.
         """
-        if not self.EMAIL_REGEX.match(email):
+        email = strip_quotes(text)
+        if email is None or not self.REGEX.match(email):
             return None
-        email = strip_quotes(email)
-        mailbox, domain = email.rsplit('@', 1)
-        domain = self.domains.clean(domain, **kwargs)
-        if domain is None or mailbox is None:
-            return
-        return '@'.join((mailbox, domain))
-
-    def normalize(self, email, **kwargs):
-        """Normalize for comparison."""
-        emails = super(EmailType, self).normalize(email, **kwargs)
-        return [e.lower() for e in emails]
-
-    def specificity(self, value):
-        return 0 if value is None else 1
+        mailbox, domain = email.rsplit("@", 1)
+        # TODO: https://pypi.python.org/pypi/publicsuffix/
+        # handle URLs by extracting the domain name
+        domain = urlparse(domain).hostname or domain
+        domain = domain.lower()
+        domain = domain.rstrip(".")
+        # handle unicode
+        try:
+            domain = domain.encode("idna").decode("ascii")
+        except UnicodeError:
+            return None
+        if domain is not None and mailbox is not None:
+            return "@".join((mailbox, domain))
+        return None
 
     # def country_hint(self, value)
     # TODO: do we want to use TLDs as country evidence?
 
-    def rdf(self, value):
-        return URIRef('mailto:%s' % value)
+    def rdf(self, value: str) -> Identifier:
+        return URIRef("mailto:%s" % value.lower())

followthemoney/types/entity.py

@@ -1,21 +1,74 @@
-from rdflib import URIRef
-from banal import is_mapping
+import re
+from typing import Any, Optional, TYPE_CHECKING
 
 from followthemoney.types.common import PropertyType
+from followthemoney.rdf import URIRef, Identifier
+from followthemoney.util import ENTITY_ID_LEN, get_entity_id, sanitize_text
+from followthemoney.util import gettext, defer as _
+from followthemoney.exc import InvalidData
+
+if TYPE_CHECKING:
+    from followthemoney.proxy import EntityProxy
 
 
 class EntityType(PropertyType):
-    name = 'entity'
-    group = 'entities'
-    prefix = 'e'
+    """A reference to another entity via its ID. This is how entities in FtM
+    become a graph: by pointing at each other using references.
+
+    Entity IDs can either be `namespaced` or `plain`, depending on the context.
+    When setting properties of this type, you can pass in an entity proxy or
+    dict of the entity, the ID will then be extracted and stored.
+    """
+
+    REGEX_RAW = r"^[0-9a-zA-Z]([0-9a-zA-Z\.\-]*[0-9a-zA-Z])?$"
+    REGEX = re.compile(REGEX_RAW)
+    name = "entity"
+    group = "entities"
+    label = _("Entity")
+    plural = _("Entities")
+    matchable = True
+    pivot = True
+    max_length = ENTITY_ID_LEN
+
+    def validate(
+        self, value: str, fuzzy: bool = False, format: Optional[str] = None
+    ) -> bool:
+        text = sanitize_text(value)
+        if text is None:
+            return False
+        return self.REGEX.match(text) is not None
+
+    def clean(
+        self,
+        raw: Any,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        proxy: Optional["EntityProxy"] = None,
+    ) -> Optional[str]:
+        entity_id = get_entity_id(raw)
+        if entity_id is None:
+            return None
+        return self.clean_text(entity_id, fuzzy=fuzzy, format=format, proxy=proxy)
 
-    def clean(self, text, **kwargs):
-        if is_mapping(text):
-            text = text.get('id')
-        return super(EntityType, self).clean(text, **kwargs)
+    def clean_text(
+        self,
+        text: str,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        proxy: Optional["EntityProxy"] = None,
+    ) -> Optional[str]:
+        """Specific types can apply their own cleaning routines here (this is called
+        by ``clean`` after the value has been converted to a string and null values
+        have been filtered)."""
+        if proxy is not None and text == proxy.id:
+            msg = gettext("Self-relationship (%s): %s")
+            raise InvalidData(msg % (proxy.schema, text))
+        if self.REGEX.match(text) is not None:
+            return text
+        return None
 
-    def specificity(self, value):
-        return 1
+    def rdf(self, value: str) -> Identifier:
+        return URIRef(f"entity:{value}")
 
-    def rdf(self, value):
-        return URIRef('urn:entity:%s' % value)
+    def caption(self, value: str) -> None:
+        return None

followthemoney/types/gender.py

@@ -0,0 +1,66 @@
+from typing import Optional, TYPE_CHECKING
+from babel.core import Locale
+
+from followthemoney.types.common import EnumType, EnumValues
+from followthemoney.rdf import URIRef, Identifier
+from followthemoney.util import gettext, defer as _
+
+if TYPE_CHECKING:
+    from followthemoney.proxy import EntityProxy
+
+
+class GenderType(EnumType):
+    """A human gender. This is not meant to be a comprehensive model of
+    the social realities of gender but a way to capture data from (mostly)
+    government databases and represent it in a way that can be used by
+    structured tools. I'm not sure this justifies the simplification."""
+
+    MALE = "male"
+    FEMALE = "female"
+    OTHER = "other"
+
+    LOOKUP = {
+        "m": MALE,
+        "man": MALE,
+        "masculin": MALE,
+        "männlich": MALE,
+        "мужской": MALE,
+        "f": FEMALE,
+        "woman": FEMALE,
+        "féminin": FEMALE,
+        "weiblich": FEMALE,
+        "женский": FEMALE,
+        "o": OTHER,
+        "d": OTHER,
+        "divers": OTHER,
+    }
+
+    name = "gender"
+    group = "genders"
+    label = _("Gender")
+    plural = _("Genders")
+    matchable = False
+    max_length = 16
+
+    def _locale_names(self, locale: Locale) -> EnumValues:
+        return {
+            self.MALE: gettext("male"),
+            self.FEMALE: gettext("female"),
+            self.OTHER: gettext("other"),
+        }
+
+    def clean_text(
+        self,
+        text: str,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        proxy: Optional["EntityProxy"] = None,
+    ) -> Optional[str]:
+        code = text.lower().strip()
+        code = self.LOOKUP.get(code, code)
+        if code not in self.codes:
+            return None
+        return code
+
+    def rdf(self, value: str) -> Identifier:
+        return URIRef(f"gender:{value}")

followthemoney/types/iban.py

@@ -1,39 +1,58 @@
-from rdflib import URIRef
-from normality import stringify
-from schwifty import IBAN
+from typing import Optional, TYPE_CHECKING
+from rigour.ids import IBAN
 
 from followthemoney.types.common import PropertyType
+from followthemoney.rdf import URIRef, Identifier
+from followthemoney.util import sanitize_text, defer as _
+
+if TYPE_CHECKING:
+    from followthemoney.proxy import EntityProxy
 
 
 class IbanType(PropertyType):
-    name = 'iban'
-    group = 'ibans'
-    prefix = 'iban'
-    strong = False
-
-    def validate(self, iban, **kwargs):
-        iban = stringify(iban)
-        if iban is None:
-            return False
-        try:
-            IBAN(iban)
-            return True
-        except ValueError as ex:
-            print(ex)
+    """An international bank account number, as defined in ISO 13616. IBANs are
+    managed by SWIFT used in the European SEPA payment system.
+
+    A notable aspect of IBANs is that they share a country prefix and validation
+    mechanism, but the specific length of an IBAN is dependent on the country
+    code defined in the first two characters: `NO8330001234567` and
+    `CY21002001950000357001234567` are both valid values."""
+
+    name = "iban"
+    group = "ibans"
+    label = _("IBAN")
+    plural = _("IBANs")
+    matchable = True
+    pivot = True
+    max_length = 64
+
+    def validate(
+        self, value: str, fuzzy: bool = False, format: Optional[str] = None
+    ) -> bool:
+        text = sanitize_text(value)
+        if text is None:
             return False
-
-    def clean_text(self, text, **kwargs):
+        return IBAN.is_valid(text)
+
+    def clean_text(
+        self,
+        text: str,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        proxy: Optional["EntityProxy"] = None,
+    ) -> Optional[str]:
         """Create a more clean, but still user-facing version of an
         instance of the type."""
-        return text.replace(" ", "").upper()
+        return IBAN.normalize(text)
+
+    def country_hint(self, value: str) -> str:
+        return value[:2].lower()
 
-    def specificity(self, value):
-        return 1
+    def rdf(self, value: str) -> Identifier:
+        return URIRef(self.node_id(value))
 
-    def country_hint(self, value):
-        value = stringify(value)
-        if value is not None:
-            return value[:2].lower()
+    def node_id(self, value: str) -> str:
+        return f"iban:{value.upper()}"
 
-    def rdf(self, value):
-        return URIRef('iban:%s' % value)
+    def caption(self, value: str) -> str:
+        return IBAN.format(value)

followthemoney/types/identifier.py

@@ -1,34 +1,61 @@
 import re
-from normality import normalize
+from typing import Optional, TYPE_CHECKING
+from rigour.ids import get_identifier_format_names, get_identifier_format
 
 from followthemoney.types.common import PropertyType
+from followthemoney.util import dampen, shortest, longest
+from followthemoney.util import defer as _
+
+if TYPE_CHECKING:
+    from followthemoney.proxy import EntityProxy
 
 
 class IdentifierType(PropertyType):
-    """Used for registration numbers, codes etc."""
-    COMPARE_CLEAN = re.compile('[\W_]+')
-    name = 'identifier'
-    group = 'identifiers'
-    prefix = 'ident'
-    strong = False
-
-    def normalize(self, text, **kwargs):
-        """Normalize for comparison."""
-        ids = super(IdentifierType, self).normalize(text, **kwargs)
-        return [normalize(i) for i in ids]
-
-    def clean_compare(self, value):
+    """Used for registration numbers and other codes assigned by an authority
+    to identify an entity. This might include tax identifiers and statistical
+    codes.
+
+    Since identifiers are high-value criteria when comparing two entities, numbers
+    should only be modelled as identifiers if they are long enough to be meaningful.
+    Four- or five-digit industry classifiers create more noise than value."""
+
+    COMPARE_CLEAN = re.compile(r"[\W_]+")
+    name = "identifier"
+    group = "identifiers"
+    label = _("Identifier")
+    plural = _("Identifiers")
+    matchable = True
+    pivot = True
+    max_length = 64
+
+    def clean_text(
+        self,
+        text: str,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        proxy: Optional["EntityProxy"] = None,
+    ) -> Optional[str]:
+        if format in get_identifier_format_names():
+            format_ = get_identifier_format(format)
+            return format_.normalize(text)
+        return text
+
+    def clean_compare(self, value: str) -> str:
         # TODO: should this be used for normalization?
-        value = self.COMPARE_CLEAN.sub('', value)
+        value = self.COMPARE_CLEAN.sub("", value)
         return value.lower()
 
-    def compare(self, left, right):
+    def compare(self, left: str, right: str) -> float:
         left = self.clean_compare(left)
         right = self.clean_compare(right)
         if left == right:
-            return .9
-        if left in right:
-            return .7
-        if right in left:
-            return .7
-        return 0
+            return 1.0
+        elif left in right or right in left:
+            return len(shortest(left, right)) / len(longest(left, right))
+        return 0.0
+
+    def _specificity(self, value: str) -> float:
+        return dampen(4, 10, value)
+
+    def node_id(self, value: str) -> str:
+        return f"id:{value}"