followthemoney 1.3.6__py3-none-any.whl → 3.8.0__py3-none-any.whl

followthemoney/types/__init__.py

@@ -1,30 +1,48 @@
+from followthemoney.types.registry import Registry
 from followthemoney.types.url import UrlType
 from followthemoney.types.name import NameType
-from followthemoney.types.domain import DomainType
 from followthemoney.types.email import EmailType
 from followthemoney.types.ip import IpType
-from followthemoney.types.iban import IbanType
 from followthemoney.types.address import AddressType
 from followthemoney.types.date import DateType
 from followthemoney.types.phone import PhoneType
 from followthemoney.types.country import CountryType
 from followthemoney.types.language import LanguageType
+from followthemoney.types.mimetype import MimeType
+from followthemoney.types.checksum import ChecksumType
 from followthemoney.types.identifier import IdentifierType
+from followthemoney.types.iban import IbanType
 from followthemoney.types.entity import EntityType
-from followthemoney.types.common import TextType, Registry
+from followthemoney.types.topic import TopicType
+from followthemoney.types.gender import GenderType
+from followthemoney.types.json import JsonType
+from followthemoney.types.string import TextType
+from followthemoney.types.string import HTMLType
+from followthemoney.types.string import StringType
+from followthemoney.types.number import NumberType
+from followthemoney.types.common import PropertyType
 
 registry = Registry()
-urls = registry.add(UrlType())
-domains = registry.add(DomainType())
-emails = registry.add(EmailType())
-ips = registry.add(IpType())
-ibans = registry.add(IbanType())
-addresses = registry.add(AddressType())
-dates = registry.add(DateType())
-phones = registry.add(PhoneType())
-countries = registry.add(CountryType())
-languages = registry.add(LanguageType())
-identifiers = registry.add(IdentifierType())
-entities = registry.add(EntityType())
-texts = registry.add(TextType())
-names = registry.add(NameType())
+registry.add(UrlType)
+registry.add(NameType)
+registry.add(EmailType)
+registry.add(IpType)
+registry.add(AddressType)
+registry.add(DateType)
+registry.add(PhoneType)
+registry.add(CountryType)
+registry.add(LanguageType)
+registry.add(MimeType)
+registry.add(ChecksumType)
+registry.add(IdentifierType)
+registry.add(IbanType)  # TODO: remove
+registry.add(EntityType)
+registry.add(TopicType)
+registry.add(GenderType)
+registry.add(JsonType)
+registry.add(TextType)
+registry.add(HTMLType)
+registry.add(StringType)
+registry.add(NumberType)
+
+__all__ = ["PropertyType", "registry"]

followthemoney/types/address.py

@@ -1,31 +1,51 @@
 import re
+from typing import Optional, TYPE_CHECKING
+from normality import slugify
 from normality.cleaning import collapse_spaces
 
 from followthemoney.types.common import PropertyType
+from followthemoney.util import defer as _
 from followthemoney.util import dampen
 
+if TYPE_CHECKING:
+    from followthemoney.proxy import EntityProxy
+
 
 class AddressType(PropertyType):
-    LINE_BREAKS = re.compile(r'(\r\n|\n|<BR/>|<BR>|\t|ESQ\.,|ESQ,|;)')
-    COMMATA = re.compile(r'(,\s?[,\.])')
-    name = 'address'
-    group = 'addresses'
-    prefix = 'addr'
+    """A geographic address used to describe a location of a residence or post
+    box. There is no specified order for the sub-parts of an address (e.g. street,
+    city, postal code), and we should consider introducing an Address schema type
+    to retain fidelity in cases where address parts are specified."""
+
+    LINE_BREAKS = re.compile(r"(\r\n|\n|<BR/>|<BR>|\t|ESQ\.,|ESQ,|;)")
+    COMMATA = re.compile(r"(,\s?[,\.])")
+    name = "address"
+    group = "addresses"
+    label = _("Address")
+    plural = _("Addresses")
+    matchable = True
+    pivot = True
 
-    def clean_text(self, address, **kwargs):
+    def clean_text(
+        self,
+        text: str,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        proxy: Optional["EntityProxy"] = None,
+    ) -> Optional[str]:
         """Basic clean-up."""
-        address = self.LINE_BREAKS.sub(', ', address)
-        address = self.COMMATA.sub(', ', address)
-        address = collapse_spaces(address)
-        if len(address):
-            return address
-
-    # TODO: normalize well-known parts like "Street", "Road", etc.
-    # TODO: consider using https://github.com/openvenues/pypostal
-    # def normalize(self, address, **kwargs):
-    #     """Make the address more compareable."""
-    #     addresses = super(AddressType, self).normalize(address, **kwargs)
-    #     return addresses
-
-    def specificity(self, value):
-        return dampen(10, 50, value) * .6
+        address = self.LINE_BREAKS.sub(", ", text)
+        address = self.COMMATA.sub(", ", address)
+        collapsed = collapse_spaces(address)
+        if collapsed is None or not len(collapsed):
+            return None
+        return collapsed
+
+    def _specificity(self, value: str) -> float:
+        return dampen(10, 60, value)
+
+    def node_id(self, value: str) -> Optional[str]:
+        slug = slugify(value)
+        if slug is None:
+            return None
+        return f"addr:{value}"

followthemoney/types/checksum.py

@@ -0,0 +1,25 @@
+from followthemoney.rdf import URIRef, Identifier
+from followthemoney.types.common import PropertyType
+from followthemoney.util import defer as _
+
+
+class ChecksumType(PropertyType):
+    """Content hashes calculated using SHA1. Checksum references are used by
+    document-typed entities in Aleph to refer to raw data in the archive
+    (e.g. the document from which the entity is extracted).
+
+    Unfortunately, this has some security implications: in order to avoid people
+    getting access to documents for which they know the checksum, properties
+    of this type are scrubbed when submitted via the normal API. Checksums can only
+    be defined by uploading a document to be ingested."""
+
+    name = "checksum"
+    group = "checksums"
+    label = _("Checksum")
+    plural = _("Checksums")
+    matchable = True
+    pivot = True
+    max_length = 40
+
+    def rdf(self, value: str) -> Identifier:
+        return URIRef(f"hash:{value}")

followthemoney/types/common.py

@@ -1,131 +1,276 @@
+from inspect import cleandoc
 from itertools import product
-from rdflib import Literal
-from banal import ensure_list, is_mapping
+from babel.core import Locale
+from banal import ensure_list
 from normality import stringify
+from typing import Any, Dict, Optional, Sequence, Callable, TYPE_CHECKING, TypedDict
+
+from followthemoney.rdf import Literal, Identifier
+from followthemoney.util import get_locale
+from followthemoney.util import gettext, sanitize_text
+
+if TYPE_CHECKING:
+    from followthemoney.proxy import EntityProxy
+
+EnumValues = Dict[str, str]
+
+
+class PropertyTypeToDict(TypedDict, total=False):
+    label: str
+    plural: str
+    description: Optional[str]
+    maxLength: int
+    group: Optional[str]
+    matchable: Optional[bool]
+    pivot: Optional[bool]
+    values: Optional[EnumValues]
 
 
 class PropertyType(object):
-    """Base class for all types."""
-    name = None
-    group = None
-    prefix = None
-    strong = False
-
-    def validate(self, text, **kwargs):
-        """Returns a boolean to indicate if this is a valid instance of
+    """Base class for all property types."""
+
+    name: str = "any"
+    """A machine-facing, variable safe name for the given type."""
+
+    group: Optional[str] = None
+    """Groups are used to invert all the properties of an entity that have a
+    given  type into a single list before indexing them. This way, in Aleph,
+    you can query for ``countries:gb`` instead of having to make a set of filters
+    like ``properties.jurisdiction:gb OR properties.country:gb OR ...``."""
+
+    label: str = "Any"
+    """A name for this type to be shown to users."""
+
+    plural: str = "Any"
+    """A plural name for this type which can be used in appropriate places in
+    a user interface."""
+
+    matchable: bool = True
+    """Matchable types allow properties to be compared with each other in order to
+    assess entity similarity. While it makes sense to compare names, countries or
+    phone numbers, the same isn't true for raw JSON blobs or descriptive text
+    snippets."""
+
+    pivot: bool = False
+    """Pivot property types are like a stronger form of :attr:`~matchable` types:
+    they will be used when value-based lookups are used to find commonalities
+    between entities. For example, pivot typed-properties are used to show all the
+    other entities that mention the same phone number, email address or name as the
+    one currently seen by the user."""
+
+    max_length: int = 250
+    """The maximum length of a single value of this type. This is used to warn when
+    adding individual values that may be malformed or too long to be stored in
+    downstream databases with fixed column lengths. The unit is unicode codepoints
+    (not bytes), the output of Python len()."""
+
+    total_size: Optional[int] = None
+    """Some types have overall size limitations in place in order to avoid generating
+    entities that are very large (upstream ElasticSearch has a 100MB document limit).
+    Once the total size of all properties of this type has exceed the given limit,
+    an entity will refuse to add further values."""
+
+    @property
+    def docs(self) -> Optional[str]:
+        if not self.__doc__:
+            return None
+
+        return cleandoc(self.__doc__)
+
+    def validate(
+        self, value: str, fuzzy: bool = False, format: Optional[str] = None
+    ) -> bool:
+        """Returns a boolean to indicate if the given value is a valid instance of
         the type."""
-        cleaned = self.clean(text, **kwargs)
+        cleaned = self.clean(value, fuzzy=fuzzy, format=format)
         return cleaned is not None
 
-    def clean(self, text, **kwargs):
-        """Create a more clean, but still user-facing version of an
-        instance of the type."""
-        text = stringify(text)
-        if text is not None:
-            return self.clean_text(text, **kwargs)
+    def clean(
+        self,
+        raw: Any,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        proxy: Optional["EntityProxy"] = None,
+    ) -> Optional[str]:
+        """Create a clean version of a value of the type, suitable for storage
+        in an entity proxy."""
+        text = sanitize_text(raw)
+        if text is None:
+            return None
+        return self.clean_text(text, fuzzy=fuzzy, format=format, proxy=proxy)
 
-    def clean_text(self, text, **kwargs):
+    def clean_text(
+        self,
+        text: str,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        proxy: Optional["EntityProxy"] = None,
+    ) -> Optional[str]:
+        """Specific types can apply their own cleaning routines here (this is called
+        by ``clean`` after the value has been converted to a string and null values
+        have been filtered)."""
         return text
 
-    def normalize(self, text, cleaned=False, **kwargs):
-        """Create a represenation ideal for comparisons, but not to be
-        shown to the user."""
-        if not cleaned:
-            text = self.clean(text, **kwargs)
-        return ensure_list(text)
-
-    def normalize_set(self, items, **kwargs):
-        """Utility to normalize a whole set of values and get unique
-        values."""
-        values = set()
-        for item in ensure_list(items):
-            values.update(self.normalize(item, **kwargs))
-        return list(values)
-
-    def specificity(self, value):
-        return 0
-
-    def compare_safe(self, left, right):
+    def join(self, values: Sequence[str]) -> str:
+        """Helper function for converting multi-valued FtM data into formats that
+        allow only a single value per field (e.g. CSV). This is not fully reversible
+        and should be used as a last option."""
+        values = ensure_list(values)
+        return "; ".join(values)
+
+    def _specificity(self, value: str) -> float:
+        return 1.0
+
+    def specificity(self, value: Optional[str]) -> float:
+        """Return a score for how specific the given value is. This can be used as a
+        weighting factor in entity comparisons in order to rate matching property
+        values by how specific they are. For example: a longer address is considered
+        to be more specific than a short one, a full date more specific than just a
+        year number, etc."""
+        if not self.matchable or value is None:
+            return 0.0
+        return self._specificity(value)
+
+    def compare_safe(self, left: Optional[str], right: Optional[str]) -> float:
+        """Compare, but support None values on either side of the comparison."""
         left = stringify(left)
         right = stringify(right)
         if left is None or right is None:
-            return 0
+            return 0.0
         return self.compare(left, right)
 
-    def compare(self, left, right):
+    def compare(self, left: str, right: str) -> float:
         """Comparisons are a float between 0 and 1. They can assume
         that the given data is cleaned, but not normalised."""
         if left.lower() == right.lower():
-            return 1 * self.specificity(left)
-        return 0
+            return 1.0 * self.specificity(left)
+        return 0.0
 
-    def compare_sets(self, left, right, func=max):
-        """Compare two sets of values and select a specific result."""
+    def compare_sets(
+        self,
+        left: Sequence[str],
+        right: Sequence[str],
+        func: Callable[[Sequence[float]], float] = max,
+    ) -> float:
+        """Compare two sets of values and select the highest-scored result."""
         results = []
-        for (l, r) in product(ensure_list(left), ensure_list(right)):
-            results.append(self.compare_safe(l, r))
+        for le, ri in product(ensure_list(left), ensure_list(right)):
+            results.append(self.compare(le, ri))
         if not len(results):
-            return 0
+            return 0.0
         return func(results)
 
-    def country_hint(self, value):
-        """Determine if the given value allows us to infer a country
-        that it may be related to."""
+    def country_hint(self, value: str) -> Optional[str]:
+        """Determine if the given value allows us to infer a country that it may
+        be related to (e.g. using a country prefix on a phone number or IBAN)."""
         return None
 
-    def ref(self, value):
-        """Generate a qualified form for storage in a triplestore."""
-        if self.prefix is None:
-            return
-        if is_mapping(value):
-            value = value.get('id')
-        value = stringify(value)
+    def rdf(self, value: str) -> Identifier:
+        """Return an RDF term to represent the given value - either a string
+        literal, or a URI reference."""
+        return Literal(value)
+
+    def pick(self, values: Sequence[str]) -> Optional[str]:
+        """Pick the best value to show to the user."""
+        raise NotImplementedError
+
+    def node_id(self, value: str) -> Optional[str]:
+        """Return an ID suitable to identify this entity as a typed node in a
+        graph representation of some FtM data. It's usually the same as the the
+        RDF form."""
+        return str(self.rdf(value))
+
+    def node_id_safe(self, value: Optional[str]) -> Optional[str]:
+        """Wrapper for node_id to handle None values."""
         if value is None:
-            return
-        return ':'.join((self.prefix, value))
+            return None
+        return self.node_id(value)
 
-    def rdf(self, value):
-        return Literal(value)
+    def caption(self, value: str) -> Optional[str]:
+        """Return a label for the given property value. This is often the same as the
+        value, but for types like countries or languages, it would return the label,
+        while other values like phone numbers can be formatted to be nicer to read."""
+        return value
 
-    def __eq__(self, other):
+    def to_dict(self) -> PropertyTypeToDict:
+        """Return a serialisable description of this data type."""
+        data: PropertyTypeToDict = {
+            "label": gettext(self.label),
+            "plural": gettext(self.plural),
+            "description": gettext(self.docs),
+            "maxLength": self.max_length,
+        }
+        if self.group:
+            data["group"] = self.group
+        if self.matchable:
+            data["matchable"] = True
+        if self.pivot:
+            data["pivot"] = True
+        return data
+
+    def __eq__(self, other: Any) -> bool:
+        if not isinstance(other, PropertyType):
+            return False
         return self.name == other.name
 
-    def __hash__(self):
+    def __hash__(self) -> int:
         return hash(self.name)
 
-    def __str__(self):
+    def __str__(self) -> str:
         return self.name
 
-    def __repr__(self):
-        return '<%s()>' % type(self).__name__
+    def __repr__(self) -> str:
+        return f"<{self.name}>"
+
 
+class EnumType(PropertyType):
+    """Enumerated type properties are used for types which have a defined set
+    of possible values, like languages and countries."""
 
-class TextType(PropertyType):
-    name = 'text'
+    def __init__(self) -> None:
+        self._names: Dict[Locale, EnumValues] = {}
+        self.codes = set(self.names.keys())
 
+    def _locale_names(self, locale: Locale) -> EnumValues:
+        return {}
 
-class Registry(object):
+    @property
+    def names(self) -> EnumValues:
+        """Return a mapping from property values to their labels in the current
+        locale."""
+        locale = get_locale()
+        if locale not in self._names:
+            self._names[locale] = self._locale_names(locale)
+        return self._names[locale]
 
-    def __init__(self):
-        self.prefixes = {}
-        self.groups = {}
-        self.names = {}
+    def validate(
+        self, value: str, fuzzy: bool = False, format: Optional[str] = None
+    ) -> bool:
+        """Make sure that the given code value is one of the supported set."""
+        if value is None:
+            return False
+        return str(value).lower().strip() in self.codes
 
-    def add(self, instance):
-        setattr(self, instance.name, instance)
-        self.names[instance.name] = instance
-        if instance.prefix is not None:
-            self.prefixes[instance.prefix] = instance
-        if instance.group is not None:
-            self.groups[instance.group] = instance
-        return instance
+    def clean_text(
+        self,
+        code: str,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        proxy: Optional["EntityProxy"] = None,
+    ) -> Optional[str]:
+        """All code values are cleaned to be lowercase and trailing whitespace is
+        removed."""
+        code = code.lower().strip()
+        if code not in self.codes:
+            return None
+        return code
 
-    def get(self, name):
-        try:
-            return getattr(self, name)
-        except AttributeError:
-            pass
+    def caption(self, value: str) -> str:
+        """Given a code value, return the label that should be shown to a user."""
+        return self.names.get(value, value)
 
-    def deref(self, ref):
-        prefix, value = ref.split(':', 1)
-        return self.prefixes.get(prefix), value
+    def to_dict(self) -> PropertyTypeToDict:
+        """When serialising the model to JSON, include all values."""
+        data = super(EnumType, self).to_dict()
+        data["values"] = self.names
+        return data