followthemoney 3.5.9__py2.py3-none-any.whl → 3.6.0__py2.py3-none-any.whl

followthemoney/__init__.py

@@ -3,7 +3,7 @@ import os
 from followthemoney.model import Model
 from followthemoney.util import set_model_locale
 
-__version__ = "3.5.9"
+__version__ = "3.6.0"
 
 
 model_path = os.path.dirname(__file__)

followthemoney/cli/cli.py

@@ -1,14 +1,14 @@
 import sys
-import json
 import click
+import orjson
 import logging
 from pathlib import Path
-from typing import Optional, TextIO
+from typing import Optional, BinaryIO, List, Any, Dict
 from banal import ensure_list
 
 from followthemoney import model
 from followthemoney.namespace import Namespace
-from followthemoney.cli.util import InPath, OutPath, path_entities, read_entities
+from followthemoney.cli.util import InPath, OutPath, path_entities
 from followthemoney.cli.util import path_writer, write_entity
 from followthemoney.proxy import EntityProxy
 
@@ -20,9 +20,10 @@ def cli() -> None:
 
 
 @cli.command("dump-model", help="Export the current schema model")
-@click.option("-o", "--outfile", type=click.File("w"), default="-")
-def dump_model(outfile: TextIO) -> None:
-    outfile.write(json.dumps(model.to_dict(), indent=2, sort_keys=True))
+@click.option("-o", "--outfile", type=click.File("wb"), default="-")
+def dump_model(outfile: BinaryIO) -> None:
+    f = orjson.OPT_INDENT_2 | orjson.OPT_SORT_KEYS
+    outfile.write(orjson.dumps(model.to_dict(), option=f))
 
 
 @cli.command("validate", help="Re-parse and validate the given data")
@@ -34,7 +35,7 @@ def validate(infile: Path, outfile: Path) -> None:
             for entity in path_entities(infile, EntityProxy, cleaned=False):
                 clean = model.make_entity(entity.schema)
                 clean.id = entity.id
-                for (prop, value) in entity.itervalues():
+                for prop, value in entity.itervalues():
                     clean.add(prop, value)
                 write_entity(outfh, clean)
     except BrokenPipeError:
@@ -46,12 +47,14 @@ def validate(infile: Path, outfile: Path) -> None:
 @click.option("-o", "--outfile", type=OutPath, default="-")  # noqa
 def import_vis(infile: Path, outfile: Path) -> None:
     with path_writer(outfile) as outfh:
-        with open(infile, "r") as infh:
-            data = json.load(infh)
+        with open(infile, "rb") as infh:
+            data: Dict[str, Any] = orjson.loads(infh.read())
             if "entities" in data:
-                entities = data.get("entities", data)
-            if "layout" in data:
+                entities: List[Dict[str, Any]] = data.get("entities", data)
+            elif "layout" in data:
                 entities = data.get("layout", {}).get("entities", data)
+            else:
+                raise click.ClickException("No entities found in VIS file")
             for entity_data in ensure_list(entities):
                 entity = EntityProxy.from_dict(model, entity_data)
                 write_entity(outfh, entity)
@@ -75,10 +78,11 @@ def sign(infile: Path, outfile: Path, signature: Optional[str]) -> None:
 @cli.command(help="Format a stream of entities to make it readable")
 @click.option("-i", "--infile", type=InPath, default="-")  # noqa
 def pretty(infile: Path) -> None:
-    stdout = click.get_text_stream("stdout")
+    stdout = click.get_binary_stream("stdout")
     try:
+        f = orjson.OPT_INDENT_2 | orjson.OPT_APPEND_NEWLINE
         for entity in path_entities(infile, EntityProxy):
-            data = json.dumps(entity.to_dict(), indent=2)
-            stdout.write(data + "\n")
+            data = orjson.dumps(entity.to_dict(), option=f)
+            stdout.write(data)
     except BrokenPipeError:
         raise click.Abort()

followthemoney/export/excel.py

@@ -1,11 +1,11 @@
 import logging
 from io import BytesIO
 from typing import Dict, List, Optional
-from openpyxl import Workbook  # type: ignore
-from openpyxl.cell import WriteOnlyCell  # type: ignore
-from openpyxl.styles import Font, PatternFill  # type: ignore
-from openpyxl.worksheet.worksheet import Worksheet  # type: ignore
-from openpyxl.utils.exceptions import IllegalCharacterError  # type: ignore
+from openpyxl import Workbook
+from openpyxl.cell import WriteOnlyCell
+from openpyxl.styles import Font, PatternFill
+from openpyxl.worksheet.worksheet import Worksheet
+from openpyxl.utils.exceptions import IllegalCharacterError
 
 from followthemoney.export.common import Exporter
 from followthemoney.proxy import E
@@ -25,7 +25,7 @@ class ExcelWriter(object):
         self.workbook = Workbook(write_only=True)
 
     def make_sheet(self, title: str, headers: List[str]) -> Worksheet:
-        sheet = self.workbook.create_sheet(title=title)
+        sheet: Worksheet = self.workbook.create_sheet(title=title)
         sheet.freeze_panes = "A2"
         sheet.sheet_properties.filterMode = True
         cells = []

followthemoney/mapping/entity.py

@@ -1,3 +1,4 @@
+import logging
 from hashlib import sha1
 from warnings import warn
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set
@@ -15,6 +16,8 @@ if TYPE_CHECKING:
     from followthemoney.model import Model
     from followthemoney.mapping.query import QueryMapping
 
+log = logging.getLogger(__name__)
+
 
 class EntityMapping(object):
 
@@ -112,16 +115,24 @@ class EntityMapping(object):
         # from that accessible to phone and address parsers.
         for prop in self.properties:
             if prop.prop.type == registry.country:
-                prop.map(proxy, record, entities)
+                discarded_values = prop.map(proxy, record, entities)
+                for value in discarded_values:
+                    log.warn(f"[{self.name}] Discarded unclean value \"{value}\" for property \"{prop.prop.qname}\".")
 
         for prop in self.properties:
             if prop.prop.type != registry.country:
-                prop.map(proxy, record, entities)
+                discarded_values = prop.map(proxy, record, entities)
+                for value in discarded_values:
+                    log.warn(f"[{self.name}] Discarding unclean value \"{value}\" for property \"{prop.prop.qname}\".")
 
         # Generate the ID at the end to avoid self-reference checks on empty
         # keys.
         proxy.id = self.compute_key(record)
         if proxy.id is None:
+            if self.id_column:
+                log.warn(f"[{self.name}] Skipping entity because no ID could be computed. Make sure that there are no empty values in the \"{self.id_column}\" column.")
+            if self.keys:
+                log.warn(f"[{self.name}] Skipping entity because no ID could be computed. Make sure that there are no empty values in key columns.")
             return None
 
         for prop in self.properties:
@@ -130,6 +141,7 @@ class EntityMapping(object):
                 # the mapping, not in the model. Basically it means: if
                 # this row of source data doesn't have that field, then do
                 # not map it again.
+                log.warn(f"[{self.name}] Skipping entity because required property \"{prop.prop.name}\" is empty.")
                 return None
         return proxy
 

followthemoney/mapping/property.py

@@ -112,13 +112,13 @@ class PropertyMapping(object):
 
     def map(
         self, proxy: EntityProxy, record: Record, entities: Dict[str, EntityProxy]
-    ) -> None:
+    ) -> List[str]:
         if self.entity is not None:
             entity = entities.get(self.entity)
             if entity is not None:
                 proxy.unsafe_add(self.prop, entity.id, cleaned=True)
                 inline_names(proxy, entity)
-            return None
+            return []
 
         # clean the values returned by the query, or by using literals, or
         # formats.
@@ -133,5 +133,17 @@ class PropertyMapping(object):
                 splote.extend(value.split(self.split))
             values = splote
 
+        discarded_values: List[str] = []
+
         for value in values:
-            proxy.unsafe_add(self.prop, value, fuzzy=self.fuzzy, format=self.format)
+            added_value = proxy.unsafe_add(
+                prop=self.prop,
+                value=value,
+                fuzzy=self.fuzzy,
+                format=self.format,
+            )
+
+            if value is not None and added_value is None:
+                discarded_values.append(value)
+
+        return discarded_values

followthemoney/mapping/sql.py

@@ -55,7 +55,7 @@ class SQLSource(Source):
         if database is None:
             raise InvalidMapping("No database in SQL mapping!")
         self.database_uri = cast(str, os.path.expandvars(database))
-        self.engine = create_engine(self.database_uri, poolclass=NullPool)  
+        self.engine = create_engine(self.database_uri, poolclass=NullPool)
         self.meta = MetaData()
 
         tables = keys_values(data, "table", "tables")

followthemoney/property.py

@@ -27,6 +27,7 @@ class PropertyDict(TypedDict, total=False):
     # stub: Optional[bool]
     rdf: Optional[str]
     range: Optional[str]
+    format: Optional[str]
 
 
 class PropertySpec(PropertyDict):
@@ -58,6 +59,7 @@ class Property:
         "matchable",
         "deprecated",
         "_range",
+        "format",
         "range",
         "stub",
         "_reverse",
@@ -113,6 +115,11 @@ class Property:
         self._range = data.get("range")
         self.range: Optional["Schema"] = None
 
+        #: If the property is of type ``identifier``, a more narrow definition of the
+        #: identifier format can be provided. For example, LEI, INN or IBAN codes
+        #: can be automatically validated.
+        self.format: Optional[str] = data.get("format")
+
         #: When a property points to another schema, a reverse property is added for
         #: various administrative reasons. These properties are, however, not real
         #: and cannot be written to. That's why they are marked as stubs and adding
@@ -169,6 +176,8 @@ class Property:
             if self.stub:
                 return gettext("Property cannot be written")
             val = get_entity_id(val)
+            if val is None:
+                continue
             if not self.type.validate(val):
                 return gettext("Invalid value")
             if val is not None:
@@ -203,6 +212,8 @@ class Property:
             data["range"] = self.range.name
         if self.reverse is not None:
             data["reverse"] = self.reverse.name
+        if self.format is not None:
+            data["format"] = self.format
         return data
 
     def __repr__(self) -> str:

followthemoney/proxy.py

@@ -194,6 +194,7 @@ class EntityProxy(object):
 
         for value in value_list(values):
             if not cleaned:
+                format = format or prop.format
                 value = prop.type.clean(value, proxy=self, fuzzy=fuzzy, format=format)
             self.unsafe_add(prop, value, cleaned=True)
         return None
@@ -205,26 +206,32 @@ class EntityProxy(object):
         cleaned: bool = False,
         fuzzy: bool = False,
         format: Optional[str] = None,
-    ) -> None:
+    ) -> Optional[str]:
         """A version of `add()` to be used only in type-checking code. This accepts
         only a single value, and performs input cleaning on the premise that the
-        value is already valid unicode."""
+        value is already valid unicode. Returns the value that has been added."""
         if not cleaned and value is not None:
+            format = format or prop.format
             value = prop.type.clean_text(value, fuzzy=fuzzy, format=format, proxy=self)
-        if value is not None:
-            # Somewhat hacky: limit the maximum size of any particular
-            # field to avoid overloading upstream aleph/elasticsearch.
-            value_size = len(value)
-            if prop.type.max_size is not None:
-                if self._size + value_size > prop.type.max_size:
-                    # msg = "[%s] too large. Rejecting additional values."
-                    # log.warning(msg, prop.name)
-                    return None
-            self._size += value_size
-            self._properties.setdefault(prop.name, list())
-            if value not in self._properties[prop.name]:
-                self._properties[prop.name].append(value)
-        return None
+
+        if value is None:
+            return None
+
+        # Somewhat hacky: limit the maximum size of any particular
+        # field to avoid overloading upstream aleph/elasticsearch.
+        value_size = len(value)
+        if prop.type.max_size is not None:
+            if self._size + value_size > prop.type.max_size:
+                # msg = "[%s] too large. Rejecting additional values."
+                # log.warning(msg, prop.name)
+                return None
+        self._size += value_size
+        self._properties.setdefault(prop.name, list())
+
+        if value not in self._properties[prop.name]:
+            self._properties[prop.name].append(value)
+
+        return value
 
     def set(
         self,
@@ -424,9 +431,12 @@ class EntityProxy(object):
         dictionary can be used to make a new proxy, and it is commonly written to disk
         or a database."""
         data = dict(self.context)
-        data.update(
-            {"id": self.id, "schema": self.schema.name, "properties": self.properties}
-        )
+        extra = {
+            "id": self.id,
+            "schema": self.schema.name,
+            "properties": self.properties,
+        }
+        data.update(extra)
         return data
 
     def to_full_dict(self, matchable: bool = False) -> Dict[str, Any]:

followthemoney/schema/Analyzable.yaml

@@ -32,6 +32,8 @@ Analyzable:
     ibanMentioned:
       label: "Detected IBANs"
       hidden: true
+      # type: identifier
+      # format: iban
       type: iban
     ipMentioned:
       label: "Detected IP addresses"

followthemoney/schema/BankAccount.yaml

@@ -28,10 +28,13 @@ BankAccount:
       type: identifier
     iban:
       label: IBAN
+      # type: identifier
+      # format: iban
       type: iban
     bic:
       label: Bank Identifier Code
       type: identifier
+      format: bic
     bank:
       label: Bank
       type: entity

followthemoney/schema/LegalEntity.yaml

@@ -107,15 +107,18 @@ LegalEntity:
       label: "INN"
       description: "Russian company ID"
       type: identifier
+      # format: inn
     ogrnCode:
       label: "OGRN"
       description: "Major State Registration Number"
       type: identifier
+      # format: ogrn
     leiCode:
       # cf. https://www.gleif.org/en/about-lei/introducing-the-legal-entity-identifier-lei
       label: "LEI"
       description: "Legal Entity Identifier"
       type: identifier
+      format: lei
     dunsCode:
       label: "D-U-N-S"
       description: "Dun & Bradstreet identifier"
@@ -124,6 +127,7 @@ LegalEntity:
       label: "SWIFT/BIC"
       description: "Bank identifier code"
       type: identifier
+      format: bic
     parent:
       label: "Parent company"
       description: "If this entity is a subsidiary, another entity (company or organisation) is its parent"

followthemoney/schema/Person.yaml

@@ -26,6 +26,10 @@ Person:
     title:
       label: Title
       rdf: http://xmlns.com/foaf/0.1/title
+    # The `firstName`, `lastName`, `secondName` etc. properties intentionally do not use
+    # the `name` property type. Many FtM tools (including Aleph) use name properties to
+    # compare/match entities, but matching entites just on e.g. a first name would lead to
+    # too many false positives.
     firstName:
       label: First name
       rdf: http://xmlns.com/foaf/0.1/givenName

followthemoney/schema/Security.yaml

@@ -23,6 +23,7 @@ Security:
       label: ISIN
       description: International Securities Identification Number
       type: identifier
+      format: isin
     registrationNumber:
       label: Registration number
       type: identifier
@@ -32,6 +33,7 @@ Security:
     figiCode:
       label: Financial Instrument Global Identifier
       type: identifier
+      format: figi
     issuer:
       label: "Issuer"
       type: entity

followthemoney/schema/Thing.yaml

@@ -57,6 +57,7 @@ Thing:
     wikidataId:
       label: Wikidata ID
       type: identifier
+      format: qid
     keywords:
       label: Keywords
     topics:

followthemoney/types/__init__.py

@@ -3,7 +3,6 @@ from followthemoney.types.url import UrlType
 from followthemoney.types.name import NameType
 from followthemoney.types.email import EmailType
 from followthemoney.types.ip import IpType
-from followthemoney.types.iban import IbanType
 from followthemoney.types.address import AddressType
 from followthemoney.types.date import DateType
 from followthemoney.types.phone import PhoneType
@@ -12,6 +11,7 @@ from followthemoney.types.language import LanguageType
 from followthemoney.types.mimetype import MimeType
 from followthemoney.types.checksum import ChecksumType
 from followthemoney.types.identifier import IdentifierType
+from followthemoney.types.iban import IbanType
 from followthemoney.types.entity import EntityType
 from followthemoney.types.topic import TopicType
 from followthemoney.types.gender import GenderType
@@ -27,7 +27,6 @@ registry.add(UrlType)
 registry.add(NameType)
 registry.add(EmailType)
 registry.add(IpType)
-registry.add(IbanType)
 registry.add(AddressType)
 registry.add(DateType)
 registry.add(PhoneType)
@@ -36,6 +35,7 @@ registry.add(LanguageType)
 registry.add(MimeType)
 registry.add(ChecksumType)
 registry.add(IdentifierType)
+registry.add(IbanType)  # TODO: remove
 registry.add(EntityType)
 registry.add(TopicType)
 registry.add(GenderType)

followthemoney/types/common.py

@@ -1,3 +1,4 @@
+from inspect import cleandoc
 from itertools import product
 from babel.core import Locale
 from banal import ensure_list
@@ -64,12 +65,17 @@ class PropertyType(object):
 
     @property
     def docs(self) -> Optional[str]:
-        return self.__doc__
+        if not self.__doc__:
+            return None
+
+        return cleandoc(self.__doc__)
 
-    def validate(self, value: str) -> bool:
+    def validate(
+        self, value: str, fuzzy: bool = False, format: Optional[str] = None
+    ) -> bool:
         """Returns a boolean to indicate if the given value is a valid instance of
         the type."""
-        cleaned = self.clean(value)
+        cleaned = self.clean(value, fuzzy=fuzzy, format=format)
         return cleaned is not None
 
     def clean(
@@ -141,7 +147,7 @@ class PropertyType(object):
     ) -> float:
         """Compare two sets of values and select the highest-scored result."""
         results = []
-        for (l, r) in product(ensure_list(left), ensure_list(right)):
+        for l, r in product(ensure_list(left), ensure_list(right)):
             results.append(self.compare(l, r))
         if not len(results):
             return 0.0
@@ -229,7 +235,9 @@ class EnumType(PropertyType):
             self._names[locale] = self._locale_names(locale)
         return self._names[locale]
 
-    def validate(self, value: str) -> bool:
+    def validate(
+        self, value: str, fuzzy: bool = False, format: Optional[str] = None
+    ) -> bool:
         """Make sure that the given code value is one of the supported set."""
         if value is None:
             return False

followthemoney/types/date.py

@@ -27,9 +27,14 @@ class DateType(PropertyType):
     plural = _("Dates")
     matchable = True
 
-    def validate(self, value: str) -> bool:
+    def validate(
+        self, value: str, fuzzy: bool = False, format: Optional[str] = None
+    ) -> bool:
         """Check if a thing is a valid date."""
-        prefix = parse(value)
+        if format is not None:
+            prefix = parse_format(value, format)
+        else:
+            prefix = parse(value)
         return prefix.precision != Precision.EMPTY
 
     def clean_text(

followthemoney/types/email.py

@@ -36,7 +36,9 @@ class EmailType(PropertyType):
     #     except:
     #         return False
 
-    def validate(self, value: str) -> bool:
+    def validate(
+        self, value: str, fuzzy: bool = False, format: Optional[str] = None
+    ) -> bool:
         """Check to see if this is a valid email address."""
         # TODO: adopt email.utils.parseaddr
         email = sanitize_text(value)

followthemoney/types/entity.py

@@ -29,7 +29,9 @@ class EntityType(PropertyType):
     matchable = True
     pivot = True
 
-    def validate(self, value: str) -> bool:
+    def validate(
+        self, value: str, fuzzy: bool = False, format: Optional[str] = None
+    ) -> bool:
         text = sanitize_text(value)
         if text is None:
             return False

followthemoney/types/iban.py

@@ -1,6 +1,5 @@
-from typing import Optional, TYPE_CHECKING, cast
-from stdnum import iban  # type: ignore
-from stdnum.exceptions import ValidationError  # type: ignore
+from typing import Optional, TYPE_CHECKING
+from rigour.ids import IBAN
 
 from followthemoney.types.common import PropertyType
 from followthemoney.rdf import URIRef, Identifier
@@ -26,12 +25,11 @@ class IbanType(PropertyType):
     matchable = True
     pivot = True
 
-    def validate(self, value: str) -> bool:
+    def validate(self, value: str, fuzzy: bool = False, format: Optional[str] = None) -> bool:
         text = sanitize_text(value)
-        try:
-            return cast(bool, iban.validate(text))
-        except ValidationError:
+        if text is None:
             return False
+        return IBAN.is_valid(text)
 
     def clean_text(
         self,
@@ -42,7 +40,7 @@ class IbanType(PropertyType):
     ) -> Optional[str]:
         """Create a more clean, but still user-facing version of an
         instance of the type."""
-        return text.replace(" ", "").upper()
+        return IBAN.normalize(text)
 
     def country_hint(self, value: str) -> str:
         return value[:2].lower()
@@ -54,4 +52,4 @@ class IbanType(PropertyType):
         return f"iban:{value.upper()}"
 
     def caption(self, value: str) -> str:
-        return cast(str, iban.format(value))
+        return IBAN.format(value)

followthemoney/types/identifier.py

@@ -1,9 +1,14 @@
 import re
+from typing import Optional, TYPE_CHECKING
+from rigour.ids import get_identifier_format_names, get_identifier_format
 
 from followthemoney.types.common import PropertyType
 from followthemoney.util import dampen, shortest, longest
 from followthemoney.util import defer as _
 
+if TYPE_CHECKING:
+    from followthemoney.proxy import EntityProxy
+
 
 class IdentifierType(PropertyType):
     """Used for registration numbers and other codes assigned by an authority
@@ -22,6 +27,18 @@ class IdentifierType(PropertyType):
     matchable = True
     pivot = True
 
+    def clean_text(
+        self,
+        text: str,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        proxy: Optional["EntityProxy"] = None,
+    ) -> Optional[str]:
+        if format in get_identifier_format_names():
+            format_ = get_identifier_format(format)
+            return format_.normalize(text)
+        return text
+
     def clean_compare(self, value: str) -> str:
         # TODO: should this be used for normalization?
         value = self.COMPARE_CLEAN.sub("", value)

followthemoney/types/ip.py

@@ -21,7 +21,9 @@ class IpType(PropertyType):
     matchable = True
     pivot = True
 
-    def validate(self, value: str) -> bool:
+    def validate(
+        self, value: str, fuzzy: bool = False, format: Optional[str] = None
+    ) -> bool:
         """Check to see if this is a valid ip address."""
         try:
             ip_address(value)

followthemoney/types/language.py

@@ -1,6 +1,6 @@
 from typing import Optional, TYPE_CHECKING
 from babel.core import Locale
-from languagecodes import iso_639_alpha3
+from rigour.langs import iso_639_alpha3
 
 from followthemoney.types.common import EnumType, EnumValues
 from followthemoney.rdf import URIRef, Identifier

followthemoney/types/mimetype.py

@@ -1,6 +1,6 @@
 from typing import Optional, TYPE_CHECKING
-from pantomime import normalize_mimetype, parse_mimetype
-from pantomime import DEFAULT
+from rigour.mime import normalize_mimetype, parse_mimetype
+from rigour.mime import DEFAULT
 
 from followthemoney.types.common import PropertyType
 from followthemoney.rdf import URIRef, Identifier

followthemoney/types/name.py

@@ -1,6 +1,6 @@
-from typing import TYPE_CHECKING, Dict, List, Optional, Sequence, Union
-
-from Levenshtein import distance, setmedian
+from typing import TYPE_CHECKING, Optional, Sequence
+from rigour.text.distance import levenshtein_similarity
+from rigour.names import pick_name
 from normality import slugify
 from normality.cleaning import collapse_spaces, strip_quotes
 
@@ -40,44 +40,15 @@ class NameType(PropertyType):
 
     def pick(self, values: Sequence[str]) -> Optional[str]:
         """From a set of names, pick the most plausible user-facing one."""
-        # Sort to get stable results when it's a coin toss:
-        values = sorted(values)
-        if not len(values):
-            return None
-        normalised: List[Union[str, bytes]] = []
-        lookup: Dict[str, List[Union[str, bytes]]] = {}
-        # We're doing this in two stages, to avoid name forms with varied casing
-        # (e.g. Smith vs. SMITH) are counted as widly different, leading to
-        # implausible median outcomes.
-        for value in values:
-            norm = slugify(value, sep=" ")
-            if norm is None:
-                continue
-            normalised.append(norm)
-            lookup.setdefault(norm, [])
-            lookup[norm].append(value)
-
-        if not normalised:
-            return None
-
-        norm = setmedian(normalised)
-        if norm is None:
-            return None
-        forms = lookup.get(norm, [])
-        if len(forms) > 1:
-            return setmedian(forms)
-        for form in forms:
-            return str(form)
-        return None
+        return pick_name(list(values))
 
     def _specificity(self, value: str) -> float:
         # TODO: insert artificial intelligence here.
         return dampen(3, 50, value)
 
     def compare(self, left: str, right: str) -> float:
-        longest = float(max(len(left), len(right), 1))
-        edits = float(distance(left[:255], right[:255]))
-        return (longest - edits) / longest
+        """Compare two names for similarity."""
+        return levenshtein_similarity(left, right)
 
     def node_id(self, value: str) -> Optional[str]:
         slug = slugify(value)

followthemoney/types/phone.py

@@ -62,7 +62,9 @@ class PhoneType(PropertyType):
             except NumberParseException:
                 pass
 
-    def validate(self, value: str) -> bool:
+    def validate(
+        self, value: str, fuzzy: bool = False, format: Optional[str] = None
+    ) -> bool:
         for num in self._parse_number(value):
             if is_valid_number(num):
                 return True

followthemoney/types/topic.py

@@ -56,7 +56,10 @@ class TopicType(EnumType):
         "fin.bank": _("Bank"),
         "fin.fund": _("Fund"),
         "fin.adivsor": _("Financial advisor"),
-        "role.pep": _("Political"),
+        "reg.action": _("Regulator action"),
+        "reg.warn": _("Regulator warning"),
+        "role.pep": _("Politican"),
+        "role.pol": _("Non-PEP"),
         "role.rca": _("Close Associate"),
         "role.judge": _("Judge"),
         "role.civil": _("Civil servant"),
@@ -74,6 +77,7 @@ class TopicType(EnumType):
         "asset.frozen": _("Frozen asset"),
         "sanction": _("Sanctioned entity"),
         "sanction.linked": _("Sanction-linked entity"),
+        "sanction.counter": _("Counter-sanctioned entity"),
         "export.control": _("Export controlled"),
         "debarment": _("Debarred entity"),
         "poi": _("Person of interest"),

followthemoney/types/url.py

@@ -1,5 +1,5 @@
 from typing import Optional, TYPE_CHECKING
-from urllib.parse import urlparse
+from rigour.urls import clean_url, compare_urls
 
 from followthemoney.types.common import PropertyType
 from followthemoney.rdf import URIRef, Identifier
@@ -33,26 +33,10 @@ class UrlType(PropertyType):
     ) -> Optional[str]:
         """Perform intensive care on URLs to make sure they have a scheme
         and a host name. If no scheme is given HTTP is assumed."""
-        try:
-            parsed = urlparse(text)
-        except (TypeError, ValueError):
-            return None
-        if not len(parsed.netloc):
-            if "." in parsed.path and not text.startswith("//"):
-                # This is a pretty weird rule meant to catch things like
-                # 'www.google.com', but it'll likely backfire in some
-                # really creative ways.
-                return self.clean_text(f"//{text}")
-            return None
-        if not len(parsed.scheme):
-            parsed = parsed._replace(scheme=self.DEFAULT_SCHEME)
-        else:
-            parsed = parsed._replace(scheme=parsed.scheme.lower())
-        if parsed.scheme not in self.SCHEMES:
-            return None
-        if not len(parsed.path):
-            parsed = parsed._replace(path="/")
-        return parsed.geturl()
+        return clean_url(text)
+
+    def compare(self, left: str, right: str) -> float:
+        return compare_urls(left, right)
 
     def _specificity(self, value: str) -> float:
         return dampen(10, 120, value)

{followthemoney-3.5.9.dist-info → followthemoney-3.6.0.dist-info}/METADATA

@@ -1,38 +1,35 @@
 Metadata-Version: 2.1
 Name: followthemoney
-Version: 3.5.9
-Summary: UNKNOWN
+Version: 3.6.0
 Home-page: https://followthemoney.tech/
 Author: Organized Crime and Corruption Reporting Project
 Author-email: data@occrp.org
 License: MIT
-Platform: UNKNOWN
 Classifier: Intended Audience :: Developers
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3.9
 Description-Content-Type: text/markdown
-Requires-Dist: babel <3.0.0,>=2.9.1
+License-File: LICENSE
+Requires-Dist: babel <3.0.0,>=2.14.0
 Requires-Dist: pyyaml <7.0.0,>=5.0.0
 Requires-Dist: types-PyYAML
 Requires-Dist: sqlalchemy2-stubs
 Requires-Dist: banal <1.1.0,>=1.0.6
+Requires-Dist: rigour <1.0.0,>=0.5.1
 Requires-Dist: click <9.0.0,>=8.0
 Requires-Dist: stringcase <2.0.0,>=1.2.0
 Requires-Dist: requests <3.0.0,>=2.21.0
-Requires-Dist: python-levenshtein <1.0.0,>=0.12.0
 Requires-Dist: normality <3.0.0,>=2.4.0
 Requires-Dist: sqlalchemy <3.0.0,>=1.4.49
 Requires-Dist: countrynames <2.0.0,>=1.13.0
-Requires-Dist: languagecodes <2.0.0,>=1.1.0
 Requires-Dist: prefixdate <1.0.0,>=0.4.0
 Requires-Dist: fingerprints <2.0.0,>=1.0.1
 Requires-Dist: phonenumbers <9.0.0,>=8.12.22
 Requires-Dist: python-stdnum <2.0.0,>=1.16
-Requires-Dist: pantomime <1.0.0,>=0.5.1
 Requires-Dist: pytz >=2021.1
 Requires-Dist: rdflib <7.1.0,>=6.2.0
-Requires-Dist: networkx <3.3,>=2.5
+Requires-Dist: networkx <3.4,>=2.5
 Requires-Dist: openpyxl <4.0.0,>=3.0.5
 Requires-Dist: orjson <4.0,>=3.7
 Provides-Extra: dev
@@ -46,6 +43,7 @@ Requires-Dist: pytest-cov ; extra == 'dev'
 Requires-Dist: types-PyYAML ; extra == 'dev'
 Requires-Dist: types-requests ; extra == 'dev'
 Requires-Dist: types-setuptools ; extra == 'dev'
+Requires-Dist: types-openpyxl ; extra == 'dev'
 Requires-Dist: flake8 >=2.6.0 ; extra == 'dev'
 Requires-Dist: transifex-client ; extra == 'dev'
 Requires-Dist: responses >=0.9.0 ; extra == 'dev'
@@ -127,5 +125,3 @@ or `major` arguments.
 When the schema is updated, please update the docs, ideally including the
 diagrams. For the RDF namespace and JavaScript version of the model, 
 run `make generate`.
-
-

{followthemoney-3.5.9.dist-info → followthemoney-3.6.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-followthemoney/__init__.py,sha256=lD-aHuRdiV82zFlHH4I7iQDZHJ7J3pBrARbNi9I0PpY,360
+followthemoney/__init__.py,sha256=r3yXZluiZbP17mu2VZIJ5Q9_ItUb_kX81WIMtyWzaK0,360
 followthemoney/compare.py,sha256=1GFkCfTzA8QR0CH90kvySR8hvl9hQRUerW5Xw2Ivmpg,5134
 followthemoney/exc.py,sha256=ynZs_UnTVxHR-iBfat_CpVLraYzVX5yLtVf5Ti14hl4,734
 followthemoney/graph.py,sha256=VNDKrUBkz_-DmKsr5v-Xm8VfxzabnTwkU_MFk92_TjA,10848
@@ -8,15 +8,15 @@ followthemoney/model.py,sha256=p4Bk0XPi9z7SKZVgG1FqeM2i45xwAlDIkHwyyUcDKk0,6426
 followthemoney/namespace.py,sha256=qYplxKn5OO3zDMf3NItKwTwDsJOnski5JbeyhssqhR8,4434
 followthemoney/offshore.py,sha256=Pf0tx-7GyhIZRueshDRqPNlxkHfGstmW5yNDID5Mnws,1060
 followthemoney/ontology.py,sha256=7PEoUKISNpkRvVhuLeE3IE9ZiNtdR8mn9_kzZ9yF9l0,2986
-followthemoney/property.py,sha256=H5VEH4ZE6pAjqxhx4QoWUEKq2w2Pyx9RX5Qh0W9AUmU,7052
-followthemoney/proxy.py,sha256=cLMEGTRaliz0dMUYLQNC9jgGMTgsiT5iaEN9t5MNGzQ,19871
+followthemoney/property.py,sha256=EXU4kH6rOD9ubmuDXE-3LGc6gwQspW9_NjSacDb4xYA,7494
+followthemoney/proxy.py,sha256=YvnAUWtu_BCsK36N6xTcoI0Ffjn-gIyj-eDVhtsjoLA,20033
 followthemoney/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 followthemoney/rdf.py,sha256=_BzWiBc61UYjiyadvGKya4P4JlJfvZtNrn8SP2hC2JM,328
 followthemoney/schema.py,sha256=Tk61vRmWaawW-1HOmu-hbYZtkD8it0fIvNtSRxBEQxA,16774
 followthemoney/util.py,sha256=ec_n8Q_CVAfN4HkUnOZB6IRbMz-p4XXxxwTm1RzLYV4,4508
 followthemoney/cli/__init__.py,sha256=Fl05wMr5-FlOSMRpmu1HJSNfPRpy8u9as5IRbGXdo4U,421
 followthemoney/cli/aggregate.py,sha256=Gwfi5Bt1LCwqbpsCu4P1Cr-QJtCWhbaqgGEzfwJUUL4,2142
-followthemoney/cli/cli.py,sha256=0ie811Mrk6AbfKMUjP2Z_QdleRWTs360U9lxKKpfsBw,3345
+followthemoney/cli/cli.py,sha256=yrPw2iyKY-E-uRWe6KN9W3ayvz-22vfpe_ZeD0RiI0c,3591
 followthemoney/cli/exports.py,sha256=HsTyIOz1KQSeObp9-9SKzSUBW158XOhpU5_Stv_2HWM,4016
 followthemoney/cli/mapping.py,sha256=aEl57en0zu57yMA2AU5y01U5Yyqo_42hkMlAdfIQP08,3284
 followthemoney/cli/sieve.py,sha256=Wh1UQxzyM9Gh60ooS4s4ydlW1b69bMzFM08tg8ttSIY,1940
@@ -24,26 +24,26 @@ followthemoney/cli/util.py,sha256=CFcS-PEwpMasMWX_Yg283O_PaAhcPwkvahFNWc13C8c,47
 followthemoney/export/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 followthemoney/export/common.py,sha256=_YrXrwsqmyboDZDhtJ_PazUUJYe1Y-Trqc9lz4YlVR8,991
 followthemoney/export/csv.py,sha256=WPAeHMEKnQE1ZL1_I5qgxki-qKJ5gytur-sTOHAubW0,2635
-followthemoney/export/excel.py,sha256=-pj4GYIDtnqxSMV5WwWchLwb24UPcQ2cKvyTh4_ARKA,2714
+followthemoney/export/excel.py,sha256=pj6zNpIbye_Zm3vhCamcqHEe9Fw-RyjtWQDCFY6608s,2645
 followthemoney/export/graph.py,sha256=v0z1FgadyFk5aQ0A5q8E9R4fSO-Tpi5JU9YTDwnRKD8,2765
 followthemoney/export/neo4j.py,sha256=JCLb1eW5FFIJXqmnpNMRzRs1KYuYkCtHZp7KJcu-JNY,7104
 followthemoney/export/rdf.py,sha256=E6RiW7oIsJdaBaLAVm6o-MTokARZtqONPuayILqTqo0,786
 followthemoney/mapping/__init__.py,sha256=iwNqzzvrzJNbNDlOCaDLlBTUrNTlnYHIB5cvo_-9oN4,82
 followthemoney/mapping/csv.py,sha256=Tvc6VSh7-ca63LEE4G0yqOCeGMETkuKzUjIkVn4_d7Q,3185
-followthemoney/mapping/entity.py,sha256=1t9AkeMwX5zaKFC1ljEVXZkWHni7hTFXcw1-5u3ZZko,4785
-followthemoney/mapping/property.py,sha256=L6hOeDTx_zJRCNVFBuJesPgNFiBdaW2N3UNjclfB3Ss,4748
+followthemoney/mapping/entity.py,sha256=1Mj6qKkFN10IdElRmULZoqTWIAJ9OSk9wpPpmWBlvA0,5697
+followthemoney/mapping/property.py,sha256=41V16HJh6da7oKdSJWyRcyMkx2XFd6iDm9-4PH7Wihw,5036
 followthemoney/mapping/query.py,sha256=8M6bOlEX2p_bbVwEwTu_1slEtU0cfRJB7ajZp-F07CE,2622
 followthemoney/mapping/source.py,sha256=sri-XpSjeHZbQtqNcz1eJYvwVSBNqGO5JwhWswiEx3c,649
-followthemoney/mapping/sql.py,sha256=1s0qnEHP8KXjGLrrdcqX_Tb-r59MS-dcNB2ky0Eeves,4754
+followthemoney/mapping/sql.py,sha256=m3Ho8B2pFsg6q2zj-y55h0O3x9eb6wzjlPBQyZ6DVcI,4752
 followthemoney/schema/Address.yaml,sha256=xCKyak0qdJC0W5jfROUq6pEMCcSupNSB6GAxx5QfN5E,1689
 followthemoney/schema/Airplane.yaml,sha256=uijTw6-1exomw1clZoDcRvj34mUBmDxNtd6K7nX3XQ4,573
-followthemoney/schema/Analyzable.yaml,sha256=KplYTaca-CDGkg5TSrxZ7aGlSaI9ThTRKBdFkNk6OVQ,1191
+followthemoney/schema/Analyzable.yaml,sha256=WvyhjYqI7QzBt3qPDgO7MDasJH1qQtgQMToHkV_xtlU,1237
 followthemoney/schema/Article.yaml,sha256=WI8517oZINSLDmgwJiArWXTjZiK6hY8X3bMlMdwaOn8,260
 followthemoney/schema/Assessment.yaml,sha256=UGxJIIe8xf2ddGWaj0l-ui7KwWwbgRPCEWC3ZFAPrhU,624
 followthemoney/schema/Asset.yaml,sha256=xQhHJJtnGLfiOaUncqGv4JR3Yn7l46yAg7p0hKFAFVE,260
 followthemoney/schema/Associate.yaml,sha256=3u3t5bpDBSQ7p5jyrHTEM0GbhBaG1ar9UKN4uquvq9g,887
 followthemoney/schema/Audio.yaml,sha256=qEoWTw_WrY7EjkztMdRQ-DRA1DZRCreib2HkOlCqKn8,446
-followthemoney/schema/BankAccount.yaml,sha256=5hyvQi4VODjmxqEkexJjBT7QqTGM3iuRo5nczZWZYlg,1323
+followthemoney/schema/BankAccount.yaml,sha256=PwQWqEox-MrhBNBRiXLr-P_tJ1qyG3cHtUsUH0QbBZo,1387
 followthemoney/schema/Call.yaml,sha256=kbVCnVxucBrEplxehXHThLSJAJjy_GhWan-IeZZjr0M,980
 followthemoney/schema/CallForTenders.yaml,sha256=2IWonTzfSbrkynMoEWqv5fekUeFM_xDKpKIbRe1XDbo,3227
 followthemoney/schema/Company.yaml,sha256=7dWBPetd7xOJtu8AlhKsjxBYzYBK5E3LssYguSf0glA,3321
@@ -67,7 +67,7 @@ followthemoney/schema/Identification.yaml,sha256=XVvZXTZZUg8qDzTga8dgF3mJqK9eSDg
 followthemoney/schema/Image.yaml,sha256=MMrK4wQJBuN07STwTmsPjXXpYZB7fPt_hQtweSBM4m0,388
 followthemoney/schema/Interest.yaml,sha256=xPYobeuxFMFXUWlQuPvDjAsr3QQDPteRz68_rq7Dvf0,276
 followthemoney/schema/Interval.yaml,sha256=WtmSoBngajhpNLtDZ8Ocpdd0I0tLG-7A4afuHXxssow,1919
-followthemoney/schema/LegalEntity.yaml,sha256=I-Wej5ksDxbC13p11YpH_tuMuufPZNERYjgoiPO2wtU,3568
+followthemoney/schema/LegalEntity.yaml,sha256=p_ty9cXsv8f7TTuz644R5SEmVePW-Gbp0ZrHgCgDIyk,3645
 followthemoney/schema/License.yaml,sha256=9Ye5vGEBhi7ttGqf0DdAGCJCN3zz5HtGu52dCcmCsQk,452
 followthemoney/schema/Membership.yaml,sha256=TzU1A-hPVXRJcNkC78N-EPhZ_qUgO14IZ8lB9WQ63AM,696
 followthemoney/schema/Mention.yaml,sha256=nBeulR_Jm4x75aJ7yNF0TAVhHJqXQaEzOutLIn_YU-4,1086
@@ -81,7 +81,7 @@ followthemoney/schema/Page.yaml,sha256=sQt_CnVyjDVGVECLQoGYZH4hxpjdPhxVRz4XJW-_1
 followthemoney/schema/Pages.yaml,sha256=deUgHFxgC_lmfRon9K6JlCyfJ-5WdoTmLiU_lr393u0,444
 followthemoney/schema/Passport.yaml,sha256=VkGMSJ2rG3_trME4wci45qmk1f_NUHYOn2YoLzrdre0,730
 followthemoney/schema/Payment.yaml,sha256=WRBJuj9ljsxLBs-0g9Z9UD87uR1RTtuUiYnWOnKr1qA,1757
-followthemoney/schema/Person.yaml,sha256=Ucqh9RHbgdpHCLoqcN6xcY-yUShAF8HLI4QI2LKXWFc,1586
+followthemoney/schema/Person.yaml,sha256=N4qaVSwDs1V3fzTYTMwg57wOlgbkjaeZuPn0QbYICG8,1886
 followthemoney/schema/PlainText.yaml,sha256=AYShjP5T_YUQypeAGJT7qdXbhBT_KahBKnwoypAqcjQ,272
 followthemoney/schema/Position.yaml,sha256=k5xdmowJfxR-A93aWMqZ1Z3EuEHsTwvXefKyy0K9DtU,1303
 followthemoney/schema/Post.yaml,sha256=NRn3NZpcDUmFW-uUQP8YAYOMYpWirD7-wrljETNh-lA,1029
@@ -91,12 +91,12 @@ followthemoney/schema/PublicBody.yaml,sha256=QqL_awWLLabrgLo4gUqY99BCmeqz_h16Unr
 followthemoney/schema/RealEstate.yaml,sha256=R_KiyvfKRYgmkZUfmo8ANQUJNi-zIfdc34gQhTgn_3A,1357
 followthemoney/schema/Representation.yaml,sha256=gSLx9nc63PgJ8UjR8MpIwzwBm-dvRTgW5vnLPcm4H_0,778
 followthemoney/schema/Sanction.yaml,sha256=3q4Xo8xHU03hjWTDypJe_fVIboSa6D8KOurjxzNGSao,931
-followthemoney/schema/Security.yaml,sha256=o17-r2Z9Uq4L7-6u9BO8k4XIXCXp5E1SMI0fyr7IJ6E,1082
+followthemoney/schema/Security.yaml,sha256=Ix8yCF5sm6JQzitZR7bKb5_P48RNaW1PAF6FDot-ks8,1120
 followthemoney/schema/Similar.yaml,sha256=gD8rZEaPQWzU-rEfsKdn62uEucF3KxYBcPMoSdnxvME,817
 followthemoney/schema/Succession.yaml,sha256=cwz1CwPl8btfUEpmr1YxF6T2WQQuLUboaW9EVEuW2Ng,735
 followthemoney/schema/Table.yaml,sha256=VcgOA2wcczHhRCwObIO39yStoMuoPZYE0VlXrAH9xCQ,635
 followthemoney/schema/TaxRoll.yaml,sha256=5DXK_rkJimpy2ZLjxx2RntYiGPXiK8Iiphr6cJjUiag,740
-followthemoney/schema/Thing.yaml,sha256=uNfrr-49DcBVHI0KTqjMIONHnBruYXCiGLE3m_hAqBU,2420
+followthemoney/schema/Thing.yaml,sha256=72gInrI4VEWxUcld7w4by1QPI46F-NR8QZJ09JyUcnc,2438
 followthemoney/schema/Trip.yaml,sha256=nLQD_ApmVJ8D56Czl7K700hhNZjzFV9FOQ3NBSQDLiM,771
 followthemoney/schema/UnknownLink.yaml,sha256=a7BUbNbrsGylewJGg2fcSody_Kmt4eEw_IKIvgHIt8c,668
 followthemoney/schema/UserAccount.yaml,sha256=V1JWwwcggCO4G9ByJY9zlQ0uOVp8HQK2mRXwqaGJnBM,763
@@ -128,28 +128,28 @@ followthemoney/translations/ru/LC_MESSAGES/followthemoney.mo,sha256=K30zowxjvE3U
 followthemoney/translations/ru/LC_MESSAGES/followthemoney.po,sha256=Gva5A2VXKfFboAtQc313CJ1SGNmjhsLZiQCwf-1ZVt8,139453
 followthemoney/translations/tr/LC_MESSAGES/followthemoney.mo,sha256=3h8kg6ryzuV2xmcG6_I0HtS1atddTmIvkeuU3bJcH14,487
 followthemoney/translations/tr/LC_MESSAGES/followthemoney.po,sha256=2yqoZ4vXTpxcSEPwSlF29cXpo3ou4CgaOhIcaalswvA,89829
-followthemoney/types/__init__.py,sha256=OeC6w66C3GP6e_XnKwmfV1xPiX2CAg47rXPSE-5eI6Q,1729
+followthemoney/types/__init__.py,sha256=X9XeM6JktzirAw5gGkyDKGM70NuiJ9Tbjoq0IwVclxU,1745
 followthemoney/types/address.py,sha256=P6ctzojJks70RZIwk5Bhc9djxejZo4Te0m5pQ9M7bAM,1651
 followthemoney/types/checksum.py,sha256=2KLOoZiYUnkButMl_Q8_I4Ju7qfQX8PcdoyzwOf-Aj4,917
-followthemoney/types/common.py,sha256=ZsZMeLLjSmnqLzilP9yBy5FeFQcbMWGrOVn17-bGfgg,9560
+followthemoney/types/common.py,sha256=qPVj-jrDMEASWFYjl02NXoDO6lfPRQxE6dERLt_dmu8,9809
 followthemoney/types/country.py,sha256=1fAwIc8gqmP004Y22uILLJxBD3m552MoDwaEMGOD58o,3053
-followthemoney/types/date.py,sha256=Q-WbAT9Frddrc2nesj18sYqtx6n8LlWm0DzH-vUWMFg,2500
-followthemoney/types/email.py,sha256=WEh5a6BkpGf0xazS0b-g2Ok81BJlF8sN-sTctXEk07Q,2680
-followthemoney/types/entity.py,sha256=vIJHg7_GUtYaIGDzrUg8CmRzQFoNeYiHF8jl3PvKzF4,2366
+followthemoney/types/date.py,sha256=XWFSKPJLpo8JSfcleyaEXih0q65ydf_d42Y_rcWZ-3s,2663
+followthemoney/types/email.py,sha256=zAnMHwC_FZh7IagpDRhGZf-GfQR6uW8d-lZP4UCdGcM,2745
+followthemoney/types/entity.py,sha256=RCDD9j7MWA_dd0Ib33dl1E6PubsiS1yfVeIOsDxWYpY,2431
 followthemoney/types/gender.py,sha256=N1xghyYtb4WZh8s3ogF9kDTRtx367o3gRZWbG30QGI8,1800
-followthemoney/types/iban.py,sha256=tClGDKF2SkNVFy-wT0ngDshvV0TFBO6Z7Rzzg1H15Lo,1848
-followthemoney/types/identifier.py,sha256=fj2mxBjG4UTvSElRYI1IXLmt4AilERG5oDm9qp9dW4w,1472
-followthemoney/types/ip.py,sha256=Td14wbIxFJKqWf3IiQToKnUAywZG3C4J7NhefcxcLxQ,1327
+followthemoney/types/iban.py,sha256=C8GdfIdDl8Ip89VsZ_hk6v3tJUrRgab_wFLo9ff2lYo,1763
+followthemoney/types/identifier.py,sha256=tX-XepAPreP6u4dzMKyy5mp507NH71DdByYA7pXyKrw,2016
+followthemoney/types/ip.py,sha256=Z4s8mIlY5-IK9Oo13_YbL9lMQpY8SE_NAz0eqaonaYo,1392
 followthemoney/types/json.py,sha256=Hefwns1-ziJf310MWvdfX5ICkOgj9cnnMJuqq1e6qKY,1676
-followthemoney/types/language.py,sha256=O91yohcv76VJ2FVCY6KuHslfiAlhPERnqhk8MHyGOoQ,2613
-followthemoney/types/mimetype.py,sha256=kcPzPmO_CJPFR96R42YtmelOdRj1nqI2_sdDitftzVw,1351
-followthemoney/types/name.py,sha256=WlTb-wwnsd4-I9hyKm3d38sEf6tBa6_Yin4UaLWh_AI,2855
+followthemoney/types/language.py,sha256=4GT8_kqCwq4tOs7s9TffM7SSr9AaszAQsNJHK27fqVY,2612
+followthemoney/types/mimetype.py,sha256=EZ5hIdn-wosfLc-GjXDaOzevxaSXPbSPHDUJmPT1h0I,1355
+followthemoney/types/name.py,sha256=EbdHQz08qXYJk2DRqtQ4cbYasDuPHWAVziJ-MyrsBLI,1864
 followthemoney/types/number.py,sha256=9l5v3hcAoJ4V6jaBS_-Kc1gtdJlQrY9L98ElC5e_PLQ,943
-followthemoney/types/phone.py,sha256=rbBjj7a5cjGuc6QXy3VwsHbAtIdfkd5CrUkBvBF9Rqg,3955
+followthemoney/types/phone.py,sha256=cppWyfZfOFZoGNjjqpMuTgxSgC1j5JFMpPHMHZYvoDQ,4020
 followthemoney/types/registry.py,sha256=hvQ1oIWz_4PpyUnKA5azbaZCulNb5LCIPyC-AQYVVyw,1936
 followthemoney/types/string.py,sha256=paSCnFt9TJrCmelmw5-OiwCBu0rkqNh6iauteJIj26k,1130
-followthemoney/types/topic.py,sha256=VEvAcJh7uyBwRVUM7P5i0g8WhkcnTEmlj5QwYx50p1M,3442
-followthemoney/types/url.py,sha256=WzPjBZe6BnE1Wpj4HotqmeCYijh3iyMcbfErt6saeEU,2162
+followthemoney/types/topic.py,sha256=6kIXdlZNeE28MPbRXaDhSI1whNA7tpkTgFwU_o_gxds,3625
+followthemoney/types/url.py,sha256=6p3ctjh0S_VciPLYuyMPwvK9147rpkhWDd4f2kahe10,1470
 tests/export/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/export/test_csv.py,sha256=r0ACsWnGdLHIZgXvbshGBQ9An0v1dJGAWulRT-Xittc,1261
 tests/export/test_excel.py,sha256=ShLtaJ1-zgq-1rzbNoGJp1bVbRKZ-HcGtU8oJpKwjjY,1696
@@ -165,22 +165,22 @@ tests/types/test_countries.py,sha256=LiIiDQsrYIFQnD7Z_fwIlLkjyX1p23yVxqdpi3-7tDU
 tests/types/test_dates.py,sha256=-S0WOSVPdz93iMgaZ5QCEfe7FOmvFvrL44hcdg9VBDM,3774
 tests/types/test_emails.py,sha256=8CyyedWe--wlbEZ7lQVVLw75pjIcI6ZpeTFUFPqxPg8,1606
 tests/types/test_entity.py,sha256=Jf7LarSILlgdWUV7bmXIQM8yh1YgCkSkrp0rFenduto,819
-tests/types/test_iban.py,sha256=7_KybOYh4CbgFfWs8VXoKI8J1qJFoEEE9o629TilzZg,6481
-tests/types/test_identifiers.py,sha256=obVGC-0nILxUU33iNevlY9I_YDKmy2n1LcxHl583WT0,725
+tests/types/test_iban.py,sha256=4gj9NUPMgKAfB4V3NZpua-c7Q5MSOokQVg4gZGl_SmM,6510
+tests/types/test_identifiers.py,sha256=yP7iv5KdVbNA_iSfVb6MpbpcUAoE2ZJDxmf49lMOFFY,6472
 tests/types/test_ip.py,sha256=lHRqT-MyPWTgtfnWisFm7N5n6TncebDp2ZEiEBIsYBM,1043
 tests/types/test_json.py,sha256=wOTW8wDwfSd2uOok40b7Y2pZKeZnayMVI6wX3hvYjgs,772
 tests/types/test_languages.py,sha256=gWxkmpf48R7Yfk2o41KGkrYrAjA-ew8Rwa4TDHunqmc,947
 tests/types/test_mimetypes.py,sha256=YYIFKJU4DhP6JMyP_sNmtLpsQnOOiZ_rJkIeJ5cgtAk,445
-tests/types/test_names.py,sha256=S4QUZhbvnqhX397QNAZLcl6sd1Wg5zF1TfqV23k1j-s,1262
+tests/types/test_names.py,sha256=zFgEojnNWbXgkFiB2YdjL1o8-UiMrJxAMaP9XwFVLvY,1060
 tests/types/test_number.py,sha256=uIoKq1aEYuxzRxI0oB30yJnd9oxPJ0r03AWba9AIgFg,474
 tests/types/test_phones.py,sha256=_GTl-0r5hok8ovUJPRqP7k9kJYMVbanjYUMLrbzFb5U,1196
 tests/types/test_registry.py,sha256=IkrHjrEiIIPd4PdMFUg_jVh_gsE6CzGXIF8s3LIbnl0,296
 tests/types/test_topic.py,sha256=m35GSwTy-r77YACGDDac09nYLBIUua42Dy7z292m9EU,707
 tests/types/test_urls.py,sha256=JVmzJ0DKDBle_xLjmOHn0hgDTCpLmljrcfbOM1GSGRw,1170
-followthemoney-3.5.9.dist-info/LICENSE,sha256=Jln3uF70A9AQySyrP9JAW6sQTubLoKVQunvuJqISv7w,1098
-followthemoney-3.5.9.dist-info/METADATA,sha256=G1s1hNYIusJ-zE1ognjLBfIw_qQdxHN6PHlG7ejBB84,4558
-followthemoney-3.5.9.dist-info/WHEEL,sha256=-G_t0oGuE7UD0DrSpVZnq1hHMBV9DD2XkS5v7XpmTnk,110
-followthemoney-3.5.9.dist-info/entry_points.txt,sha256=ONq-BcCB8Cnk_K1nvWqJPKSRm6sDW0RibLFp0gGtz-k,594
-followthemoney-3.5.9.dist-info/namespace_packages.txt,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-followthemoney-3.5.9.dist-info/top_level.txt,sha256=O33_kGPehy6-69B_k01UgMK55_JOhwqe2de7ND23b-Y,21
-followthemoney-3.5.9.dist-info/RECORD,,
+followthemoney-3.6.0.dist-info/LICENSE,sha256=Jln3uF70A9AQySyrP9JAW6sQTubLoKVQunvuJqISv7w,1098
+followthemoney-3.6.0.dist-info/METADATA,sha256=WO2TZ3GZpkAnhwWOT8LYGaQJjbFIJQVEQc1c6HeBxRc,4494
+followthemoney-3.6.0.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
+followthemoney-3.6.0.dist-info/entry_points.txt,sha256=wpgi-5-jyqCy-yw0eYZbnhBuPja_q3RJmiEl7NymXtQ,593
+followthemoney-3.6.0.dist-info/namespace_packages.txt,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+followthemoney-3.6.0.dist-info/top_level.txt,sha256=O33_kGPehy6-69B_k01UgMK55_JOhwqe2de7ND23b-Y,21
+followthemoney-3.6.0.dist-info/RECORD,,

{followthemoney-3.5.9.dist-info → followthemoney-3.6.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.42.0)
+Generator: bdist_wheel (0.43.0)
 Root-Is-Purelib: true
 Tag: py2-none-any
 Tag: py3-none-any

{followthemoney-3.5.9.dist-info → followthemoney-3.6.0.dist-info}/entry_points.txt

@@ -15,4 +15,3 @@ gexf = followthemoney.cli.exports:export_gexf
 mapping = followthemoney.cli.mapping:run_mapping
 rdf = followthemoney.cli.exports:export_rdf
 sieve = followthemoney.cli.sieve:sieve
-

tests/types/test_iban.py

@@ -15,6 +15,7 @@ class IbansTest(unittest.TestCase):
         rdf = ibans.rdf("GB29NWBK60161331926819")
         assert "iban:GB29NWBK60161331926819" in rdf
         nid = ibans.node_id("gb29NWBK60161331926819")
+        assert nid is not None
         assert "iban:GB" in nid
 
     def test_domain_validity(self):
@@ -22,7 +23,7 @@ class IbansTest(unittest.TestCase):
         self.assertTrue(ibans.validate("GB29NWBK60161331926819"))
         self.assertFalse(ibans.validate("GB28 NWBK 6016 1331 9268 19"))
         self.assertFalse(ibans.validate("GB29NWBKN0161331926819"))
-        self.assertFalse(ibans.validate(None))
+        self.assertFalse(ibans.validate(""))
         self.assertTrue(ibans.validate("AL35202111090000000001234567"))
         self.assertTrue(ibans.validate("AD1400080001001234567890"))
         self.assertTrue(ibans.validate("AT483200000012345864"))

tests/types/test_identifiers.py

@@ -4,6 +4,9 @@ from followthemoney.types import registry
 
 identifiers = registry.identifier
 
+def iban_valid(text: str) -> bool:
+    return registry.identifier.validate(text, format='iban')
+
 
 class IdentifiersTest(unittest.TestCase):
     def test_parse(self):
@@ -21,3 +24,99 @@ class IdentifiersTest(unittest.TestCase):
         assert comp > 0, comp
         comp = identifiers.compare_sets(["9818700"], ["AS9818700"])
         assert comp > 0, comp
+        iban = "AE460090000000123456789"
+        comp = identifiers.compare_safe(iban, iban)
+        assert comp == 1, comp
+        
+    def test_iban_parse(self):
+        val = "GB29 NWBK 6016 1331 9268 19"
+        self.assertEqual(identifiers.clean(val), val)
+        self.assertEqual(
+            identifiers.clean(val, format='iban'),
+            "GB29NWBK60161331926819"
+        )
+
+    def test_specificity(self):
+        self.assertEqual(identifiers.specificity("VG21PACG0000000123456789"), 1)
+
+    def test_iban_validation(self):
+        self.assertTrue(iban_valid("GB29 NWBK 6016 1331 9268 19"))
+        self.assertTrue(iban_valid("GB29NWBK60161331926819"))
+        self.assertFalse(iban_valid("GB28 NWBK 6016 1331 9268 19"))
+        self.assertFalse(iban_valid("GB29NWBKN0161331926819"))
+        self.assertFalse(iban_valid(""))
+        self.assertTrue(iban_valid("AL35202111090000000001234567"))
+        self.assertTrue(iban_valid("AD1400080001001234567890"))
+        self.assertTrue(iban_valid("AT483200000012345864"))
+        self.assertTrue(iban_valid("AZ96AZEJ00000000001234567890"))
+        self.assertTrue(iban_valid("BH02CITI00001077181611"))
+        # self.assertTrue(iban_valid('BY86AKBB10100000002966000000'))
+        self.assertTrue(iban_valid("BE71096123456769"))
+        self.assertTrue(iban_valid("BA275680000123456789"))
+        self.assertTrue(iban_valid("BR1500000000000010932840814P2"))
+        self.assertTrue(iban_valid("BG18RZBB91550123456789"))
+        # self.assertTrue(iban_valid('CR37012600000123456789'))
+        self.assertTrue(iban_valid("HR1723600001101234565"))
+        self.assertTrue(iban_valid("CY21002001950000357001234567"))
+        self.assertTrue(iban_valid("CZ5508000000001234567899"))
+        self.assertTrue(iban_valid("DK9520000123456789"))
+        self.assertTrue(iban_valid("DO22ACAU00000000000123456789"))
+        # self.assertTrue(iban_valid('SV43ACAT00000000000000123123'))
+        self.assertTrue(iban_valid("EE471000001020145685"))
+        self.assertTrue(iban_valid("FO9264600123456789"))
+        self.assertTrue(iban_valid("FI1410093000123458"))
+        self.assertTrue(iban_valid("FR7630006000011234567890189"))
+        self.assertTrue(iban_valid("GE60NB0000000123456789"))
+        self.assertTrue(iban_valid("DE91100000000123456789"))
+        self.assertTrue(iban_valid("GI04BARC000001234567890"))
+        self.assertTrue(iban_valid("GR9608100010000001234567890"))
+        self.assertTrue(iban_valid("GL8964710123456789"))
+        self.assertTrue(iban_valid("GT20AGRO00000000001234567890"))
+        self.assertTrue(iban_valid("HU93116000060000000012345676"))
+        self.assertTrue(iban_valid("IS030001121234561234567890"))
+        # self.assertTrue(iban_valid('IQ20CBIQ861800101010500'))
+        self.assertTrue(iban_valid("IE64IRCE92050112345678"))
+        self.assertTrue(iban_valid("IL170108000000012612345"))
+        self.assertTrue(iban_valid("IT60X0542811101000000123456"))
+        self.assertTrue(iban_valid("JO71CBJO0000000000001234567890"))
+        self.assertTrue(iban_valid("KZ563190000012344567"))
+        self.assertTrue(iban_valid("XK051212012345678906"))
+        self.assertTrue(iban_valid("KW81CBKU0000000000001234560101"))
+        self.assertTrue(iban_valid("LV97HABA0012345678910"))
+        self.assertTrue(iban_valid("LB92000700000000123123456123"))
+        self.assertTrue(iban_valid("LI7408806123456789012"))
+        self.assertTrue(iban_valid("LT601010012345678901"))
+        self.assertTrue(iban_valid("LU120010001234567891"))
+        self.assertTrue(iban_valid("MK07200002785123453"))
+        self.assertTrue(iban_valid("MT31MALT01100000000000000000123"))
+        self.assertTrue(iban_valid("MR1300020001010000123456753"))
+        self.assertTrue(iban_valid("MU43BOMM0101123456789101000MUR"))
+        self.assertTrue(iban_valid("MD21EX000000000001234567"))
+        self.assertTrue(iban_valid("MC5810096180790123456789085"))
+        self.assertTrue(iban_valid("ME25505000012345678951"))
+        self.assertTrue(iban_valid("NL02ABNA0123456789"))
+        self.assertTrue(iban_valid("NO8330001234567"))
+        self.assertTrue(iban_valid("PK36SCBL0000001123456702"))
+        self.assertTrue(iban_valid("PS92PALS000000000400123456702"))
+        self.assertTrue(iban_valid("PL10105000997603123456789123"))
+        self.assertTrue(iban_valid("PT50002700000001234567833"))
+        self.assertTrue(iban_valid("QA54QNBA000000000000693123456"))
+        self.assertTrue(iban_valid("RO09BCYP0000001234567890"))
+        self.assertTrue(iban_valid("LC14BOSL123456789012345678901234"))
+        self.assertTrue(iban_valid("SM76P0854009812123456789123"))
+        self.assertTrue(iban_valid("ST23000200000289355710148"))
+        self.assertTrue(iban_valid("SA4420000001234567891234"))
+        self.assertTrue(iban_valid("RS35105008123123123173"))
+        self.assertTrue(iban_valid("SC52BAHL01031234567890123456USD"))
+        self.assertTrue(iban_valid("SK8975000000000012345671"))
+        self.assertTrue(iban_valid("SI56192001234567892"))
+        self.assertTrue(iban_valid("ES7921000813610123456789"))
+        self.assertTrue(iban_valid("SE1412345678901234567890"))
+        self.assertTrue(iban_valid("CH5604835012345678009"))
+        self.assertTrue(iban_valid("TL380080012345678910157"))
+        self.assertTrue(iban_valid("TN4401000067123456789123"))
+        self.assertTrue(iban_valid("TR320010009999901234567890"))
+        self.assertTrue(iban_valid("UA903052992990004149123456789"))
+        self.assertTrue(iban_valid("AE460090000000123456789"))
+        self.assertTrue(iban_valid("GB98MIDL07009312345678"))
+        self.assertTrue(iban_valid("VG21PACG0000000123456789"))

tests/types/test_names.py

@@ -20,13 +20,7 @@ class NamesTest(unittest.TestCase):
         self.assertEqual(names.pick(["Banana"]), "Banana")
         self.assertEqual(names.pick([]), None)
         values = ["Robert Smith", "Rob Smith", "Robert SMITH"]
-        self.assertEqual(names.pick(values), "Robert SMITH")
-
-        # handle dirty edgecases
-        values = ["", "(", "Peter"]
-        self.assertEqual(names.pick(values), "Peter")
-        values = ["", "("]
-        self.assertEqual(names.pick(values), None)
+        self.assertEqual(names.pick(values), "Robert Smith")
 
     def test_domain_validity(self):
         self.assertTrue(names.validate("huhu"))