followthemoney 4.0.3__py3-none-any.whl → 4.1.0__py3-none-any.whl

followthemoney/__init__.py

@@ -9,7 +9,7 @@ from followthemoney.statement import Statement, StatementEntity, SE
 from followthemoney.dataset import Dataset, DefaultDataset, DS
 from followthemoney.util import set_model_locale
 
-__version__ = "4.0.3"
+__version__ = "4.1.0"
 
 # Data model singleton
 model = Model.instance()

followthemoney/compare.py

@@ -1,7 +1,8 @@
 import math
 from itertools import islice, product
 from typing import Dict, Generator, Iterable, List, Optional
-from normality import normalize
+from normality import ascii_text
+from rigour.text.scripts import can_latinize
 from rigour.names import tokenize_name, remove_person_prefixes
 from rigour.names import replace_org_types_compare
 from followthemoney.exc import InvalidData
@@ -91,7 +92,7 @@ def _normalize_names(
     can_person = schema.is_a("LegalEntity") and not schema.is_a("Organization")
     can_org = schema.is_a("LegalEntity") and not schema.is_a("Person")
     for name in names:
-        plain = normalize(name, ascii=True)
+        plain = name.lower().strip()
         if plain is not None and plain not in seen:
             seen.add(plain)
             yield plain
@@ -102,6 +103,9 @@ def _normalize_names(
         if can_org:
             name = replace_org_types_compare(name)
         tokens = tokenize_name(name.lower())
+        for token in tokens:
+            if can_latinize(token):
+                token = ascii_text(token) or token
         fp = " ".join(sorted(tokens))
         if fp is not None and len(fp) > 6 and fp not in seen:
             seen.add(fp)

followthemoney/dataset/util.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-from normality import slugify
+from normality import slugify_text
 from typing import Annotated, Any
 from rigour.time import datetime_iso
 from pydantic import AfterValidator, BeforeValidator, HttpUrl, PlainSerializer
@@ -11,7 +11,7 @@ def dataset_name_check(value: str) -> str:
     """Check that the given value is a valid dataset name. This doesn't convert
     or clean invalid names, but raises an error if they are not compliant to
     force the user to fix an invalid name"""
-    if slugify(value, sep="_") != value:
+    if slugify_text(value, sep="_") != value:
         raise ValueError("Invalid %s: %r" % ("dataset name", value))
     return value
 

followthemoney/namespace.py

@@ -26,9 +26,9 @@ the server without compromising isolation.
 import hmac
 from typing import Any, Optional, Tuple, Union
 
-from followthemoney.types import registry
 from followthemoney.proxy import E
-from followthemoney.util import key_bytes, get_entity_id
+from followthemoney.types import registry
+from followthemoney.util import get_entity_id, key_bytes
 
 
 class Namespace(object):
@@ -69,9 +69,11 @@ class Namespace(object):
         digest.update(key_bytes(entity_id))
         return digest.hexdigest()
 
-    def sign(self, entity_id: str) -> Optional[str]:
+    def sign(self, entity_id: Optional[str]) -> Optional[str]:
         """Apply a namespace signature to an entity ID, removing any
         previous namespace marker."""
+        if entity_id is None:
+            return None
         parsed_id, _ = self.parse(entity_id)
         if not len(self.bname):
             return parsed_id

followthemoney/proxy.py

@@ -57,7 +57,7 @@ class EntityProxy(object):
         #: A unique identifier for this entity, usually a hashed natural key,
         #: a UUID, or a very simple slug. Can be signed using a
         #: :class:`~followthemoney.namespace.Namespace`.
-        self.id = str(data["id"]) if "id" in data else None
+        self.id = str(data["id"]) if data.get("id") else None
         if not cleaned:
             self.id = sanitize_text(self.id)
 

followthemoney/types/address.py

@@ -1,7 +1,6 @@
 import re
 from typing import Optional, TYPE_CHECKING
-from normality import slugify
-from normality.cleaning import collapse_spaces
+from normality import slugify_text, collapse_spaces
 from rigour.addresses import normalize_address
 from rigour.text.distance import levenshtein_similarity
 
@@ -39,7 +38,7 @@ class AddressType(PropertyType):
         address = self.LINE_BREAKS.sub(", ", text)
         address = self.COMMATA.sub(", ", address)
         collapsed = collapse_spaces(address)
-        if collapsed is None or not len(collapsed):
+        if collapsed is None:
             return None
         return collapsed
 
@@ -54,7 +53,10 @@ class AddressType(PropertyType):
         return dampen(10, 60, value)
 
     def node_id(self, value: str) -> Optional[str]:
-        slug = slugify(normalize_address(value))
+        normalized = normalize_address(value)
+        if normalized is None:
+            return None
+        slug = slugify_text(normalized)
         if slug is None:
             return None
-        return f"addr:{value}"
+        return f"addr:{slug}"

followthemoney/types/common.py

@@ -204,9 +204,10 @@ class PropertyType(object):
         return data
 
     def __eq__(self, other: Any) -> bool:
-        if not isinstance(other, PropertyType):
+        try:
+            return self.name == other.name  # type: ignore
+        except AttributeError:
             return False
-        return self.name == other.name
 
     def __hash__(self) -> int:
         return hash(self.name)

followthemoney/types/email.py

@@ -1,11 +1,14 @@
 import re
 import logging
 from typing import Optional, TYPE_CHECKING
-from urllib.parse import urlparse
-from normality.cleaning import strip_quotes
+from rigour.env import ENCODING
 
 from followthemoney.types.common import PropertyType
-from followthemoney.util import sanitize_text, defer as _
+from followthemoney.util import defer as _
+
+
+# Regex to filter out invalid emails from a CSV file:
+# csvgrep -c value -r '^(?![a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$)' contrib/statements_emails.csv > contrib/test_invalid_emails.csv
 
 log = logging.getLogger(__name__)
 
@@ -17,8 +20,9 @@ class EmailType(PropertyType):
     """Internet mail address (e.g. user@example.com). These are notoriously hard
     to validate, but we use an irresponsibly simple rule and hope for the best."""
 
-    REGEX_RAW = r"^[^@\s]+@[^@\s]+\.\w+$"
-    REGEX = re.compile(REGEX_RAW)
+    DOMAIN_RE = re.compile(r"^(?!-)(?:[a-z0-9-]{1,63}(?<!-)\.)+[a-z0-9-]{2,}$", re.U)
+    LOCAL_RE = re.compile(r"^[^<>()\[\]\,;:\?\s@\"]{1,64}$", re.U)
+
     name = "email"
     group = "emails"
     label = _("E-Mail Address")
@@ -35,18 +39,29 @@ class EmailType(PropertyType):
     #     except:
     #         return False
 
+    def clean_domain_part(self, domain: str) -> Optional[str]:
+        """Clean and normalize the domain part of the email."""
+        domain = domain.rstrip(".").lower()
+        try:
+            # Convert domain to IDNA encoding if it contains non-ASCII characters. This should
+            # be idempotent for domains that are already IDNA-encoded.
+            domain = domain.encode("idna").decode(ENCODING)
+
+            # Check if the domain matches the regex pattern, which requires labels to be
+            # alphanumeric and hyphenated, and the TLD to be at least two characters long.
+            if self.DOMAIN_RE.match(domain) is None:
+                return None
+
+            domain = domain.encode(ENCODING).decode("idna")
+            return domain
+        except UnicodeError:
+            return None
+
     def validate(
         self, value: str, fuzzy: bool = False, format: Optional[str] = None
     ) -> bool:
         """Check to see if this is a valid email address."""
-        # TODO: adopt email.utils.parseaddr
-        email = sanitize_text(value)
-        if email is None or not self.REGEX.match(email):
-            return False
-        _, domain = email.rsplit("@", 1)
-        if len(domain) < 4 or "." not in domain:
-            return False
-        return True
+        return self.clean_text(value, fuzzy=fuzzy, format=format) is not None
 
     def clean_text(
         self,
@@ -59,23 +74,24 @@ class EmailType(PropertyType):
 
         Returns None if this is not an email address.
         """
-        email = strip_quotes(text)
-        if email is None or not self.REGEX.match(email):
-            return None
-        mailbox, domain = email.rsplit("@", 1)
         # TODO: https://pypi.python.org/pypi/publicsuffix/
-        # handle URLs by extracting the domain name
-        domain = urlparse(domain).hostname or domain
-        domain = domain.lower()
-        domain = domain.rstrip(".")
-        # handle unicode
+        # handle URLs by extracting the domain names
+        # or TODO: adopt email.utils.parseaddr
+
+        # Remove mailto: prefix if present
+        email = text.strip()
+        if email.startswith("mailto:"):
+            email = email[7:]
+
         try:
-            domain = domain.encode("idna").decode("ascii")
-        except UnicodeError:
-            return None
-        if domain is not None and mailbox is not None:
-            return "@".join((mailbox, domain))
-        return None
+            local, domain = email.rsplit("@", 1)
+            """Clean and validate the local part of the email."""
+            if self.LOCAL_RE.match(local) is None:
+                return None
 
-    # def country_hint(self, value)
-    # TODO: do we want to use TLDs as country evidence?
+            domain_clean = self.clean_domain_part(domain)
+            if domain_clean is None:
+                return None
+            return f"{local}@{domain_clean}"
+        except ValueError:
+            return None

followthemoney/types/name.py

@@ -1,5 +1,5 @@
 from typing import TYPE_CHECKING, Optional, Sequence
-from normality import slugify
+from normality import slugify_text
 from normality.cleaning import collapse_spaces, strip_quotes
 from rigour.env import MAX_NAME_LENGTH
 from rigour.names import pick_name, tokenize_name
@@ -38,6 +38,8 @@ class NameType(PropertyType):
     ) -> Optional[str]:
         """Basic clean-up."""
         name = strip_quotes(text)
+        if name is None:
+            return None
         return collapse_spaces(name)
 
     def pick(self, values: Sequence[str]) -> Optional[str]:
@@ -61,7 +63,7 @@ class NameType(PropertyType):
         )
 
     def node_id(self, value: str) -> Optional[str]:
-        slug = slugify(value)
+        slug = slugify_text(value)
         if slug is None:
             return None
         return f"name:{slug}"

followthemoney/util.py

@@ -1,6 +1,7 @@
 import os
 import sys
 import logging
+import unicodedata
 from hashlib import sha1
 from babel import Locale
 from gettext import translation
@@ -8,7 +9,6 @@ from gettext import translation
 from threading import local
 from typing import cast, Dict, Any, List, Optional, TypeVar, Union
 from normality import stringify
-from normality.cleaning import compose_nfc
 from normality.cleaning import remove_unsafe_chars
 from normality.encoding import DEFAULT_ENCODING
 from banal import is_mapping, unique_list, ensure_list
@@ -69,9 +69,7 @@ def sanitize_text(value: Any, encoding: str = DEFAULT_ENCODING) -> Optional[str]
     if text is None:
         return None
     try:
-        text = compose_nfc(text)
-        if text is None:
-            return None
+        text = unicodedata.normalize("NFC", text)
     except (SystemError, Exception) as ex:
         log.warning("Cannot NFC text: %s", ex)
         return None

{followthemoney-4.0.3.dist-info → followthemoney-4.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: followthemoney
-Version: 4.0.3
+Version: 4.1.0
 Summary: A data model for anti corruption data modeling and analysis.
 Project-URL: Documentation, https://followthemoney.tech/
 Project-URL: Repository, https://github.com/opensanctions/followthemoney.git
@@ -41,7 +41,7 @@ Requires-Dist: banal<1.1.0,>=1.0.6
 Requires-Dist: click<9.0.0,>=8.0
 Requires-Dist: countrynames<2.0.0,>=1.13.0
 Requires-Dist: networkx<3.5,>=2.5
-Requires-Dist: normality<3.0.0,>=2.6.1
+Requires-Dist: normality<4.0.0,>=3.0.0
 Requires-Dist: openpyxl<4.0.0,>=3.0.5
 Requires-Dist: orjson<4.0,>=3.10.18
 Requires-Dist: phonenumbers<10.0.0,>=8.12.22
@@ -51,7 +51,7 @@ Requires-Dist: pytz>=2021.1
 Requires-Dist: pyyaml<7.0.0,>=5.0.0
 Requires-Dist: rdflib<7.2.0,>=6.2.0
 Requires-Dist: requests<3.0.0,>=2.21.0
-Requires-Dist: rigour<2.0.0,>=1.0.0
+Requires-Dist: rigour<2.0.0,>=1.1.0
 Requires-Dist: sqlalchemy[mypy]<3.0.0,>=2.0.0
 Provides-Extra: dev
 Requires-Dist: build; extra == 'dev'

{followthemoney-4.0.3.dist-info → followthemoney-4.1.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
-followthemoney/__init__.py,sha256=GmOdQXf1GougYiuFmDULXblg9PKnVRmTQkxLPGc1LgI,856
-followthemoney/compare.py,sha256=rtITMzJOXLDOSj7yKPfOxFaknIu6kRpiLDIM22zakpI,5619
+followthemoney/__init__.py,sha256=XG3sMmmgBvV0kMOTtFP0JSqRQnlTSqtDEwtQ4MFxkSs,856
+followthemoney/compare.py,sha256=bZlnj2VMoe67q4Lyq_VwS1a-EJnEK1kC8prbs8jyL9E,5774
 followthemoney/entity.py,sha256=9wLKE3iFapxRQWOs_OAMzK3wtklf2HXaHaMYydIInWE,3045
 followthemoney/exc.py,sha256=GyMgwY4QVm87hLevDfV7gM1MJsDqfNCi_UQw7F_A8X8,858
 followthemoney/graph.py,sha256=7X1CGHGvmktS2LSZqld2iXWzG7B831eCNYyBqamqEJ8,10921
@@ -7,13 +7,13 @@ followthemoney/helpers.py,sha256=Btb6BlHg_c-qCXZo-NP_LURKG-qu-QD3Fj1ev_c7Xic,795
 followthemoney/messages.py,sha256=zUEa9CFecU8nRafIzhN6TKCh1kEihiIyIS1qr8PxY4g,806
 followthemoney/model.py,sha256=bWFVNa-DhYzc8BdSXBZdG2ev6Nh9uHx6i4tin8DvEEU,7374
 followthemoney/names.py,sha256=LODQqExKEHdH4z6Mmbhlm0KeKRzGcptaSWzYXZ7lONI,1120
-followthemoney/namespace.py,sha256=cp7X8aGaZ8HHf7SOfHr2vJHPI2todz2DoyLdiZLNMyg,4472
+followthemoney/namespace.py,sha256=utggu9IGA8bhgEYom3OUB1KxkAJR_TrMNbY5MUF_db8,4536
 followthemoney/ontology.py,sha256=WWY_PYQGl5Ket4zZBuZglzQxD2Bh9UqHok6GJNNX7GA,3001
 followthemoney/property.py,sha256=RDTzTXJeeLFLptQL1_gr1S1T-vdDe-8MGMwsRaGQh0I,7665
-followthemoney/proxy.py,sha256=KhByvSQk1BrZxTinKjjUsbEuD96RveQQ4LRzQIz6pUA,19617
+followthemoney/proxy.py,sha256=LD4K1oPABXMX212UZxwLu7XOHRDyVBwTlqudTUsUZRQ,19619
 followthemoney/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 followthemoney/schema.py,sha256=WYnPE4Lego0pJHlojECEv0aO9Miw_YIvEb35HoDo4Zk,18087
-followthemoney/util.py,sha256=DhhcRilSetZpzvCew56AE6zNwenW5a4Y-KtmKM43rjc,4447
+followthemoney/util.py,sha256=QeNZI0rJPI2KcK2JQ-ka3rk_IobMkFSVN0jk-JJhAWI,4391
 followthemoney/value.py,sha256=BJ4Sj5Tg2kMrslR6FjQUr96d8Kt75U7ny9NgzVGT0ZE,2335
 followthemoney/cli/__init__.py,sha256=0mmz84uhXRp2qUn3syKnDXofU3MMAAe291s7htqX0Bg,187
 followthemoney/cli/aggregate.py,sha256=xQTFpU3cVVj7fplpX4OJVrRlTVpn6b9kBr_Vb87pKfg,2164
@@ -29,7 +29,7 @@ followthemoney/dataset/coverage.py,sha256=rBnKs7VngCtIuaDqrF5D0ygCHg8NAMkYbmtl73
 followthemoney/dataset/dataset.py,sha256=wWUzWsdzDW9qXLy8lS6Bpy08WMcaNU30oiMXU8jfo14,4724
 followthemoney/dataset/publisher.py,sha256=nexZe9XexV8WI5Id999vf5OH_DPUmiKQ_GT3c59eF44,893
 followthemoney/dataset/resource.py,sha256=S_-tNjMwHQ8LcSOsZO_xhXD-vLK90wyxtIRBbyCJ0Xo,1164
-followthemoney/dataset/util.py,sha256=ajUIBRF64dizdgy9LAp2abvFXRFOWCqQX9sDbToWFYo,1607
+followthemoney/dataset/util.py,sha256=mfVTXdbNnWly6cXo4SjNzHuJK1c1uNBwULYOVg1gK5I,1617
 followthemoney/export/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 followthemoney/export/common.py,sha256=5b-Qlu3MaA0kSzzMAP93FAWncpgiioENnCnHikWYxhs,1021
 followthemoney/export/csv.py,sha256=reWq1jYIv7sY2PEI4JwIxahYNNqnSiPfMCS3kQX4RZ8,2652
@@ -141,12 +141,12 @@ followthemoney/translations/ru/LC_MESSAGES/followthemoney.po,sha256=7SQWytOTvoAQ
 followthemoney/translations/tr/LC_MESSAGES/followthemoney.mo,sha256=SC84e_ZF_oFJG1NKdyZY_W6Kb6POORZB6wdeAcEWmnE,487
 followthemoney/translations/tr/LC_MESSAGES/followthemoney.po,sha256=AZC3marhtVVq8Ck1FOgnt4sbDMz548nX48O9GDwImbQ,89826
 followthemoney/types/__init__.py,sha256=rWwQeiuMh2BNIuvhpMfJ4bPADDvt9Axu1eedvNFi0qY,3350
-followthemoney/types/address.py,sha256=11k7kxHAoY46-ypkOQELFCJjrk7yssTOs55vdRWQJBY,2089
+followthemoney/types/address.py,sha256=8qapsxfAPFml9MyAcRxN5m_YkmkZYUUJyBXAiAAYUoE,2133
 followthemoney/types/checksum.py,sha256=zZrU8WX4CY3Vta_vOyfgDNzIwbmtje7AaDv3O1fBMnk,823
-followthemoney/types/common.py,sha256=d9t4BMqcjoMqqKu7UWVF6WKckV0FnRsKQ1LA3OgEq0c,10065
+followthemoney/types/common.py,sha256=4ks7zPT8rknrGSd4JFc1zRkS-TL4SX-25_ZbjcVDos0,10081
 followthemoney/types/country.py,sha256=mUCjwhUbA5Ef5HYuKb1KbH4aZ3MxaNwE1p77uOZMuG0,1745
 followthemoney/types/date.py,sha256=PjcaEyW6CBzf0-gHWKUsKjWIaD3AVBEl0zLSRQOVXxc,3105
-followthemoney/types/email.py,sha256=zuX7TthKGtkUwN_kEMMip17iMYQDJc5kQUP8j4N6R-A,2598
+followthemoney/types/email.py,sha256=L3RTYrMABlNQF7hCynXGfzoj6YNEHW5JAY_BwuhoZdA,3375
 followthemoney/types/entity.py,sha256=oDxVEhuxyU1ScpOpebPpUm3o0I9j_p7Qrq-t5yNpluQ,2338
 followthemoney/types/gender.py,sha256=fi9iKLbjAUxDCLBtU1MxWidxv7KgCY2eH5746FYlEGk,1725
 followthemoney/types/identifier.py,sha256=hzD188FtwG0w3TcmbnDwnUMc8MZVcWgQJKGAvrwygc4,2296
@@ -154,14 +154,14 @@ followthemoney/types/ip.py,sha256=mMFTODFiXAJROCUYJvoLAShyIiTIWVmMBh5zT_GquYM,13
 followthemoney/types/json.py,sha256=V3qJD5RxJykNX51u3w1Nx9xqoNBnkulhzkJI9XMYKFo,1690
 followthemoney/types/language.py,sha256=SXgRRH-DyPmyyrqYurSyMiG6WHB8a0Gw81XxroEGD-c,2747
 followthemoney/types/mimetype.py,sha256=NdpqVLx3Bre_myYvnbjmdd5wZBf01tllrbhegjO8_m0,1263
-followthemoney/types/name.py,sha256=Y7g-SkmO_9PgzpBubS79OsPhp4h73dEGVql78Bq2x0Y,2258
+followthemoney/types/name.py,sha256=1CzwNRgbvNpE5C9hm3PbMmJILCbpTDey5l3eL--bkTQ,2317
 followthemoney/types/number.py,sha256=OdVuHDd4IYIIHhx_317JKeMjBAGtsJ2TAcxoZKZ4MkY,3948
 followthemoney/types/phone.py,sha256=r8uRqWinS0CYnYBTs405k5gO4jeatUDgjdzzijoMKJE,3811
 followthemoney/types/string.py,sha256=fqyTauAm4mNnNaoH-yH087RBbNh-G5ZZUO3awTGQUUg,1230
 followthemoney/types/topic.py,sha256=CS5IoI8gm4MSVxfV6K4mGd20_tT1SaKMkcOt_ObSsAg,3678
 followthemoney/types/url.py,sha256=QFpS_JIV8unFHuh_uGv22SWUUkocBoOpzLsAJWom_gI,1455
-followthemoney-4.0.3.dist-info/METADATA,sha256=yUOxdiPQceDy0QyIYrvGxXJq4CYCplr-fSFipACeg1s,6791
-followthemoney-4.0.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-followthemoney-4.0.3.dist-info/entry_points.txt,sha256=caoFTlf213jhg5sz3TNSofutjUTzaKtWATuSIdd9Cps,653
-followthemoney-4.0.3.dist-info/licenses/LICENSE,sha256=H6_EVXisnJC0-18CjXIaqrBSFq_VH3OnS7u3dccOv6g,1148
-followthemoney-4.0.3.dist-info/RECORD,,
+followthemoney-4.1.0.dist-info/METADATA,sha256=aHjJGapyWo-mWOU29zQqjpGS2szD62G2D36wAraAYTE,6791
+followthemoney-4.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+followthemoney-4.1.0.dist-info/entry_points.txt,sha256=caoFTlf213jhg5sz3TNSofutjUTzaKtWATuSIdd9Cps,653
+followthemoney-4.1.0.dist-info/licenses/LICENSE,sha256=H6_EVXisnJC0-18CjXIaqrBSFq_VH3OnS7u3dccOv6g,1148
+followthemoney-4.1.0.dist-info/RECORD,,