followthemoney 4.1.0__py3-none-any.whl → 4.1.1__py3-none-any.whl

followthemoney/__init__.py

@@ -9,7 +9,7 @@ from followthemoney.statement import Statement, StatementEntity, SE
 from followthemoney.dataset import Dataset, DefaultDataset, DS
 from followthemoney.util import set_model_locale
 
-__version__ = "4.1.0"
+__version__ = "4.1.1"
 
 # Data model singleton
 model = Model.instance()

followthemoney/entity.py

@@ -4,7 +4,7 @@ from rigour.names import pick_name
 
 from followthemoney.proxy import EntityProxy
 from followthemoney.schema import Schema
-from followthemoney.statement.util import BASE_ID
+from followthemoney.statement import BASE_ID, Statement
 
 VE = TypeVar("VE", bound="ValueEntity")
 
@@ -38,11 +38,14 @@ class ValueEntity(EntityProxy):
         # add data from statement dict if present.
         # this updates the dataset and referents set
         for stmt_data in data.pop("statements", []):
-            self.datasets.add(stmt_data["dataset"])
-            if stmt_data["entity_id"] != self.id:
-                self.referents.add(stmt_data["entity_id"])
-            if stmt_data["prop"] != BASE_ID:
-                self.add(stmt_data["prop"], stmt_data["value"])
+            stmt = Statement.from_dict(stmt_data)
+            self.datasets.add(stmt.dataset)
+            if stmt.schema != self.schema.name:
+                self.schema = schema.model.common_schema(self.schema, stmt.schema)
+            if stmt.entity_id != self.id:
+                self.referents.add(stmt.entity_id)
+            if stmt.prop != BASE_ID:
+                self.add(stmt.prop, stmt.value)
 
     def merge(self: VE, other: EntityProxy) -> VE:
         merged = super().merge(other)

followthemoney/statement/__init__.py

@@ -3,6 +3,7 @@ from followthemoney.statement.serialize import CSV, JSON, PACK, FORMATS
 from followthemoney.statement.serialize import write_statements
 from followthemoney.statement.serialize import read_statements, read_path_statements
 from followthemoney.statement.entity import SE, StatementEntity
+from followthemoney.statement.util import BASE_ID
 
 __all__ = [
     "Statement",
@@ -13,6 +14,7 @@ __all__ = [
     "JSON",
     "PACK",
     "FORMATS",
+    "BASE_ID",
     "write_statements",
     "read_statements",
     "read_path_statements",

followthemoney/statement/entity.py

@@ -363,39 +363,49 @@ class StatementEntity(EntityProxy):
         self.extra_referents.update(other.extra_referents)
         return self
 
-    def to_dict(self) -> Dict[str, Any]:
+    def to_context_dict(self) -> Dict[str, Any]:
+        """Return a dictionary representation of the entity for context."""
         data: Dict[str, Any] = {
             "id": self.id,
             "caption": self.caption,
             "schema": self.schema.name,
-            "properties": self.properties,
-            "referents": list(self.referents),
-            "datasets": list(self.datasets),
         }
-        if self.first_seen is not None:
-            data["first_seen"] = self.first_seen
-        if self.last_seen is not None:
-            data["last_seen"] = self.last_seen
+        referents: Set[Optional[str]] = set(self.extra_referents)
+        datasets = set(self.datasets)
+        first_seen = None
+        last_seen = None
+        for stmts in self._statements.values():
+            for stmt in stmts:
+                if stmt.first_seen is not None:
+                    if first_seen is None or stmt.first_seen < first_seen:
+                        first_seen = stmt.first_seen
+                if stmt.last_seen is not None:
+                    if last_seen is None or stmt.last_seen > last_seen:
+                        last_seen = stmt.last_seen
+                if stmt.entity_id is not None and stmt.entity_id != self.id:
+                    referents.add(stmt.entity_id)
+                datasets.add(stmt.dataset)
+
+        data["referents"] = list(referents)
+        data["datasets"] = list(datasets)
+
+        if first_seen is not None:
+            data["first_seen"] = first_seen
+        if last_seen is not None:
+            data["last_seen"] = last_seen
         if self.last_change is not None:
             data["last_change"] = self.last_change
         return data
 
+    def to_dict(self) -> Dict[str, Any]:
+        data = self.to_context_dict()
+        data["properties"] = self.properties
+        return data
+
     def to_statement_dict(self) -> Dict[str, Any]:
         """Return a dictionary representation of the entity's statements."""
-        data: Dict[str, Any] = {
-            "id": self.id,
-            "caption": self.caption,
-            "schema": self.schema.name,
-            "statements": [stmt.to_dict() for stmt in self.statements],
-            "referents": list(self.referents),
-            "datasets": list(self.datasets),
-        }
-        if self.first_seen is not None:
-            data["first_seen"] = self.first_seen
-        if self.last_seen is not None:
-            data["last_seen"] = self.last_seen
-        if self.last_change is not None:
-            data["last_change"] = self.last_change
+        data = self.to_context_dict()
+        data["statements"] = [stmt.to_dict() for stmt in self.statements]
         return data
 
     def __len__(self) -> int:

followthemoney/types/address.py

@@ -1,6 +1,6 @@
 import re
 from typing import Optional, TYPE_CHECKING
-from normality import slugify_text, collapse_spaces
+from normality import slugify_text, squash_spaces
 from rigour.addresses import normalize_address
 from rigour.text.distance import levenshtein_similarity
 
@@ -37,8 +37,8 @@ class AddressType(PropertyType):
         """Basic clean-up."""
         address = self.LINE_BREAKS.sub(", ", text)
         address = self.COMMATA.sub(", ", address)
-        collapsed = collapse_spaces(address)
-        if collapsed is None:
+        collapsed = squash_spaces(address)
+        if len(collapsed) < 1:
             return None
         return collapsed
 

followthemoney/types/name.py

@@ -1,6 +1,6 @@
 from typing import TYPE_CHECKING, Optional, Sequence
 from normality import slugify_text
-from normality.cleaning import collapse_spaces, strip_quotes
+from normality.cleaning import squash_spaces, strip_quotes
 from rigour.env import MAX_NAME_LENGTH
 from rigour.names import pick_name, tokenize_name
 from rigour.text.distance import levenshtein_similarity
@@ -40,7 +40,10 @@ class NameType(PropertyType):
         name = strip_quotes(text)
         if name is None:
             return None
-        return collapse_spaces(name)
+        name = squash_spaces(name)
+        if len(name) == 0:
+            return None
+        return name
 
     def pick(self, values: Sequence[str]) -> Optional[str]:
         """From a set of names, pick the most plausible user-facing one."""

followthemoney/util.py

@@ -74,10 +74,11 @@ def sanitize_text(value: Any, encoding: str = DEFAULT_ENCODING) -> Optional[str]
         log.warning("Cannot NFC text: %s", ex)
         return None
     text = remove_unsafe_chars(text)
-    if text is None:
-        return None
     byte_text = text.encode(DEFAULT_ENCODING, "replace")
-    return byte_text.decode(DEFAULT_ENCODING, "replace")
+    text = byte_text.decode(DEFAULT_ENCODING, "replace")
+    if len(text) == 0:
+        return None
+    return text
 
 
 def key_bytes(key: Any) -> bytes:

{followthemoney-4.1.0.dist-info → followthemoney-4.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: followthemoney
-Version: 4.1.0
+Version: 4.1.1
 Summary: A data model for anti corruption data modeling and analysis.
 Project-URL: Documentation, https://followthemoney.tech/
 Project-URL: Repository, https://github.com/opensanctions/followthemoney.git
@@ -41,7 +41,7 @@ Requires-Dist: banal<1.1.0,>=1.0.6
 Requires-Dist: click<9.0.0,>=8.0
 Requires-Dist: countrynames<2.0.0,>=1.13.0
 Requires-Dist: networkx<3.5,>=2.5
-Requires-Dist: normality<4.0.0,>=3.0.0
+Requires-Dist: normality<4.0.0,>=3.0.1
 Requires-Dist: openpyxl<4.0.0,>=3.0.5
 Requires-Dist: orjson<4.0,>=3.10.18
 Requires-Dist: phonenumbers<10.0.0,>=8.12.22
@@ -51,7 +51,7 @@ Requires-Dist: pytz>=2021.1
 Requires-Dist: pyyaml<7.0.0,>=5.0.0
 Requires-Dist: rdflib<7.2.0,>=6.2.0
 Requires-Dist: requests<3.0.0,>=2.21.0
-Requires-Dist: rigour<2.0.0,>=1.1.0
+Requires-Dist: rigour<2.0.0,>=1.1.1
 Requires-Dist: sqlalchemy[mypy]<3.0.0,>=2.0.0
 Provides-Extra: dev
 Requires-Dist: build; extra == 'dev'

{followthemoney-4.1.0.dist-info → followthemoney-4.1.1.dist-info}/RECORD

@@ -1,6 +1,6 @@
-followthemoney/__init__.py,sha256=XG3sMmmgBvV0kMOTtFP0JSqRQnlTSqtDEwtQ4MFxkSs,856
+followthemoney/__init__.py,sha256=_drb_fsELoJ6qpcyjNSJbn8OVaBH-ZdeDQKaFJ1S_Qk,856
 followthemoney/compare.py,sha256=bZlnj2VMoe67q4Lyq_VwS1a-EJnEK1kC8prbs8jyL9E,5774
-followthemoney/entity.py,sha256=9wLKE3iFapxRQWOs_OAMzK3wtklf2HXaHaMYydIInWE,3045
+followthemoney/entity.py,sha256=hHY9yysn_iFTtXqcHg4hHhYfmgLw4prWul8rD1X82Y0,3184
 followthemoney/exc.py,sha256=GyMgwY4QVm87hLevDfV7gM1MJsDqfNCi_UQw7F_A8X8,858
 followthemoney/graph.py,sha256=7X1CGHGvmktS2LSZqld2iXWzG7B831eCNYyBqamqEJ8,10921
 followthemoney/helpers.py,sha256=Btb6BlHg_c-qCXZo-NP_LURKG-qu-QD3Fj1ev_c7Xic,7956
@@ -13,7 +13,7 @@ followthemoney/property.py,sha256=RDTzTXJeeLFLptQL1_gr1S1T-vdDe-8MGMwsRaGQh0I,76
 followthemoney/proxy.py,sha256=LD4K1oPABXMX212UZxwLu7XOHRDyVBwTlqudTUsUZRQ,19619
 followthemoney/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 followthemoney/schema.py,sha256=WYnPE4Lego0pJHlojECEv0aO9Miw_YIvEb35HoDo4Zk,18087
-followthemoney/util.py,sha256=QeNZI0rJPI2KcK2JQ-ka3rk_IobMkFSVN0jk-JJhAWI,4391
+followthemoney/util.py,sha256=LoCSp1iE6VwXjotCkBXFRppeQs55726GzOuNIu3CvRE,4409
 followthemoney/value.py,sha256=BJ4Sj5Tg2kMrslR6FjQUr96d8Kt75U7ny9NgzVGT0ZE,2335
 followthemoney/cli/__init__.py,sha256=0mmz84uhXRp2qUn3syKnDXofU3MMAAe291s7htqX0Bg,187
 followthemoney/cli/aggregate.py,sha256=xQTFpU3cVVj7fplpX4OJVrRlTVpn6b9kBr_Vb87pKfg,2164
@@ -112,8 +112,8 @@ followthemoney/schema/Vehicle.yaml,sha256=Ypl4A5HJFOZfZh3DK0ewN-hyJuCMcovR0mPNdd
 followthemoney/schema/Vessel.yaml,sha256=nFaUJ_0BzFJstvog1iDvwV9DHKHr9ky4DLb1NZGGh1E,1096
 followthemoney/schema/Video.yaml,sha256=LY3DYMWTHXiAhL0hxBCNCz50cp2sPbUlEhhig5Fbjos,327
 followthemoney/schema/Workbook.yaml,sha256=iikWPElz4klA7SkWH7eae6xqhbkMCIP_3zdeXzFEMU0,354
-followthemoney/statement/__init__.py,sha256=PvhLPhmQrezBKCe8rEwJlyTWlrnCzSfyfchVc8gXXEA,568
-followthemoney/statement/entity.py,sha256=92tOai7Yt5GZkOylZcy7866P0iLJsYEzmt-2T7WbXMg,15540
+followthemoney/statement/__init__.py,sha256=7m2VUCAuqNZXIY0WFJRFkw5UG14QuxATL4f_xbqKwhw,633
+followthemoney/statement/entity.py,sha256=gXvBTBwHL-GfmBXdw9HoP6WSbiwBOrTTOllQwkIObyQ,15986
 followthemoney/statement/serialize.py,sha256=9eXzQ1biR2mSxWRID5C7xDdku4b4ZImHeRJ53yLZ0yo,7225
 followthemoney/statement/statement.py,sha256=Ae-EYuzS8S12BkaRqrvMuI1C7YwlRKa5C_pTBELyNMM,8029
 followthemoney/statement/util.py,sha256=B-ozuRc1TWvpop52873Pqt5OPj8H6uk4KyRJLfAhr10,780
@@ -141,7 +141,7 @@ followthemoney/translations/ru/LC_MESSAGES/followthemoney.po,sha256=7SQWytOTvoAQ
 followthemoney/translations/tr/LC_MESSAGES/followthemoney.mo,sha256=SC84e_ZF_oFJG1NKdyZY_W6Kb6POORZB6wdeAcEWmnE,487
 followthemoney/translations/tr/LC_MESSAGES/followthemoney.po,sha256=AZC3marhtVVq8Ck1FOgnt4sbDMz548nX48O9GDwImbQ,89826
 followthemoney/types/__init__.py,sha256=rWwQeiuMh2BNIuvhpMfJ4bPADDvt9Axu1eedvNFi0qY,3350
-followthemoney/types/address.py,sha256=8qapsxfAPFml9MyAcRxN5m_YkmkZYUUJyBXAiAAYUoE,2133
+followthemoney/types/address.py,sha256=nMFCj5QJyqA1ddpUmDLpRTum0nGXE-J70_WGnaLXnYo,2130
 followthemoney/types/checksum.py,sha256=zZrU8WX4CY3Vta_vOyfgDNzIwbmtje7AaDv3O1fBMnk,823
 followthemoney/types/common.py,sha256=4ks7zPT8rknrGSd4JFc1zRkS-TL4SX-25_ZbjcVDos0,10081
 followthemoney/types/country.py,sha256=mUCjwhUbA5Ef5HYuKb1KbH4aZ3MxaNwE1p77uOZMuG0,1745
@@ -154,14 +154,14 @@ followthemoney/types/ip.py,sha256=mMFTODFiXAJROCUYJvoLAShyIiTIWVmMBh5zT_GquYM,13
 followthemoney/types/json.py,sha256=V3qJD5RxJykNX51u3w1Nx9xqoNBnkulhzkJI9XMYKFo,1690
 followthemoney/types/language.py,sha256=SXgRRH-DyPmyyrqYurSyMiG6WHB8a0Gw81XxroEGD-c,2747
 followthemoney/types/mimetype.py,sha256=NdpqVLx3Bre_myYvnbjmdd5wZBf01tllrbhegjO8_m0,1263
-followthemoney/types/name.py,sha256=1CzwNRgbvNpE5C9hm3PbMmJILCbpTDey5l3eL--bkTQ,2317
+followthemoney/types/name.py,sha256=ZWGDebv01qByh_yBYOVoS3Edlm3_JVPShQMklKc6ZOA,2384
 followthemoney/types/number.py,sha256=OdVuHDd4IYIIHhx_317JKeMjBAGtsJ2TAcxoZKZ4MkY,3948
 followthemoney/types/phone.py,sha256=r8uRqWinS0CYnYBTs405k5gO4jeatUDgjdzzijoMKJE,3811
 followthemoney/types/string.py,sha256=fqyTauAm4mNnNaoH-yH087RBbNh-G5ZZUO3awTGQUUg,1230
 followthemoney/types/topic.py,sha256=CS5IoI8gm4MSVxfV6K4mGd20_tT1SaKMkcOt_ObSsAg,3678
 followthemoney/types/url.py,sha256=QFpS_JIV8unFHuh_uGv22SWUUkocBoOpzLsAJWom_gI,1455
-followthemoney-4.1.0.dist-info/METADATA,sha256=aHjJGapyWo-mWOU29zQqjpGS2szD62G2D36wAraAYTE,6791
-followthemoney-4.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-followthemoney-4.1.0.dist-info/entry_points.txt,sha256=caoFTlf213jhg5sz3TNSofutjUTzaKtWATuSIdd9Cps,653
-followthemoney-4.1.0.dist-info/licenses/LICENSE,sha256=H6_EVXisnJC0-18CjXIaqrBSFq_VH3OnS7u3dccOv6g,1148
-followthemoney-4.1.0.dist-info/RECORD,,
+followthemoney-4.1.1.dist-info/METADATA,sha256=OatNsAWxjixfh_s-iGY046rcZRoCs9p-6rPLwB8VxTM,6791
+followthemoney-4.1.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+followthemoney-4.1.1.dist-info/entry_points.txt,sha256=caoFTlf213jhg5sz3TNSofutjUTzaKtWATuSIdd9Cps,653
+followthemoney-4.1.1.dist-info/licenses/LICENSE,sha256=H6_EVXisnJC0-18CjXIaqrBSFq_VH3OnS7u3dccOv6g,1148
+followthemoney-4.1.1.dist-info/RECORD,,