followthemoney 4.5.1__py3-none-any.whl → 4.5.2__py3-none-any.whl

followthemoney/__init__.py

@@ -9,7 +9,7 @@ from followthemoney.statement import Statement, StatementEntity, SE
 from followthemoney.dataset import Dataset, UndefinedDataset, DS
 from followthemoney.util import set_model_locale
 
-__version__ = "4.5.1"
+__version__ = "4.5.2"
 
 # Data model singleton
 model = Model.instance()

followthemoney/proxy.py

@@ -1,6 +1,6 @@
 import hashlib
 import logging
-from typing import TYPE_CHECKING, cast, Any
+from typing import TYPE_CHECKING, Iterable, Any, cast
 from typing import Dict, Generator, List, Optional, Set, Tuple, Union, Type, TypeVar
 from itertools import product
 from banal import ensure_dict
@@ -117,11 +117,28 @@ class EntityProxy(object):
             an empty list instead of raising an error.
         :return: A list of values.
         """
+        if prop in self.schema.properties:
+            return self._properties.get(prop, [])  # type: ignore
+
         prop_name = self._prop_name(prop, quiet=quiet)
         if prop_name is None:
             return []
         return self._properties.get(prop_name, [])
 
+    def get_prop(self, prop: Property) -> Iterable[str]:
+        """Get all values of a property, returning an empty list if the property
+        does not exist. This has better performance characteristics than `get()`
+        as it does not need to resolve the property name.
+
+        :param prop: can be given as a name or an instance of
+            :class:`~followthemoney.property.Property`.
+        :return: An iterable of values.
+        """
+        try:
+            return self._properties[prop.name]
+        except KeyError:
+            return []
+
     def first(self, prop: P, quiet: bool = False) -> Optional[str]:
         """Get only the first value set for the property.
 
@@ -311,10 +328,11 @@ class EntityProxy(object):
         combined = set()
         for prop_name, values in self._properties.items():
             prop = self.schema.properties[prop_name]
+            if prop.type is not type_:
+                continue
             if matchable and not prop.matchable:
                 continue
-            if prop.type == type_:
-                combined.update(values)
+            combined.update(values)
         return list(combined)
 
     @property

followthemoney/statement/entity.py

@@ -184,6 +184,13 @@ class StatementEntity(EntityProxy):
             return []
         return list({s.value for s in self._statements[prop_name]})
 
+    def get_prop(self, prop: Property) -> Iterable[str]:
+        try:
+            statements = self._statements[prop.name]
+            return {s.value for s in statements}
+        except KeyError:
+            return []
+
     def get_statements(self, prop: P, quiet: bool = False) -> List[Statement]:
         prop_name = self._prop_name(prop, quiet=quiet)
         if prop_name is None or prop_name not in self._statements:
@@ -341,14 +348,16 @@ class StatementEntity(EntityProxy):
     def get_type_statements(
         self, type_: PropertyType, matchable: bool = False
     ) -> List[Statement]:
-        combined = []
+        combined: List[Statement] = []
         for prop_name, statements in self._statements.items():
             prop = self.schema.properties[prop_name]
+            # Used in performance-critical code paths:
+            if prop.type is not type_:
+                continue
             if matchable and not prop.matchable:
                 continue
-            if prop.type == type_:
-                for statement in statements:
-                    combined.append(statement)
+            for statement in statements:
+                combined.append(statement)
         return combined
 
     @property

followthemoney/statement/serialize.py

@@ -1,5 +1,4 @@
 import csv
-import sys
 import click
 import orjson
 import logging
@@ -13,6 +12,7 @@ from rigour.env import ENCODING
 
 from followthemoney.statement.statement import Statement, StatementDict
 from followthemoney.statement.util import unpack_prop
+from followthemoney.util import ENTITY_VALUE_MAX, PROP_VALUE_MAX
 
 log = logging.getLogger(__name__)
 
@@ -50,12 +50,12 @@ LEGACY_PACK_COLUMNS = [
     "first_seen",
     "last_seen",
 ]
-csv.field_size_limit(sys.maxsize)
+csv.field_size_limit(PROP_VALUE_MAX)
 
 
 def read_json_statements(
     fh: BinaryIO,
-    max_line: int = 40 * 1024 * 1024,
+    max_line: int = ENTITY_VALUE_MAX,
 ) -> Generator[Statement, None, None]:
     while line := fh.readline(max_line):
         data = orjson.loads(line)

followthemoney/types/common.py

@@ -1,7 +1,6 @@
 from inspect import cleandoc
 from itertools import product
 from babel.core import Locale
-from banal import ensure_list
 from normality import stringify
 from typing import Any, Dict, Optional, Sequence, Callable, TYPE_CHECKING, TypedDict
 
@@ -115,7 +114,6 @@ class PropertyType(object):
         """Helper function for converting multi-valued FtM data into formats that
         allow only a single value per field (e.g. CSV). This is not fully reversible
         and should be used as a last option."""
-        values = ensure_list(values)
         return "; ".join(values)
 
     def _specificity(self, value: str) -> float:
@@ -154,7 +152,7 @@ class PropertyType(object):
     ) -> float:
         """Compare two sets of values and select the highest-scored result."""
         results = []
-        for le, ri in product(ensure_list(left), ensure_list(right)):
+        for le, ri in product(left, right):
             results.append(self.compare(le, ri))
         if not len(results):
             return 0.0

followthemoney/types/json.py

@@ -1,6 +1,5 @@
 import json
 from typing import Any, Optional, Sequence, TYPE_CHECKING
-from banal import ensure_list
 
 from followthemoney.types.common import PropertyType
 from followthemoney.util import sanitize_text, defer as _
@@ -48,7 +47,7 @@ class JsonType(PropertyType):
 
     def join(self, values: Sequence[str]) -> str:
         """Turn multiple values into a JSON array."""
-        values = [self.unpack(v) for v in ensure_list(values)]
+        values = [self.unpack(v) for v in values]
         data = self.pack(values)
         if data is None:
             return "[]"

followthemoney/types/string.py

@@ -1,6 +1,5 @@
 from followthemoney.types.common import PropertyType
-from followthemoney.util import const, defer as _
-from followthemoney.util import MEGABYTE
+from followthemoney.util import PROP_VALUE_MAX, const, defer as _
 
 
 class StringType(PropertyType):
@@ -24,7 +23,7 @@ class TextType(StringType):
     name = "text"
     label = _("Text")
     plural = _("Texts")
-    total_size = 30 * MEGABYTE
+    total_size = PROP_VALUE_MAX
     max_length = 65000
 
 
@@ -39,5 +38,5 @@ class HTMLType(StringType):
     name = const("html")
     label = _("HTML")
     plural = _("HTMLs")
-    total_size = 30 * MEGABYTE
+    total_size = PROP_VALUE_MAX
     max_length = 65000

followthemoney/util.py

@@ -14,6 +14,8 @@ from rigour.env import ENCODING
 from banal import is_mapping, unique_list, ensure_list
 
 MEGABYTE = 1024 * 1024
+PROP_VALUE_MAX = 30 * MEGABYTE
+ENTITY_VALUE_MAX = 50 * MEGABYTE
 HASH_ENCODING = "utf-8"
 DEFAULT_LOCALE = "en"
 ENTITY_ID_LEN = 200

{followthemoney-4.5.1.dist-info → followthemoney-4.5.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: followthemoney
-Version: 4.5.1
+Version: 4.5.2
 Summary: A data model for anti corruption data modeling and analysis.
 Project-URL: Documentation, https://followthemoney.tech/
 Project-URL: Repository, https://github.com/opensanctions/followthemoney.git

{followthemoney-4.5.1.dist-info → followthemoney-4.5.2.dist-info}/RECORD

@@ -1,4 +1,4 @@
-followthemoney/__init__.py,sha256=xxSC9m3THm5tFr_l8TY-RAEKb-ymcueZsmV4JVEjPyM,894
+followthemoney/__init__.py,sha256=nl5VzBB-U4sYL6j-D7z6ndi6LcvdcZ-ErMyjWbZnFhk,894
 followthemoney/compare.py,sha256=6y6fqtbbfW7ee4_EVXPcKCIr75GjPqXvtHfvExJ-KK0,6119
 followthemoney/entity.py,sha256=biAjuuHlwsVT02imAsaWP0YtgdfU8skCntzBU3mgJpg,4052
 followthemoney/exc.py,sha256=GyMgwY4QVm87hLevDfV7gM1MJsDqfNCi_UQw7F_A8X8,858
@@ -10,11 +10,11 @@ followthemoney/names.py,sha256=LODQqExKEHdH4z6Mmbhlm0KeKRzGcptaSWzYXZ7lONI,1120
 followthemoney/namespace.py,sha256=utggu9IGA8bhgEYom3OUB1KxkAJR_TrMNbY5MUF_db8,4536
 followthemoney/ontology.py,sha256=WWY_PYQGl5Ket4zZBuZglzQxD2Bh9UqHok6GJNNX7GA,3001
 followthemoney/property.py,sha256=1w7p9aKLxRqFRnl3PlssqmvulSErl_0D5T2SefT3UFU,8675
-followthemoney/proxy.py,sha256=ZD8jK88oj1aUTpF7s6r91g8tmTiEfubUKX7CGV5I9rE,20714
+followthemoney/proxy.py,sha256=nA8trRbMcPfbmoEwHQK8kdiEbZqOpTuwT_GfQkfZj8U,21415
 followthemoney/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 followthemoney/schema.py,sha256=dwZg0eZF7yaxP9fJ5NQUKWadWZYTo9U-sVzzXZn_6do,18500
 followthemoney/settings.py,sha256=wkwrkDPypsAICLo-d7l-zpKrKe_fTVYPllZO-RX3oGM,507
-followthemoney/util.py,sha256=JCZ35h_qrqxWO1FTxpa0eoOiko6DWTD9u1QKmH5ZaLU,4150
+followthemoney/util.py,sha256=hkUe2IOCxgZEppANr1vGcMChC4t9RcudSDHuRH2D_Hg,4214
 followthemoney/value.py,sha256=BJ4Sj5Tg2kMrslR6FjQUr96d8Kt75U7ny9NgzVGT0ZE,2335
 followthemoney/cli/__init__.py,sha256=0mmz84uhXRp2qUn3syKnDXofU3MMAAe291s7htqX0Bg,187
 followthemoney/cli/aggregate.py,sha256=xQTFpU3cVVj7fplpX4OJVrRlTVpn6b9kBr_Vb87pKfg,2164
@@ -115,8 +115,8 @@ followthemoney/schema/Vessel.yaml,sha256=zWHUfSK8g6Pz58ZyCaK0AFJ4u_UHjEIUGC4c_7o
 followthemoney/schema/Video.yaml,sha256=LY3DYMWTHXiAhL0hxBCNCz50cp2sPbUlEhhig5Fbjos,327
 followthemoney/schema/Workbook.yaml,sha256=iikWPElz4klA7SkWH7eae6xqhbkMCIP_3zdeXzFEMU0,354
 followthemoney/statement/__init__.py,sha256=7m2VUCAuqNZXIY0WFJRFkw5UG14QuxATL4f_xbqKwhw,633
-followthemoney/statement/entity.py,sha256=jwT1OiM6NiGbCWTZIG4wi0a-ROGy-GN6Vpq9pyp5mbs,19759
-followthemoney/statement/serialize.py,sha256=TLpeCrpgFrQ6_AA3tcNxZhcOjlo_-c1MNfh3zE3idj4,7509
+followthemoney/statement/entity.py,sha256=vpiHsg2uOXnyIDYHRfMr0TWoOvGSzbiEfO2qHx2txQ8,20072
+followthemoney/statement/serialize.py,sha256=fbvTVSvBeeusXIXWw9-_3NZaoncDvEnet6my6diUj7k,7566
 followthemoney/statement/statement.py,sha256=bAwrrKyYRdJVxRGuXDQIOA7cdEngk1NKUaij_gAwSd4,11876
 followthemoney/statement/util.py,sha256=jHBwK3FIBynUJZRlBBOHayalAFrqpXf2f2JwkHi0zAU,1450
 followthemoney/translations/messages.pot,sha256=JhtY9NJ9wP_EAX4APxOqMyvKcX53oIC9kAxBsliJkf4,107703
@@ -145,7 +145,7 @@ followthemoney/translations/tr/LC_MESSAGES/followthemoney.po,sha256=AZC3marhtVVq
 followthemoney/types/__init__.py,sha256=rWwQeiuMh2BNIuvhpMfJ4bPADDvt9Axu1eedvNFi0qY,3350
 followthemoney/types/address.py,sha256=Gc-hqz00dRRkeANqkyPD2wtt7ksR9wMf4CX-U-5XvMo,2214
 followthemoney/types/checksum.py,sha256=_0ev2Wwtd4iX_bLz0Lu-xcJIxNfH_V9kBKKtuZhoAwg,802
-followthemoney/types/common.py,sha256=4ks7zPT8rknrGSd4JFc1zRkS-TL4SX-25_ZbjcVDos0,10081
+followthemoney/types/common.py,sha256=d2z3E0z93SZRU6mxlQ-ShQdaNU9ilD0FvaE5MpJwxhE,9988
 followthemoney/types/country.py,sha256=pwDiI_ipts9Oi2U7fHALYMJPCJHOqyI_2Liq7XI2XrA,2086
 followthemoney/types/date.py,sha256=SGk8q8qICIrA5Lf6wPbDU6v6oJYEiu1dZFZ18BO7i80,3551
 followthemoney/types/email.py,sha256=L3RTYrMABlNQF7hCynXGfzoj6YNEHW5JAY_BwuhoZdA,3375
@@ -153,17 +153,17 @@ followthemoney/types/entity.py,sha256=56h6x8Ct7hWZIC3BjZHmRKGy9Ff2vuULNWH3xDRsKi
 followthemoney/types/gender.py,sha256=XY9us98Sk25O1xnHN-88tbv9pHy6Mn7SR8GRYi6v5gI,1683
 followthemoney/types/identifier.py,sha256=TYJwE7urjHFxEcDuiZMxGoCN6n34rAIdCt5_96Y7vI0,2198
 followthemoney/types/ip.py,sha256=rCXkRrh_jDeWAhswCgSe6Z4uhIW7yvLAxIEw4x1SM3A,1279
-followthemoney/types/json.py,sha256=Hefwns1-ziJf310MWvdfX5ICkOgj9cnnMJuqq1e6qKY,1676
+followthemoney/types/json.py,sha256=8aKPnDXiNHAh-h96D1rxFaM_FyB6O_LVBptFn90RnvU,1633
 followthemoney/types/language.py,sha256=ymEXaHAPIZGlGySUqzadB7tMB6mW1ASsl1G6EtqKdls,2730
 followthemoney/types/mimetype.py,sha256=oqVP8EfGckPAI3WAziHomp6oUN7KXdIPWzGZPsRtIA8,1242
 followthemoney/types/name.py,sha256=zd0aC4VGp1SYUI8Rj0-ZXlrpUI7ZcnJIljZqsEsV-CY,2363
 followthemoney/types/number.py,sha256=vpAyhmc7UQlIm8h7Z5k8k4cTk37ykRF-AgYA1r_g1QQ,3934
 followthemoney/types/phone.py,sha256=_HanfxxTV7jp75gZO2evBc9HWwQTxEMQRaoVDcoXDIQ,3790
-followthemoney/types/string.py,sha256=SEh3xqQCnm377PGvwfR6ao85pHJCNeCUWBKnvccrJ7I,1216
+followthemoney/types/string.py,sha256=1h92OUSgVVxu40N4BCiBZvX1VXFmvEgwtfICO92x9kc,1193
 followthemoney/types/topic.py,sha256=9FIH_WmwVOFg1CJRBF4KeE6vNTn-QQkzsKU5XaMqNJ0,4604
 followthemoney/types/url.py,sha256=sSHKtzvm4kc-VTvNCPIDykOG1hUoawhORj6Bklo0a2A,1434
-followthemoney-4.5.1.dist-info/METADATA,sha256=kRhvZB4Bvc1aVmGx2n2dyvYoOGhU16uUBqKf69C7bck,6747
-followthemoney-4.5.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-followthemoney-4.5.1.dist-info/entry_points.txt,sha256=caoFTlf213jhg5sz3TNSofutjUTzaKtWATuSIdd9Cps,653
-followthemoney-4.5.1.dist-info/licenses/LICENSE,sha256=H6_EVXisnJC0-18CjXIaqrBSFq_VH3OnS7u3dccOv6g,1148
-followthemoney-4.5.1.dist-info/RECORD,,
+followthemoney-4.5.2.dist-info/METADATA,sha256=Z568uQ_TOejHVvbnqmRZKHIIySLcx8bd58l6K3cTCMo,6747
+followthemoney-4.5.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+followthemoney-4.5.2.dist-info/entry_points.txt,sha256=caoFTlf213jhg5sz3TNSofutjUTzaKtWATuSIdd9Cps,653
+followthemoney-4.5.2.dist-info/licenses/LICENSE,sha256=H6_EVXisnJC0-18CjXIaqrBSFq_VH3OnS7u3dccOv6g,1148
+followthemoney-4.5.2.dist-info/RECORD,,