followthemoney 4.2.2__py3-none-any.whl → 4.3.0__py3-none-any.whl

followthemoney/__init__.py

@@ -9,7 +9,7 @@ from followthemoney.statement import Statement, StatementEntity, SE
 from followthemoney.dataset import Dataset, DefaultDataset, DS
 from followthemoney.util import set_model_locale
 
-__version__ = "4.2.2"
+__version__ = "4.3.0"
 
 # Data model singleton
 model = Model.instance()

followthemoney/helpers.py

@@ -13,8 +13,7 @@ from itertools import product
 from datetime import datetime, timedelta
 
 from followthemoney.types import registry
-from followthemoney.proxy import E
-from followthemoney.util import join_text
+from followthemoney.proxy import E, EntityProxy
 
 PROV_MIN_DATES = ("createdAt", "authoredAt", "publishedAt")
 PROV_MAX_DATES = ("modifiedAt", "retrievedAt")
@@ -47,7 +46,7 @@ def simplify_provenance(proxy: E) -> E:
 
 
 def entity_filename(
-    proxy: E, base_name: Optional[str] = None, extension: Optional[str] = None
+    proxy: EntityProxy, base_name: Optional[str] = None, extension: Optional[str] = None
 ) -> Optional[str]:
     """Derive a safe filename for the given entity."""
     if proxy.schema.is_a("Document"):
@@ -85,7 +84,7 @@ def name_entity(entity: E) -> E:
 
 
 def check_person_cutoff(
-    entity: E,
+    entity: EntityProxy,
     death_cutoff: datetime = datetime(2000, 1, 1),
     birth_cutoff: Optional[datetime] = None,
 ) -> bool:
@@ -153,17 +152,17 @@ def combine_names(entity: E) -> E:
     This is of course impossible to do culturally correctly for the whole planet at
     once, so it should be mostly used for internal-facing (e.g. matching) processes."""
     if entity.schema.is_a("Person"):
-        first_names = entity.get("firstName")
-        second_names = entity.get("secondName") + [""]
-        middle_names = entity.get("middleName") + [""]
-        father_names = entity.get("fatherName") + [""]
-        mother_names = entity.get("motherName") + [""]
         last_names = entity.get("lastName")
-        for (first, second, middle, father, mother, last) in product(
-            first_names, second_names, middle_names, father_names, mother_names, last_names
-        ):
-            name = squash_spaces(" ".join([first, second, middle, father, mother, last]))
-            if name is not None:
+        names_seq = [entity.get("firstName")]
+        names_seq.append(entity.get("secondName"))
+        names_seq.append(entity.get("middleName"))
+        names_seq.append(entity.get("fatherName"))
+        names_seq.append(entity.get("motherName"))
+        names_seq.append(last_names)
+        names_seq = [n for n in names_seq if len(n)]
+        for pairing in product(*names_seq):
+            name = squash_spaces(" ".join(pairing))
+            if len(name):
                 entity.add("alias", name)
 
         # If no first name is given, at least add the last name:

followthemoney/proxy.py

@@ -323,7 +323,7 @@ class EntityProxy(object):
     @property
     def countries(self) -> List[str]:
         """Get the set of all country-type values set of the entity."""
-        return self.get_type_values(registry.country)
+        return self.get_type_values(registry.country, matchable=True)
 
     @property
     def temporal_start(self) -> Optional[Tuple[Property, str]]:
@@ -386,6 +386,8 @@ class EntityProxy(object):
         countries = set(self.countries)
         if not len(countries):
             for prop, value in self.itervalues():
+                if not prop.matchable:
+                    continue
                 hint = prop.type.country_hint(value)
                 if hint is not None:
                     countries.add(hint)

followthemoney/schema/Company.yaml

@@ -84,6 +84,7 @@ Company:
     bikCode:
       label: "BIK"
       description: "Russian bank account code"
+      type: identifier
     pfrNumber:
       label: "PFR Number"
       description: "(RU, ПФР) Pension Fund Registration number. AAA-BBB-CCCCCC, where AAA is organisation region, BBB is district, CCCCCC number at a specific branch"

followthemoney/schema/LegalEntity.yaml

@@ -102,6 +102,17 @@ LegalEntity:
     bvdId:
       label: Bureau van Dijk ID
       type: identifier
+    sayariId:
+      label: Sayari Entity ID
+      type: identifier
+    brightQueryId:
+      label: BrightQuery ID
+      type: identifier
+    brightQueryOrgId:
+      label: BrightQuery Organization ID
+      type: identifier
+      hidden: true
+      matchable: false
     uscCode:
       # cf. https://en.wikipedia.org/wiki/Unified_Social_Credit_Identifier
       label: "USCC"

followthemoney/schema/Person.yaml

@@ -31,16 +31,23 @@ Person:
     # too many false positives.
     firstName:
       label: First name
+      description: "The part of a name that indicates the person, also often called given name or forename"
     secondName:
       label: Second name
+      description: "Deprecated, use one of the other more specific name properties instead."
+      deprecated: true
     middleName:
       label: Middle name
+      description: "The part of name written between a person's given name and family name. Often abbreviated as a middle initial."
     fatherName:
       label: Patronymic
+      description: "The part of a name based on the given name of one's father"
     motherName:
       label: Matronymic
+      description: "The part of a name based on the given name of one's mother"
     lastName:
       label: Last name
+      description: "The part of a name that indicates one's family, also often called surname or family name"
     nameSuffix:
       label: Name suffix
     birthDate:
@@ -92,6 +99,8 @@ Person:
       label: Religion
     political:
       label: Political association
+    profession:
+      label: Profession
     education:
       label: Education
     spokenLanguage:

followthemoney/statement/entity.py

@@ -7,6 +7,7 @@ from rigour.names.pick import pick_lang_name
 
 from followthemoney.model import Model
 from followthemoney.exc import InvalidData
+from followthemoney.schema import Schema
 from followthemoney.types.common import PropertyType
 from followthemoney.property import Property
 from followthemoney.util import gettext
@@ -477,12 +478,38 @@ class StatementEntity(EntityProxy):
         dataset: Dataset,
         statements: Iterable[Statement],
     ) -> SE:
-        obj: Optional[SE] = None
+        model = Model.instance()
+        canonical_id: Optional[str] = None
+        schemata: Set[str] = set()
+        first_seens: Set[str] = set()
+        props: Dict[str, Set[Statement]] = {}
         for stmt in statements:
-            if obj is None:
-                data = {"schema": stmt.schema, "id": stmt.canonical_id}
-                obj = cls(dataset, data)
-            obj.add_statement(stmt)
-        if obj is None:
-            raise ValueError("No statements given!")
+            schemata.add(stmt.schema)
+            canonical_id = stmt.canonical_id or canonical_id or stmt.entity_id
+            if stmt.prop == BASE_ID:
+                if stmt.first_seen is not None:
+                    first_seens.add(stmt.first_seen)
+            else:
+                if stmt.prop not in props:
+                    props[stmt.prop] = set()
+                props[stmt.prop].add(stmt)
+
+        schema: Optional[Schema] = None
+        for name in schemata:
+            if schema is None:
+                schema = model.get(name)
+            elif schema.name != name:
+                try:
+                    schema = model.common_schema(schema, name)
+                except InvalidData as exc:
+                    raise InvalidData(f"{canonical_id}: {exc}") from exc
+
+        if schema is None:
+            err = "No valid schema for entity: %s %r" % (canonical_id, schemata)
+            raise InvalidData(err)
+
+        data = {"schema": schema, "id": canonical_id}
+        obj = cls(dataset, data)
+        obj.last_change = max(first_seens, default=None)
+        obj._statements = {p: s for p, s in props.items()}
         return obj

followthemoney/types/address.py

@@ -47,7 +47,9 @@ class AddressType(PropertyType):
         right_norm = normalize_address(right)
         if left_norm is None or right_norm is None:
             return 0.0
-        return levenshtein_similarity(left_norm, right_norm, max_edits=3)
+        base_len = min(len(left_norm), len(right_norm))
+        max_edits = int(base_len * 0.33)
+        return levenshtein_similarity(left_norm, right_norm, max_edits=max_edits)
 
     def _specificity(self, value: str) -> float:
         return dampen(10, 60, value)

{followthemoney-4.2.2.dist-info → followthemoney-4.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: followthemoney
-Version: 4.2.2
+Version: 4.3.0
 Summary: A data model for anti corruption data modeling and analysis.
 Project-URL: Documentation, https://followthemoney.tech/
 Project-URL: Repository, https://github.com/opensanctions/followthemoney.git
@@ -48,9 +48,9 @@ Requires-Dist: prefixdate<1.0.0,>=0.5.0
 Requires-Dist: pydantic<3.0.0,>=2.11.0
 Requires-Dist: pytz>=2021.1
 Requires-Dist: pyyaml<7.0.0,>=5.0.0
-Requires-Dist: rdflib<7.2.0,>=6.2.0
+Requires-Dist: rdflib<7.3.0,>=6.2.0
 Requires-Dist: requests<3.0.0,>=2.21.0
-Requires-Dist: rigour<2.0.0,>=1.3.1
+Requires-Dist: rigour<2.0.0,>=1.3.13
 Requires-Dist: sqlalchemy[mypy]<3.0.0,>=2.0.0
 Provides-Extra: dev
 Requires-Dist: build; extra == 'dev'

{followthemoney-4.2.2.dist-info → followthemoney-4.3.0.dist-info}/RECORD

@@ -1,16 +1,16 @@
-followthemoney/__init__.py,sha256=pTkibO7jJUHnObuIGY1rsjFL9siMpBusU7E9OylQDBI,856
+followthemoney/__init__.py,sha256=SGtXmqy22FtLMX0kcGDSUMbWNGUvoA13F5SUobauMg8,856
 followthemoney/compare.py,sha256=bZlnj2VMoe67q4Lyq_VwS1a-EJnEK1kC8prbs8jyL9E,5774
 followthemoney/entity.py,sha256=bBiX7hNquXemS3vYCUHKtWI_IqX43Z6i8RQDbZ7gXsg,3449
 followthemoney/exc.py,sha256=GyMgwY4QVm87hLevDfV7gM1MJsDqfNCi_UQw7F_A8X8,858
 followthemoney/graph.py,sha256=7X1CGHGvmktS2LSZqld2iXWzG7B831eCNYyBqamqEJ8,10921
-followthemoney/helpers.py,sha256=EsneNJ5DZXHTPUYfXLxGESphsPwCtGiWv-71vvVBWus,7982
+followthemoney/helpers.py,sha256=KCdv1XAE7KQEXBiXp52Kvuck7wMaeNVBM3uaFemcvb4,7873
 followthemoney/messages.py,sha256=zUEa9CFecU8nRafIzhN6TKCh1kEihiIyIS1qr8PxY4g,806
 followthemoney/model.py,sha256=bWFVNa-DhYzc8BdSXBZdG2ev6Nh9uHx6i4tin8DvEEU,7374
 followthemoney/names.py,sha256=LODQqExKEHdH4z6Mmbhlm0KeKRzGcptaSWzYXZ7lONI,1120
 followthemoney/namespace.py,sha256=utggu9IGA8bhgEYom3OUB1KxkAJR_TrMNbY5MUF_db8,4536
 followthemoney/ontology.py,sha256=WWY_PYQGl5Ket4zZBuZglzQxD2Bh9UqHok6GJNNX7GA,3001
 followthemoney/property.py,sha256=9qZ_o2iA-1llLMJ3O2hsW7c2XhkFU1YbvVqretGYUSA,7913
-followthemoney/proxy.py,sha256=LD4K1oPABXMX212UZxwLu7XOHRDyVBwTlqudTUsUZRQ,19619
+followthemoney/proxy.py,sha256=5pMyP0JWWuBcvMIu3AXlW6Za7EEWWdFD2EZkD8UkbQw,19703
 followthemoney/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 followthemoney/schema.py,sha256=WYnPE4Lego0pJHlojECEv0aO9Miw_YIvEb35HoDo4Zk,18087
 followthemoney/util.py,sha256=LoCSp1iE6VwXjotCkBXFRppeQs55726GzOuNIu3CvRE,4409
@@ -54,7 +54,7 @@ followthemoney/schema/Audio.yaml,sha256=Eb1rZGUEOX7XDAj_1YIN28NCBzMvkopQBNwgHt_k
 followthemoney/schema/BankAccount.yaml,sha256=60v-VD296lW1Qq7fx--CzxfPNwfCcyMV6xIl8OrSy5g,1431
 followthemoney/schema/Call.yaml,sha256=kbVCnVxucBrEplxehXHThLSJAJjy_GhWan-IeZZjr0M,980
 followthemoney/schema/CallForTenders.yaml,sha256=2IWonTzfSbrkynMoEWqv5fekUeFM_xDKpKIbRe1XDbo,3227
-followthemoney/schema/Company.yaml,sha256=v6OFHZjEIMQPSP9re6nAhI2kLk1MHfP9LfcCc7M-Ifc,3432
+followthemoney/schema/Company.yaml,sha256=qWhk6HoSlep6oEjviXoV9ACTnfo7WZ5DSDvAkXMaQ5M,3455
 followthemoney/schema/Contract.yaml,sha256=aSPB64T1h-0nuLDv6krasUvvoPZgo6sWUbv60c3vmzI,1541
 followthemoney/schema/ContractAward.yaml,sha256=b2spaZHYCaP1yR1RCsrI7mUjk-fAF7BUE3dc8Vl3cUQ,1689
 followthemoney/schema/CourtCase.yaml,sha256=lcovnY0Ne_xcggvkqfCW_RHvsRKo8kFTCPCyovAXRtI,599
@@ -75,7 +75,7 @@ followthemoney/schema/Identification.yaml,sha256=6txjZs6-3Kn94c3G4tDeDt9Jb4FW55-
 followthemoney/schema/Image.yaml,sha256=wuznboWECGiV96_GQiXq1-oKNoxO8zKisR4xyusnEn8,394
 followthemoney/schema/Interest.yaml,sha256=VUrehmsN1WgtS1oAa5jn_JGtSkZGGYLGNahp-R5JhOQ,282
 followthemoney/schema/Interval.yaml,sha256=8YJQ51GI-GxvbjYs3uC593kQtCepWW_7ZiNnlbPm2aM,2084
-followthemoney/schema/LegalEntity.yaml,sha256=S56ALyeCZVgjpNy9EqkVkPBIs6QaGuZw31s9bdtEeC4,4789
+followthemoney/schema/LegalEntity.yaml,sha256=586aA1F5CO8rwf49LG3_iobm13zM4O21gL_TONte8oM,5054
 followthemoney/schema/License.yaml,sha256=bXESXY-JpSmc5sthZe4sssXhx50UoLPAMED9FvEUyRU,534
 followthemoney/schema/Membership.yaml,sha256=IPmaOX4Ai2r4sGcA5ig2WmLvWHb38akdxp4smEdDWOE,710
 followthemoney/schema/Mention.yaml,sha256=nBeulR_Jm4x75aJ7yNF0TAVhHJqXQaEzOutLIn_YU-4,1086
@@ -89,7 +89,7 @@ followthemoney/schema/Page.yaml,sha256=YjYqaH2sOry0z4xh44CsX_eyuRClD6ZS0d2o2uQXF
 followthemoney/schema/Pages.yaml,sha256=KKPGZ06Ehp5mWIGnYfHUBN9jT03bk8nakw0pB5bA_7E,450
 followthemoney/schema/Passport.yaml,sha256=rpuLC86sdXnHF-prFQM4mAqYzlSGWKvPE4Cphtn2KRw,805
 followthemoney/schema/Payment.yaml,sha256=WRBJuj9ljsxLBs-0g9Z9UD87uR1RTtuUiYnWOnKr1qA,1757
-followthemoney/schema/Person.yaml,sha256=G6L6bf8WQtOC1Xr1TKWRCJt8JlyQKheBPtH1ZmjjS3w,2132
+followthemoney/schema/Person.yaml,sha256=wS_URvex5pG_P5cv0UPafpaz3Jh9Z7CxHZjb3GwB85k,2798
 followthemoney/schema/PlainText.yaml,sha256=hfnVi-HmQeDbqDquSpkPJax9hNm86ioXGr4hzNzyPFE,278
 followthemoney/schema/Position.yaml,sha256=ZpxjWOLxwva_on32r9WD5ys0Ty3YxCju41mg9HG-pe0,1308
 followthemoney/schema/Project.yaml,sha256=2svtyGJopS0UrqPiuYGpBzj30V7k3LRDX4N1U56y4yY,462
@@ -114,7 +114,7 @@ followthemoney/schema/Vessel.yaml,sha256=zWHUfSK8g6Pz58ZyCaK0AFJ4u_UHjEIUGC4c_7o
 followthemoney/schema/Video.yaml,sha256=LY3DYMWTHXiAhL0hxBCNCz50cp2sPbUlEhhig5Fbjos,327
 followthemoney/schema/Workbook.yaml,sha256=iikWPElz4klA7SkWH7eae6xqhbkMCIP_3zdeXzFEMU0,354
 followthemoney/statement/__init__.py,sha256=7m2VUCAuqNZXIY0WFJRFkw5UG14QuxATL4f_xbqKwhw,633
-followthemoney/statement/entity.py,sha256=mUf-6uvrYvqq9azdHu7qFGkGK_0j9gT6nxscX_JXWR0,17673
+followthemoney/statement/entity.py,sha256=oeudwhqfYLJKqbzxEydasMHqevkDASNyYN6s0yddW6I,18755
 followthemoney/statement/serialize.py,sha256=9eXzQ1biR2mSxWRID5C7xDdku4b4ZImHeRJ53yLZ0yo,7225
 followthemoney/statement/statement.py,sha256=Ae-EYuzS8S12BkaRqrvMuI1C7YwlRKa5C_pTBELyNMM,8029
 followthemoney/statement/util.py,sha256=B-ozuRc1TWvpop52873Pqt5OPj8H6uk4KyRJLfAhr10,780
@@ -142,7 +142,7 @@ followthemoney/translations/ru/LC_MESSAGES/followthemoney.po,sha256=7SQWytOTvoAQ
 followthemoney/translations/tr/LC_MESSAGES/followthemoney.mo,sha256=SC84e_ZF_oFJG1NKdyZY_W6Kb6POORZB6wdeAcEWmnE,487
 followthemoney/translations/tr/LC_MESSAGES/followthemoney.po,sha256=AZC3marhtVVq8Ck1FOgnt4sbDMz548nX48O9GDwImbQ,89826
 followthemoney/types/__init__.py,sha256=rWwQeiuMh2BNIuvhpMfJ4bPADDvt9Axu1eedvNFi0qY,3350
-followthemoney/types/address.py,sha256=nMFCj5QJyqA1ddpUmDLpRTum0nGXE-J70_WGnaLXnYo,2130
+followthemoney/types/address.py,sha256=KsKLa9H8FfSkYF8VGslRnt-miGeni1rKICS8t1rzfzo,2235
 followthemoney/types/checksum.py,sha256=zZrU8WX4CY3Vta_vOyfgDNzIwbmtje7AaDv3O1fBMnk,823
 followthemoney/types/common.py,sha256=4ks7zPT8rknrGSd4JFc1zRkS-TL4SX-25_ZbjcVDos0,10081
 followthemoney/types/country.py,sha256=n8vihijDVud_3Ra-as4Ize0jf_HbcdKVR5YX3TlKZy0,1533
@@ -161,8 +161,8 @@ followthemoney/types/phone.py,sha256=r8uRqWinS0CYnYBTs405k5gO4jeatUDgjdzzijoMKJE
 followthemoney/types/string.py,sha256=fqyTauAm4mNnNaoH-yH087RBbNh-G5ZZUO3awTGQUUg,1230
 followthemoney/types/topic.py,sha256=Mi0Gx0m3bDeTmyuvM6jdRMqv81O03U4eI99R13KGu2Y,4503
 followthemoney/types/url.py,sha256=QFpS_JIV8unFHuh_uGv22SWUUkocBoOpzLsAJWom_gI,1455
-followthemoney-4.2.2.dist-info/METADATA,sha256=Q7n2yZuDQJM27GTkVB0nMJgbMSVJLJ1RNCMmzABRFbY,6747
-followthemoney-4.2.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-followthemoney-4.2.2.dist-info/entry_points.txt,sha256=caoFTlf213jhg5sz3TNSofutjUTzaKtWATuSIdd9Cps,653
-followthemoney-4.2.2.dist-info/licenses/LICENSE,sha256=H6_EVXisnJC0-18CjXIaqrBSFq_VH3OnS7u3dccOv6g,1148
-followthemoney-4.2.2.dist-info/RECORD,,
+followthemoney-4.3.0.dist-info/METADATA,sha256=dyrwBXiefNsSpmwOcsP-TsEnJBqmn9DY9vJsMhFVwk0,6748
+followthemoney-4.3.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+followthemoney-4.3.0.dist-info/entry_points.txt,sha256=caoFTlf213jhg5sz3TNSofutjUTzaKtWATuSIdd9Cps,653
+followthemoney-4.3.0.dist-info/licenses/LICENSE,sha256=H6_EVXisnJC0-18CjXIaqrBSFq_VH3OnS7u3dccOv6g,1148
+followthemoney-4.3.0.dist-info/RECORD,,