followthemoney 3.8.0__py3-none-any.whl → 3.8.1__py3-none-any.whl

followthemoney/__init__.py

@@ -3,7 +3,7 @@ import os
 from followthemoney.model import Model
 from followthemoney.util import set_model_locale
 
-__version__ = "3.8.0"
+__version__ = "3.8.1"
 
 
 model_path = os.path.dirname(__file__)

followthemoney/rdf.py

@@ -4,6 +4,6 @@ from rdflib import Namespace
 from rdflib.term import Identifier, URIRef, Literal
 from rdflib import RDF, SKOS, XSD
 
-NS = Namespace("https://w3id.org/ftm#")
+NS = Namespace("https://schema.followthemoney.tech/#")
 
 __all__ = ["NS", "XSD", "RDF", "SKOS", "Identifier", "URIRef", "Literal"]

followthemoney/schema/Message.yaml

@@ -21,6 +21,11 @@ Message:
     - title
     - threadTopic
     - fileName
+  temporalExtent:
+    start:
+      - date
+      - authoredAt
+      - publishedAt
   properties:
     subject:
       label: Subject

followthemoney/schema/Occupancy.yaml

@@ -35,6 +35,10 @@ Occupancy:
         label: "Position holders"
       type: entity
       range: Position
+    declarationDate:
+      label: "Declaration date"
+      type: date
+      matchable: false
     status:
       label: "Status"
       type: string

followthemoney/schema/Passport.yaml

@@ -36,6 +36,7 @@ Passport:
       label: "Place of birth"
     gender:
       label: "Gender"
+      type: gender
     personalNumber:
       label: "Personal number"
       type: identifier

followthemoney/schema/Sanction.yaml

@@ -7,10 +7,10 @@ Sanction:
   matchable: false
   featured:
     - entity
+    - country
     - authority
     - program
     - startDate
-    - endDate
   required:
     - entity
   caption:

followthemoney/schema.py

@@ -105,8 +105,8 @@ class Schema:
         "edge_source",
         "edge_target",
         "edge_caption",
-        "temporal_start",
-        "temporal_end",
+        "_temporal_start",
+        "_temporal_end",
         "_extends",
         "extends",
         "schemata",
@@ -183,8 +183,8 @@ class Schema:
         #: Specify which properties should be used to represent this schema in a
         #: timeline.
         temporal_extent = data.get("temporalExtent", {})
-        self.temporal_start = set(temporal_extent.get("start", []))
-        self.temporal_end = set(temporal_extent.get("end", []))
+        self._temporal_start = ensure_list(temporal_extent.get("start", []))
+        self._temporal_end = ensure_list(temporal_extent.get("end", []))
 
         #: Direct parent schemata of this schema.
         self._extends = ensure_list(data.get("extends", []))
@@ -211,6 +211,8 @@ class Schema:
     def generate(self, model: "Model") -> None:
         """While loading the schema, this function will validate and
         load the hierarchy, properties, and flags of the definition."""
+        temporal_start: Optional[List[str]] = None
+        temporal_end: Optional[List[str]] = None
         for extends in self._extends:
             parent = model.get(extends)
             if parent is None:
@@ -227,8 +229,22 @@ class Schema:
                 self.names.add(ancestor.name)
                 ancestor.descendants.add(self)
 
-            self.temporal_start |= parent.temporal_start
-            self.temporal_end |= parent.temporal_end
+            if len(self._temporal_start) == 0 and parent.temporal_start:
+                if (
+                    temporal_start is not None
+                    and temporal_start != parent.temporal_start
+                ):
+                    raise InvalidModel(
+                        "Conflicting temporal start properties: %s" % self.name
+                    )
+                temporal_start = parent.temporal_start
+
+            if len(self._temporal_end) == 0 and parent.temporal_end:
+                if temporal_end is not None and temporal_end != parent.temporal_end:
+                    raise InvalidModel(
+                        "Conflicting temporal start properties: %s" % self.name
+                    )
+                temporal_end = parent.temporal_end
 
         for prop in list(self.properties.values()):
             prop.generate(model)
@@ -310,18 +326,38 @@ class Schema:
         return self.get(self.edge_target)
 
     @property
-    def temporal_start_props(self) -> Set[Property]:
+    def temporal_start(self) -> List[str]:
+        """The entity properties to be used as the start when representing the entity
+        in a timeline."""
+        if not len(self._temporal_start):
+            for parent in self.extends:
+                if len(parent.temporal_start):
+                    return parent.temporal_start
+        return self._temporal_start
+
+    @property
+    def temporal_end(self) -> List[str]:
+        """The entity properties to be used as the end when representing the entity
+        in a timeline."""
+        if not len(self._temporal_end):
+            for parent in self.extends:
+                if len(parent.temporal_end):
+                    return parent.temporal_end
+        return self._temporal_end
+
+    @property
+    def temporal_start_props(self) -> List[Property]:
         """The entity properties to be used as the start when representing the entity
         in a timeline."""
         props = [self.get(prop_name) for prop_name in self.temporal_start]
-        return set([prop for prop in props if prop is not None])
+        return [prop for prop in props if prop is not None]
 
     @property
-    def temporal_end_props(self) -> Set[Property]:
+    def temporal_end_props(self) -> List[Property]:
         """The entity properties to be used as the end when representing the entity
         in a timeline."""
         props = [self.get(prop_name) for prop_name in self.temporal_end]
-        return set([prop for prop in props if prop is not None])
+        return [prop for prop in props if prop is not None]
 
     @property
     def sorted_properties(self) -> List[Property]:
@@ -408,16 +444,10 @@ class Schema:
                 "label": self.edge_label,
                 "directed": self.edge_directed,
             }
-        start_props = [
-            prop.name for prop in self.temporal_start_props if prop.schema == self
-        ]
-        end_props = [
-            prop.name for prop in self.temporal_end_props if prop.schema == self
-        ]
-        if start_props or end_props:
+        if len(self.temporal_start) or len(self.temporal_end):
             data["temporalExtent"] = {
-                "start": sorted(start_props),
-                "end": sorted(end_props),
+                "start": self.temporal_start,
+                "end": self.temporal_end,
             }
         if len(self.featured):
             data["featured"] = self.featured

followthemoney/types/address.py

@@ -2,6 +2,8 @@ import re
 from typing import Optional, TYPE_CHECKING
 from normality import slugify
 from normality.cleaning import collapse_spaces
+from rigour.addresses import normalize_address
+from rigour.text.distance import levenshtein_similarity
 
 from followthemoney.types.common import PropertyType
 from followthemoney.util import defer as _
@@ -41,11 +43,18 @@ class AddressType(PropertyType):
             return None
         return collapsed
 
+    def compare(self, left: str, right: str) -> float:
+        left_norm = normalize_address(left)
+        right_norm = normalize_address(right)
+        if left_norm is None or right_norm is None:
+            return 0.0
+        return levenshtein_similarity(left_norm, right_norm, max_edits=3)
+
     def _specificity(self, value: str) -> float:
         return dampen(10, 60, value)
 
     def node_id(self, value: str) -> Optional[str]:
-        slug = slugify(value)
+        slug = slugify(normalize_address(value))
         if slug is None:
             return None
         return f"addr:{value}"

followthemoney/types/country.py

@@ -1,10 +1,11 @@
 import countrynames
 from typing import Optional, TYPE_CHECKING
 from babel.core import Locale
+from rigour.territories import get_territory, get_ftm_countries
 
 from followthemoney.rdf import URIRef, Identifier
 from followthemoney.types.common import EnumType, EnumValues
-from followthemoney.util import gettext, defer as _
+from followthemoney.util import defer as _
 
 if TYPE_CHECKING:
     from followthemoney.proxy import EntityProxy
@@ -24,52 +25,7 @@ class CountryType(EnumType):
     max_length = 16
 
     def _locale_names(self, locale: Locale) -> EnumValues:
-        # extra territories that OCCRP is interested in.
-        names = {
-            "zz": gettext("Global"),
-            "eu": gettext("European Union"),
-            "un": gettext("United Nations"),
-            "zr": gettext("Zaire"),
-            # Overwrite "Czechia" label:
-            "cz": gettext("Czech Republic"),
-            "xk": gettext("Kosovo"),
-            "dd": gettext("East Germany"),
-            "yucs": gettext("Yugoslavia"),
-            "csxx": gettext("Serbia and Montenegro"),
-            "cshh": gettext("Czechoslovakia"),
-            "suhh": gettext("Soviet Union"),
-            "ge-ab": gettext("Abkhazia (Occupied Georgia)"),
-            "x-so": gettext("South Ossetia (Occupied Georgia)"),
-            "ua-lpr": gettext("Luhansk (Occupied Ukraine)"),
-            "ua-dpr": gettext("Donetsk (Occupied Ukraine)"),
-            "ua-cri": gettext("Crimea (Occupied Ukraine)"),
-            "so-som": gettext("Somaliland"),
-            "cy-trnc": gettext("Northern Cyprus"),
-            "az-nk": gettext("Nagorno-Karabakh"),
-            "iq-kr": gettext("Kurdistan"),
-            "cn-xz": gettext("Tibet"),
-            "cq": gettext("Sark"),
-            "gb-wls": gettext("Wales"),
-            "gb-sct": gettext("Scotland"),
-            "gb-nir": gettext("Northern Ireland"),
-            "md-pmr": gettext("Transnistria (PMR)"),
-            "pk-km": gettext("Kashmir"),
-        }
-        for code, label in locale.territories.items():
-            code = code.lower()
-            if code in names:
-                continue
-            try:
-                int(code)
-            except ValueError:
-                names[code] = label
-        # Remove some ISO-3611 codes that are not countries:
-        names.pop("xa", None)
-        names.pop("xb", None)
-        names.pop("qo", None)
-        names.pop("ea", None)
-        names.pop("ez", None)
-        return names
+        return {t.code: t.name for t in get_ftm_countries()}
 
     def clean_text(
         self,
@@ -82,6 +38,11 @@ class CountryType(EnumType):
 
         The input may be a country code, a country name, etc.
         """
+        territory = get_territory(text)
+        if territory is not None:
+            ftm_country = territory.ftm_country
+            if ftm_country is not None:
+                return ftm_country
         code = countrynames.to_code(text, fuzzy=fuzzy)
         if code is not None:
             lower = code.lower()
@@ -93,4 +54,4 @@ class CountryType(EnumType):
         return value
 
     def rdf(self, value: str) -> Identifier:
-        return URIRef(f"iso-3166-1:{value}")
+        return URIRef(f"iso-3166:{value}")

{followthemoney-3.8.0.dist-info → followthemoney-3.8.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: followthemoney
-Version: 3.8.0
+Version: 3.8.1
 Summary: A data model for anti corruption data modeling and analysis.
 Project-URL: Documentation, https://followthemoney.tech/
 Project-URL: Repository, https://github.com/alephdata/followthemoney.git
@@ -51,7 +51,7 @@ Requires-Dist: pytz>=2021.1
 Requires-Dist: pyyaml<7.0.0,>=5.0.0
 Requires-Dist: rdflib<7.2.0,>=6.2.0
 Requires-Dist: requests<3.0.0,>=2.21.0
-Requires-Dist: rigour<1.0.0,>=0.8.1
+Requires-Dist: rigour<1.0.0,>=0.9.3
 Requires-Dist: sqlalchemy2-stubs
 Requires-Dist: sqlalchemy<3.0.0,>=1.4.49
 Requires-Dist: stringcase<2.0.0,>=1.2.0

{followthemoney-3.8.0.dist-info → followthemoney-3.8.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-followthemoney/__init__.py,sha256=-5MmNEJ1l81MQncQPSsLxZ-nf7ct3KDn1FrEZ5NRvNc,360
+followthemoney/__init__.py,sha256=Cuo89JNtffF2ZZmMgYEsvlkWKVYHXnFnFvzQ6BRX6Gw,360
 followthemoney/compare.py,sha256=1GFkCfTzA8QR0CH90kvySR8hvl9hQRUerW5Xw2Ivmpg,5134
 followthemoney/exc.py,sha256=ynZs_UnTVxHR-iBfat_CpVLraYzVX5yLtVf5Ti14hl4,734
 followthemoney/graph.py,sha256=VNDKrUBkz_-DmKsr5v-Xm8VfxzabnTwkU_MFk92_TjA,10848
@@ -11,8 +11,8 @@ followthemoney/ontology.py,sha256=7PEoUKISNpkRvVhuLeE3IE9ZiNtdR8mn9_kzZ9yF9l0,29
 followthemoney/property.py,sha256=zi9ss1v0e8Wmv-FuLtZd2aod5iTLfBekBxuOTgIOUMU,7718
 followthemoney/proxy.py,sha256=M7RIPF0k-P3v7GYKYhRVVaO1cnUf5FArJepE-2c0KMQ,20033
 followthemoney/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-followthemoney/rdf.py,sha256=_BzWiBc61UYjiyadvGKya4P4JlJfvZtNrn8SP2hC2JM,328
-followthemoney/schema.py,sha256=Tk61vRmWaawW-1HOmu-hbYZtkD8it0fIvNtSRxBEQxA,16774
+followthemoney/rdf.py,sha256=9wPs-tqN9QILuvrmH_YheNaFQctyIQqoIEqcS465QHs,343
+followthemoney/schema.py,sha256=vPG2C4r0ar3F74fbU_v-YPq4I3Qfmu9gzxMFWksU7I4,18097
 followthemoney/util.py,sha256=dMaCas9EjUPQ9q_a4qtV1pQOxWfxPjyjwR0e84XSN9c,4396
 followthemoney/cli/__init__.py,sha256=Fl05wMr5-FlOSMRpmu1HJSNfPRpy8u9as5IRbGXdo4U,421
 followthemoney/cli/aggregate.py,sha256=Gwfi5Bt1LCwqbpsCu4P1Cr-QJtCWhbaqgGEzfwJUUL4,2142
@@ -71,15 +71,15 @@ followthemoney/schema/LegalEntity.yaml,sha256=1AouOu97lbKeLP1VNaprN_oACdTvcptGHo
 followthemoney/schema/License.yaml,sha256=9Ye5vGEBhi7ttGqf0DdAGCJCN3zz5HtGu52dCcmCsQk,452
 followthemoney/schema/Membership.yaml,sha256=IPmaOX4Ai2r4sGcA5ig2WmLvWHb38akdxp4smEdDWOE,710
 followthemoney/schema/Mention.yaml,sha256=nBeulR_Jm4x75aJ7yNF0TAVhHJqXQaEzOutLIn_YU-4,1086
-followthemoney/schema/Message.yaml,sha256=89k3kdN2w_Vfz1UjbUUPojZtMsAvmZGiDSkHeSNMl8w,1484
+followthemoney/schema/Message.yaml,sha256=PAxZ2NRFVvnOlp9Ohh5fJDEThjJ0jm3M2YCbJ9KtMuE,1565
 followthemoney/schema/Note.yaml,sha256=NohwtFupxIssZuEgQowiQWqKit4uQ-OatAu3yp9eJj4,411
-followthemoney/schema/Occupancy.yaml,sha256=-alsy15cz3RXC_2xYN6rXxwxSOW_hazCNyIvmp7Jawo,773
+followthemoney/schema/Occupancy.yaml,sha256=hFXUAmPxxy3hNEe92pGKgMp80a9A-v_fNDIZd98NzcY,866
 followthemoney/schema/Organization.yaml,sha256=wPXU1ni0-3QzvttDq-gIjbAYHzcWoo3nsLGLw6cnHKI,1064
 followthemoney/schema/Ownership.yaml,sha256=tLWESE9VX0aUuhe6C1pToq2-auPVZBdE3xvBmTRfmPc,1057
 followthemoney/schema/Package.yaml,sha256=gPr-P3lcg7OOAav_KVa8baK4yK57JwfcXwxXheD96UQ,310
 followthemoney/schema/Page.yaml,sha256=sQt_CnVyjDVGVECLQoGYZH4hxpjdPhxVRz4XJW-_1OU,1107
 followthemoney/schema/Pages.yaml,sha256=KKPGZ06Ehp5mWIGnYfHUBN9jT03bk8nakw0pB5bA_7E,450
-followthemoney/schema/Passport.yaml,sha256=gFVnLRkNzLfjQqDOArk8IIXghqCTcpAu0TDsizAIju4,786
+followthemoney/schema/Passport.yaml,sha256=rpuLC86sdXnHF-prFQM4mAqYzlSGWKvPE4Cphtn2KRw,805
 followthemoney/schema/Payment.yaml,sha256=WRBJuj9ljsxLBs-0g9Z9UD87uR1RTtuUiYnWOnKr1qA,1757
 followthemoney/schema/Person.yaml,sha256=EsMcdOzGa8v8zQ_AgvDHSUFL0SflPMhdSdC7TWwW0sU,2354
 followthemoney/schema/PlainText.yaml,sha256=hfnVi-HmQeDbqDquSpkPJax9hNm86ioXGr4hzNzyPFE,278
@@ -90,7 +90,7 @@ followthemoney/schema/ProjectParticipant.yaml,sha256=xNehEu90uqUfboNouezhZQ8ZQLx
 followthemoney/schema/PublicBody.yaml,sha256=BNfLBqH1OapoEninAjWmqZx_n-G5QUnzzydW7300TiY,301
 followthemoney/schema/RealEstate.yaml,sha256=NWFHXqEHskYQN-kvQESZpu74nztShqoYSZEjZAr-DHM,1363
 followthemoney/schema/Representation.yaml,sha256=sCvFnUDQaElq2cqSB0rILcMYb2gaMZqlzxlHxyX9IGg,792
-followthemoney/schema/Sanction.yaml,sha256=fKkfVPQvY_R2zyQfEkuradilBAwVQWRkWw1Z4_Fn1V8,1259
+followthemoney/schema/Sanction.yaml,sha256=VpvVXReuA3xF1V0tlDyK_LItnL6G_Xt3HivnCf9_zag,1259
 followthemoney/schema/Security.yaml,sha256=w8Och0cslWjHPAs60HZ6JarEXdIbqGlIbN1NlvgN_7Y,1212
 followthemoney/schema/Similar.yaml,sha256=gD8rZEaPQWzU-rEfsKdn62uEucF3KxYBcPMoSdnxvME,817
 followthemoney/schema/Succession.yaml,sha256=RMJQqZ4Fv88N1RvWTAgjYg9BB5cELSj5CCAjM681Fpg,749
@@ -129,10 +129,10 @@ followthemoney/translations/ru/LC_MESSAGES/followthemoney.po,sha256=7SQWytOTvoAQ
 followthemoney/translations/tr/LC_MESSAGES/followthemoney.mo,sha256=SC84e_ZF_oFJG1NKdyZY_W6Kb6POORZB6wdeAcEWmnE,487
 followthemoney/translations/tr/LC_MESSAGES/followthemoney.po,sha256=AZC3marhtVVq8Ck1FOgnt4sbDMz548nX48O9GDwImbQ,89826
 followthemoney/types/__init__.py,sha256=X9XeM6JktzirAw5gGkyDKGM70NuiJ9Tbjoq0IwVclxU,1745
-followthemoney/types/address.py,sha256=P6ctzojJks70RZIwk5Bhc9djxejZo4Te0m5pQ9M7bAM,1651
+followthemoney/types/address.py,sha256=NnJli6jIs5DfePL4m3dA3Xwv7GA8sdl1J6Mtm-QLeOY,2068
 followthemoney/types/checksum.py,sha256=OqjCsBPyMIV3_Y20kTTvjkyayy32pBtaI5KKwYQb6lY,937
 followthemoney/types/common.py,sha256=Yk-7LZ6uDgFzKXUZxQ_j5miN2ZMnlBOziU0iFC7FE9I,10202
-followthemoney/types/country.py,sha256=PlkqZ9_06UO3f47YFYyL3gWwpm151FFlWAV3fodtPfQ,3409
+followthemoney/types/country.py,sha256=1ORYH_wV1cbT7HQzoKi3EbKVAy4G59nsuGk4m6Rm3-M,1834
 followthemoney/types/date.py,sha256=4lZCd0aws-T3HE2BYkb-a-t8iQDr0nqFSAEBoUKiTlM,2683
 followthemoney/types/email.py,sha256=zAnMHwC_FZh7IagpDRhGZf-GfQR6uW8d-lZP4UCdGcM,2745
 followthemoney/types/entity.py,sha256=FA8xXNCkiM7HlF6k5lxTlb_B_LKlUwaoOMaVqAdjgYc,2477
@@ -150,8 +150,8 @@ followthemoney/types/registry.py,sha256=hvQ1oIWz_4PpyUnKA5azbaZCulNb5LCIPyC-AQYV
 followthemoney/types/string.py,sha256=grDn1OgKWxIzVxGEdp0HjVKIqtQ9W4SW2ty4quv3Pcs,1202
 followthemoney/types/topic.py,sha256=6g57JKTFF_I233-afvuaswFxMaN_Kmg5MZ4_iAKZh9c,3754
 followthemoney/types/url.py,sha256=r7Pd6Yfn--amwMi_nHoTLMwm5SH8h50SMgQaa2G4PJ0,1492
-followthemoney-3.8.0.dist-info/METADATA,sha256=8VnbpVAf1DUAE8xYD6U2OE-5bBGdGlcMMx10e_EIqaY,6145
-followthemoney-3.8.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-followthemoney-3.8.0.dist-info/entry_points.txt,sha256=xvTXjAz0CiZplq4V3iQXlmBexaJyW3zNucIvcDP6L_c,593
-followthemoney-3.8.0.dist-info/licenses/LICENSE,sha256=3tfmmk9RtT1eh67a-NDRwcmOLzztbCtqlHW6O1U92ZA,1098
-followthemoney-3.8.0.dist-info/RECORD,,
+followthemoney-3.8.1.dist-info/METADATA,sha256=7gIc__XwnT9cUwDDaG2AQT3zT51gbYTHm1XEd71V6QY,6145
+followthemoney-3.8.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+followthemoney-3.8.1.dist-info/entry_points.txt,sha256=xvTXjAz0CiZplq4V3iQXlmBexaJyW3zNucIvcDP6L_c,593
+followthemoney-3.8.1.dist-info/licenses/LICENSE,sha256=3tfmmk9RtT1eh67a-NDRwcmOLzztbCtqlHW6O1U92ZA,1098
+followthemoney-3.8.1.dist-info/RECORD,,