followthemoney 3.5.8__py2.py3-none-any.whl → 3.6.0__py2.py3-none-any.whl

followthemoney/__init__.py

@@ -3,7 +3,7 @@ import os
 from followthemoney.model import Model
 from followthemoney.util import set_model_locale
 
-__version__ = "3.5.8"
+__version__ = "3.6.0"
 
 
 model_path = os.path.dirname(__file__)

followthemoney/cli/cli.py

@@ -1,14 +1,14 @@
 import sys
-import json
 import click
+import orjson
 import logging
 from pathlib import Path
-from typing import Optional, TextIO
+from typing import Optional, BinaryIO, List, Any, Dict
 from banal import ensure_list
 
 from followthemoney import model
 from followthemoney.namespace import Namespace
-from followthemoney.cli.util import InPath, OutPath, path_entities, read_entities
+from followthemoney.cli.util import InPath, OutPath, path_entities
 from followthemoney.cli.util import path_writer, write_entity
 from followthemoney.proxy import EntityProxy
 
@@ -20,9 +20,10 @@ def cli() -> None:
 
 
 @cli.command("dump-model", help="Export the current schema model")
-@click.option("-o", "--outfile", type=click.File("w"), default="-")
-def dump_model(outfile: TextIO) -> None:
-    outfile.write(json.dumps(model.to_dict(), indent=2, sort_keys=True))
+@click.option("-o", "--outfile", type=click.File("wb"), default="-")
+def dump_model(outfile: BinaryIO) -> None:
+    f = orjson.OPT_INDENT_2 | orjson.OPT_SORT_KEYS
+    outfile.write(orjson.dumps(model.to_dict(), option=f))
 
 
 @cli.command("validate", help="Re-parse and validate the given data")
@@ -34,7 +35,7 @@ def validate(infile: Path, outfile: Path) -> None:
             for entity in path_entities(infile, EntityProxy, cleaned=False):
                 clean = model.make_entity(entity.schema)
                 clean.id = entity.id
-                for (prop, value) in entity.itervalues():
+                for prop, value in entity.itervalues():
                     clean.add(prop, value)
                 write_entity(outfh, clean)
     except BrokenPipeError:
@@ -46,12 +47,14 @@ def validate(infile: Path, outfile: Path) -> None:
 @click.option("-o", "--outfile", type=OutPath, default="-")  # noqa
 def import_vis(infile: Path, outfile: Path) -> None:
     with path_writer(outfile) as outfh:
-        with open(infile, "r") as infh:
-            data = json.load(infh)
+        with open(infile, "rb") as infh:
+            data: Dict[str, Any] = orjson.loads(infh.read())
             if "entities" in data:
-                entities = data.get("entities", data)
-            if "layout" in data:
+                entities: List[Dict[str, Any]] = data.get("entities", data)
+            elif "layout" in data:
                 entities = data.get("layout", {}).get("entities", data)
+            else:
+                raise click.ClickException("No entities found in VIS file")
             for entity_data in ensure_list(entities):
                 entity = EntityProxy.from_dict(model, entity_data)
                 write_entity(outfh, entity)
@@ -75,10 +78,11 @@ def sign(infile: Path, outfile: Path, signature: Optional[str]) -> None:
 @cli.command(help="Format a stream of entities to make it readable")
 @click.option("-i", "--infile", type=InPath, default="-")  # noqa
 def pretty(infile: Path) -> None:
-    stdout = click.get_text_stream("stdout")
+    stdout = click.get_binary_stream("stdout")
     try:
+        f = orjson.OPT_INDENT_2 | orjson.OPT_APPEND_NEWLINE
         for entity in path_entities(infile, EntityProxy):
-            data = json.dumps(entity.to_dict(), indent=2)
-            stdout.write(data + "\n")
+            data = orjson.dumps(entity.to_dict(), option=f)
+            stdout.write(data)
     except BrokenPipeError:
         raise click.Abort()

followthemoney/export/excel.py

@@ -1,11 +1,11 @@
 import logging
 from io import BytesIO
 from typing import Dict, List, Optional
-from openpyxl import Workbook  # type: ignore
-from openpyxl.cell import WriteOnlyCell  # type: ignore
-from openpyxl.styles import Font, PatternFill  # type: ignore
-from openpyxl.worksheet.worksheet import Worksheet  # type: ignore
-from openpyxl.utils.exceptions import IllegalCharacterError  # type: ignore
+from openpyxl import Workbook
+from openpyxl.cell import WriteOnlyCell
+from openpyxl.styles import Font, PatternFill
+from openpyxl.worksheet.worksheet import Worksheet
+from openpyxl.utils.exceptions import IllegalCharacterError
 
 from followthemoney.export.common import Exporter
 from followthemoney.proxy import E
@@ -25,7 +25,7 @@ class ExcelWriter(object):
         self.workbook = Workbook(write_only=True)
 
     def make_sheet(self, title: str, headers: List[str]) -> Worksheet:
-        sheet = self.workbook.create_sheet(title=title)
+        sheet: Worksheet = self.workbook.create_sheet(title=title)
         sheet.freeze_panes = "A2"
         sheet.sheet_properties.filterMode = True
         cells = []

followthemoney/mapping/entity.py

@@ -1,3 +1,4 @@
+import logging
 from hashlib import sha1
 from warnings import warn
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set
@@ -15,6 +16,8 @@ if TYPE_CHECKING:
     from followthemoney.model import Model
     from followthemoney.mapping.query import QueryMapping
 
+log = logging.getLogger(__name__)
+
 
 class EntityMapping(object):
 
@@ -112,16 +115,24 @@ class EntityMapping(object):
         # from that accessible to phone and address parsers.
         for prop in self.properties:
             if prop.prop.type == registry.country:
-                prop.map(proxy, record, entities)
+                discarded_values = prop.map(proxy, record, entities)
+                for value in discarded_values:
+                    log.warn(f"[{self.name}] Discarded unclean value \"{value}\" for property \"{prop.prop.qname}\".")
 
         for prop in self.properties:
             if prop.prop.type != registry.country:
-                prop.map(proxy, record, entities)
+                discarded_values = prop.map(proxy, record, entities)
+                for value in discarded_values:
+                    log.warn(f"[{self.name}] Discarding unclean value \"{value}\" for property \"{prop.prop.qname}\".")
 
         # Generate the ID at the end to avoid self-reference checks on empty
         # keys.
         proxy.id = self.compute_key(record)
         if proxy.id is None:
+            if self.id_column:
+                log.warn(f"[{self.name}] Skipping entity because no ID could be computed. Make sure that there are no empty values in the \"{self.id_column}\" column.")
+            if self.keys:
+                log.warn(f"[{self.name}] Skipping entity because no ID could be computed. Make sure that there are no empty values in key columns.")
             return None
 
         for prop in self.properties:
@@ -130,6 +141,7 @@ class EntityMapping(object):
                 # the mapping, not in the model. Basically it means: if
                 # this row of source data doesn't have that field, then do
                 # not map it again.
+                log.warn(f"[{self.name}] Skipping entity because required property \"{prop.prop.name}\" is empty.")
                 return None
         return proxy
 

followthemoney/mapping/property.py

@@ -112,13 +112,13 @@ class PropertyMapping(object):
 
     def map(
         self, proxy: EntityProxy, record: Record, entities: Dict[str, EntityProxy]
-    ) -> None:
+    ) -> List[str]:
         if self.entity is not None:
             entity = entities.get(self.entity)
             if entity is not None:
                 proxy.unsafe_add(self.prop, entity.id, cleaned=True)
                 inline_names(proxy, entity)
-            return None
+            return []
 
         # clean the values returned by the query, or by using literals, or
         # formats.
@@ -133,5 +133,17 @@ class PropertyMapping(object):
                 splote.extend(value.split(self.split))
             values = splote
 
+        discarded_values: List[str] = []
+
         for value in values:
-            proxy.unsafe_add(self.prop, value, fuzzy=self.fuzzy, format=self.format)
+            added_value = proxy.unsafe_add(
+                prop=self.prop,
+                value=value,
+                fuzzy=self.fuzzy,
+                format=self.format,
+            )
+
+            if value is not None and added_value is None:
+                discarded_values.append(value)
+
+        return discarded_values

followthemoney/mapping/sql.py

@@ -55,7 +55,7 @@ class SQLSource(Source):
         if database is None:
             raise InvalidMapping("No database in SQL mapping!")
         self.database_uri = cast(str, os.path.expandvars(database))
-        self.engine = create_engine(self.database_uri, poolclass=NullPool)  
+        self.engine = create_engine(self.database_uri, poolclass=NullPool)
         self.meta = MetaData()
 
         tables = keys_values(data, "table", "tables")

followthemoney/property.py

@@ -27,6 +27,7 @@ class PropertyDict(TypedDict, total=False):
     # stub: Optional[bool]
     rdf: Optional[str]
     range: Optional[str]
+    format: Optional[str]
 
 
 class PropertySpec(PropertyDict):
@@ -58,6 +59,7 @@ class Property:
         "matchable",
         "deprecated",
         "_range",
+        "format",
         "range",
         "stub",
         "_reverse",
@@ -113,6 +115,11 @@ class Property:
         self._range = data.get("range")
         self.range: Optional["Schema"] = None
 
+        #: If the property is of type ``identifier``, a more narrow definition of the
+        #: identifier format can be provided. For example, LEI, INN or IBAN codes
+        #: can be automatically validated.
+        self.format: Optional[str] = data.get("format")
+
         #: When a property points to another schema, a reverse property is added for
         #: various administrative reasons. These properties are, however, not real
         #: and cannot be written to. That's why they are marked as stubs and adding
@@ -169,6 +176,8 @@ class Property:
             if self.stub:
                 return gettext("Property cannot be written")
             val = get_entity_id(val)
+            if val is None:
+                continue
             if not self.type.validate(val):
                 return gettext("Invalid value")
             if val is not None:
@@ -203,6 +212,8 @@ class Property:
             data["range"] = self.range.name
         if self.reverse is not None:
             data["reverse"] = self.reverse.name
+        if self.format is not None:
+            data["format"] = self.format
         return data
 
     def __repr__(self) -> str:

followthemoney/proxy.py

@@ -194,6 +194,7 @@ class EntityProxy(object):
 
         for value in value_list(values):
             if not cleaned:
+                format = format or prop.format
                 value = prop.type.clean(value, proxy=self, fuzzy=fuzzy, format=format)
             self.unsafe_add(prop, value, cleaned=True)
         return None
@@ -205,26 +206,32 @@ class EntityProxy(object):
         cleaned: bool = False,
         fuzzy: bool = False,
         format: Optional[str] = None,
-    ) -> None:
+    ) -> Optional[str]:
         """A version of `add()` to be used only in type-checking code. This accepts
         only a single value, and performs input cleaning on the premise that the
-        value is already valid unicode."""
+        value is already valid unicode. Returns the value that has been added."""
         if not cleaned and value is not None:
+            format = format or prop.format
             value = prop.type.clean_text(value, fuzzy=fuzzy, format=format, proxy=self)
-        if value is not None:
-            # Somewhat hacky: limit the maximum size of any particular
-            # field to avoid overloading upstream aleph/elasticsearch.
-            value_size = len(value)
-            if prop.type.max_size is not None:
-                if self._size + value_size > prop.type.max_size:
-                    # msg = "[%s] too large. Rejecting additional values."
-                    # log.warning(msg, prop.name)
-                    return None
-            self._size += value_size
-            self._properties.setdefault(prop.name, list())
-            if value not in self._properties[prop.name]:
-                self._properties[prop.name].append(value)
-        return None
+
+        if value is None:
+            return None
+
+        # Somewhat hacky: limit the maximum size of any particular
+        # field to avoid overloading upstream aleph/elasticsearch.
+        value_size = len(value)
+        if prop.type.max_size is not None:
+            if self._size + value_size > prop.type.max_size:
+                # msg = "[%s] too large. Rejecting additional values."
+                # log.warning(msg, prop.name)
+                return None
+        self._size += value_size
+        self._properties.setdefault(prop.name, list())
+
+        if value not in self._properties[prop.name]:
+            self._properties[prop.name].append(value)
+
+        return value
 
     def set(
         self,
@@ -424,9 +431,12 @@ class EntityProxy(object):
         dictionary can be used to make a new proxy, and it is commonly written to disk
         or a database."""
         data = dict(self.context)
-        data.update(
-            {"id": self.id, "schema": self.schema.name, "properties": self.properties}
-        )
+        extra = {
+            "id": self.id,
+            "schema": self.schema.name,
+            "properties": self.properties,
+        }
+        data.update(extra)
         return data
 
     def to_full_dict(self, matchable: bool = False) -> Dict[str, Any]:

followthemoney/schema/Analyzable.yaml

@@ -32,6 +32,8 @@ Analyzable:
     ibanMentioned:
       label: "Detected IBANs"
       hidden: true
+      # type: identifier
+      # format: iban
       type: iban
     ipMentioned:
       label: "Detected IP addresses"

followthemoney/schema/BankAccount.yaml

@@ -28,10 +28,13 @@ BankAccount:
       type: identifier
     iban:
       label: IBAN
+      # type: identifier
+      # format: iban
       type: iban
     bic:
       label: Bank Identifier Code
       type: identifier
+      format: bic
     bank:
       label: Bank
       type: entity

followthemoney/schema/LegalEntity.yaml

@@ -107,15 +107,18 @@ LegalEntity:
       label: "INN"
       description: "Russian company ID"
       type: identifier
+      # format: inn
     ogrnCode:
       label: "OGRN"
       description: "Major State Registration Number"
       type: identifier
+      # format: ogrn
     leiCode:
       # cf. https://www.gleif.org/en/about-lei/introducing-the-legal-entity-identifier-lei
       label: "LEI"
       description: "Legal Entity Identifier"
       type: identifier
+      format: lei
     dunsCode:
       label: "D-U-N-S"
       description: "Dun & Bradstreet identifier"
@@ -124,6 +127,7 @@ LegalEntity:
       label: "SWIFT/BIC"
       description: "Bank identifier code"
       type: identifier
+      format: bic
     parent:
       label: "Parent company"
       description: "If this entity is a subsidiary, another entity (company or organisation) is its parent"

followthemoney/schema/Person.yaml

@@ -26,6 +26,10 @@ Person:
     title:
       label: Title
       rdf: http://xmlns.com/foaf/0.1/title
+    # The `firstName`, `lastName`, `secondName` etc. properties intentionally do not use
+    # the `name` property type. Many FtM tools (including Aleph) use name properties to
+    # compare/match entities, but matching entites just on e.g. a first name would lead to
+    # too many false positives.
     firstName:
       label: First name
       rdf: http://xmlns.com/foaf/0.1/givenName

followthemoney/schema/Security.yaml

@@ -23,6 +23,7 @@ Security:
       label: ISIN
       description: International Securities Identification Number
       type: identifier
+      format: isin
     registrationNumber:
       label: Registration number
       type: identifier
@@ -32,6 +33,7 @@ Security:
     figiCode:
       label: Financial Instrument Global Identifier
       type: identifier
+      format: figi
     issuer:
       label: "Issuer"
       type: entity

followthemoney/schema/Thing.yaml

@@ -57,6 +57,7 @@ Thing:
     wikidataId:
       label: Wikidata ID
       type: identifier
+      format: qid
     keywords:
       label: Keywords
     topics:

followthemoney/translations/ru/LC_MESSAGES/followthemoney.po

@@ -4,8 +4,8 @@
 # FIRST AUTHOR <EMAIL@ADDRESS>, 2024.
 # 
 # Translators:
-# pudo <friedrich@pudo.org>, 2021
-# jen occrp, 2023
+# pudo <friedrich@pudo.org>, 2022
+# jen occrp, 2024
 # 
 #, fuzzy
 msgid ""
@@ -13,8 +13,8 @@ msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
 "POT-Creation-Date: 2024-01-04 14:09+0100\n"
-"PO-Revision-Date: 2021-05-07 13:11+0000\n"
-"Last-Translator: jen occrp, 2023\n"
+"PO-Revision-Date: 2022-11-21 11:38+0000\n"
+"Last-Translator: jen occrp, 2024\n"
 "Language-Team: Russian (https://app.transifex.com/aleph/teams/76591/ru/)\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
@@ -1029,17 +1029,17 @@ msgstr "Тикер"
 #. Company.properties.permId.label
 #: followthemoney/schema/Company.yaml
 msgid "PermID"
-msgstr ""
+msgstr "PermID"
 
 #. Company.properties.permId.description
 #: followthemoney/schema/Company.yaml
 msgid "LSEG/Refinitiv code for a company"
-msgstr ""
+msgstr "LSEG/Refinitiv код компании"
 
 #. Company.properties.ricCode.label
 #: followthemoney/schema/Company.yaml
 msgid "Reuters Instrument Code"
-msgstr ""
+msgstr "RIC"
 
 #. Contract.label
 #. ContractAward.properties.contract.label
@@ -2779,7 +2779,7 @@ msgstr "Должностное лицо"
 #. Occupancy.properties.holder.reverse.label
 #: followthemoney/schema/Occupancy.yaml
 msgid "Positions held"
-msgstr ""
+msgstr "Занимаемые должности"
 
 #. Occupancy.properties.post.label
 #: followthemoney/schema/Occupancy.yaml
@@ -2789,7 +2789,7 @@ msgstr "Занимаемая должность"
 #. Occupancy.properties.post.reverse.label
 #: followthemoney/schema/Occupancy.yaml
 msgid "Position holders"
-msgstr ""
+msgstr "Занимающие должность"
 
 #. Organization.plural
 #: followthemoney/schema/Organization.yaml
@@ -3366,18 +3366,20 @@ msgstr "Дата записи"
 #. RealEstate.properties.parent.label
 #: followthemoney/schema/RealEstate.yaml
 msgid "Parent unit"
-msgstr ""
+msgstr "Основной объект недвижимости"
 
 #. RealEstate.properties.parent.description
 #: followthemoney/schema/RealEstate.yaml
 msgid ""
 "If this entity is a subunit, another entity (real estate) is its parent"
 msgstr ""
+"Вспомогательный объект недвижимости по отношению к другому — основному "
+"объекту недвижимости"
 
 #. RealEstate.properties.parent.reverse.label
 #: followthemoney/schema/RealEstate.yaml
 msgid "Subunits"
-msgstr ""
+msgstr "Вспомогательные объекты недвижимости"
 
 #. Representation.label
 #: followthemoney/schema/Representation.yaml
@@ -3486,7 +3488,7 @@ msgstr "Котируемый финансовый актив."
 #. Security.properties.figiCode.label
 #: followthemoney/schema/Security.yaml
 msgid "Financial Instrument Global Identifier"
-msgstr ""
+msgstr "Глобальный идентификатор финансового инструмента"
 
 #. Security.properties.issuer.label
 #: followthemoney/schema/Security.yaml
@@ -4240,7 +4242,7 @@ msgstr "Финансовые преступления"
 
 #: followthemoney/types/topic.py:30
 msgid "Environmental violations"
-msgstr ""
+msgstr "Нарушения в области охраны окружающей среды"
 
 #: followthemoney/types/topic.py:31
 msgid "Theft"
@@ -4280,7 +4282,7 @@ msgstr "Офшорная компания"
 
 #: followthemoney/types/topic.py:40
 msgid "Public listed company"
-msgstr ""
+msgstr "Компания, зарегистрированная на фондовой бирже"
 
 #: followthemoney/types/topic.py:41
 msgid "Government"
@@ -4308,31 +4310,31 @@ msgstr "Межправительственная организация"
 
 #: followthemoney/types/topic.py:47
 msgid "Head of government or state"
-msgstr ""
+msgstr "Глава правительства или государства"
 
 #: followthemoney/types/topic.py:48
 msgid "Civil service"
-msgstr ""
+msgstr "Государственная служба"
 
 #: followthemoney/types/topic.py:49
 msgid "Executive branch of government"
-msgstr ""
+msgstr "Исполнительная власть"
 
 #: followthemoney/types/topic.py:50
 msgid "Legislative branch of government"
-msgstr ""
+msgstr "Законодательная власть"
 
 #: followthemoney/types/topic.py:51
 msgid "Judicial branch of government"
-msgstr ""
+msgstr "Судебная власть"
 
 #: followthemoney/types/topic.py:52
 msgid "Security services"
-msgstr ""
+msgstr "Спецслужбы"
 
 #: followthemoney/types/topic.py:53
 msgid "Central banking and financial integrity"
-msgstr ""
+msgstr "Центральный банк и финансовая безупречность"
 
 #: followthemoney/types/topic.py:54
 msgid "Financial services"
@@ -4416,7 +4418,7 @@ msgstr "Подсанкционное физическое или юридиче
 
 #: followthemoney/types/topic.py:76
 msgid "Export controlled"
-msgstr ""
+msgstr "Экспортный контроль"
 
 #: followthemoney/types/topic.py:77
 msgid "Debarred entity"

followthemoney/types/__init__.py

@@ -3,7 +3,6 @@ from followthemoney.types.url import UrlType
 from followthemoney.types.name import NameType
 from followthemoney.types.email import EmailType
 from followthemoney.types.ip import IpType
-from followthemoney.types.iban import IbanType
 from followthemoney.types.address import AddressType
 from followthemoney.types.date import DateType
 from followthemoney.types.phone import PhoneType
@@ -12,6 +11,7 @@ from followthemoney.types.language import LanguageType
 from followthemoney.types.mimetype import MimeType
 from followthemoney.types.checksum import ChecksumType
 from followthemoney.types.identifier import IdentifierType
+from followthemoney.types.iban import IbanType
 from followthemoney.types.entity import EntityType
 from followthemoney.types.topic import TopicType
 from followthemoney.types.gender import GenderType
@@ -27,7 +27,6 @@ registry.add(UrlType)
 registry.add(NameType)
 registry.add(EmailType)
 registry.add(IpType)
-registry.add(IbanType)
 registry.add(AddressType)
 registry.add(DateType)
 registry.add(PhoneType)
@@ -36,6 +35,7 @@ registry.add(LanguageType)
 registry.add(MimeType)
 registry.add(ChecksumType)
 registry.add(IdentifierType)
+registry.add(IbanType)  # TODO: remove
 registry.add(EntityType)
 registry.add(TopicType)
 registry.add(GenderType)

followthemoney/types/common.py

@@ -1,3 +1,4 @@
+from inspect import cleandoc
 from itertools import product
 from babel.core import Locale
 from banal import ensure_list
@@ -64,12 +65,17 @@ class PropertyType(object):
 
     @property
     def docs(self) -> Optional[str]:
-        return self.__doc__
+        if not self.__doc__:
+            return None
+
+        return cleandoc(self.__doc__)
 
-    def validate(self, value: str) -> bool:
+    def validate(
+        self, value: str, fuzzy: bool = False, format: Optional[str] = None
+    ) -> bool:
         """Returns a boolean to indicate if the given value is a valid instance of
         the type."""
-        cleaned = self.clean(value)
+        cleaned = self.clean(value, fuzzy=fuzzy, format=format)
         return cleaned is not None
 
     def clean(
@@ -141,7 +147,7 @@ class PropertyType(object):
     ) -> float:
         """Compare two sets of values and select the highest-scored result."""
         results = []
-        for (l, r) in product(ensure_list(left), ensure_list(right)):
+        for l, r in product(ensure_list(left), ensure_list(right)):
             results.append(self.compare(l, r))
         if not len(results):
             return 0.0
@@ -229,7 +235,9 @@ class EnumType(PropertyType):
             self._names[locale] = self._locale_names(locale)
         return self._names[locale]
 
-    def validate(self, value: str) -> bool:
+    def validate(
+        self, value: str, fuzzy: bool = False, format: Optional[str] = None
+    ) -> bool:
         """Make sure that the given code value is one of the supported set."""
         if value is None:
             return False

followthemoney/types/date.py

@@ -27,9 +27,14 @@ class DateType(PropertyType):
     plural = _("Dates")
     matchable = True
 
-    def validate(self, value: str) -> bool:
+    def validate(
+        self, value: str, fuzzy: bool = False, format: Optional[str] = None
+    ) -> bool:
         """Check if a thing is a valid date."""
-        prefix = parse(value)
+        if format is not None:
+            prefix = parse_format(value, format)
+        else:
+            prefix = parse(value)
         return prefix.precision != Precision.EMPTY
 
     def clean_text(