followthemoney 3.5.9__tar.gz → 3.6.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of followthemoney might be problematic. Click here for more details.

Files changed (168) hide show
  1. followthemoney-3.6.1/PKG-INFO +94 -0
  2. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/__init__.py +1 -1
  3. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/cli/cli.py +18 -14
  4. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/export/excel.py +6 -6
  5. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/mapping/entity.py +14 -2
  6. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/mapping/property.py +15 -3
  7. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/mapping/sql.py +1 -1
  8. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/property.py +11 -0
  9. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/proxy.py +29 -19
  10. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Analyzable.yaml +2 -0
  11. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/BankAccount.yaml +3 -0
  12. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/LegalEntity.yaml +4 -0
  13. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Person.yaml +4 -0
  14. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Security.yaml +2 -0
  15. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Thing.yaml +1 -0
  16. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/types/__init__.py +2 -2
  17. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/types/common.py +13 -5
  18. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/types/date.py +7 -2
  19. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/types/email.py +3 -1
  20. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/types/entity.py +3 -1
  21. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/types/iban.py +7 -9
  22. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/types/identifier.py +17 -0
  23. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/types/ip.py +3 -1
  24. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/types/language.py +1 -1
  25. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/types/mimetype.py +2 -2
  26. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/types/name.py +11 -35
  27. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/types/phone.py +3 -1
  28. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/types/topic.py +5 -1
  29. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/types/url.py +5 -21
  30. followthemoney-3.6.1/followthemoney.egg-info/PKG-INFO +94 -0
  31. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney.egg-info/requires.txt +4 -5
  32. {followthemoney-3.5.9 → followthemoney-3.6.1}/setup.py +5 -6
  33. followthemoney-3.5.9/PKG-INFO +0 -91
  34. followthemoney-3.5.9/followthemoney.egg-info/PKG-INFO +0 -91
  35. {followthemoney-3.5.9 → followthemoney-3.6.1}/LICENSE +0 -0
  36. {followthemoney-3.5.9 → followthemoney-3.6.1}/MANIFEST.in +0 -0
  37. {followthemoney-3.5.9 → followthemoney-3.6.1}/README.md +0 -0
  38. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/cli/__init__.py +0 -0
  39. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/cli/aggregate.py +0 -0
  40. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/cli/exports.py +0 -0
  41. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/cli/mapping.py +0 -0
  42. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/cli/sieve.py +0 -0
  43. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/cli/util.py +0 -0
  44. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/compare.py +0 -0
  45. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/exc.py +0 -0
  46. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/export/__init__.py +0 -0
  47. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/export/common.py +0 -0
  48. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/export/csv.py +0 -0
  49. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/export/graph.py +0 -0
  50. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/export/neo4j.py +0 -0
  51. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/export/rdf.py +0 -0
  52. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/graph.py +0 -0
  53. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/helpers.py +0 -0
  54. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/mapping/__init__.py +0 -0
  55. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/mapping/csv.py +0 -0
  56. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/mapping/query.py +0 -0
  57. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/mapping/source.py +0 -0
  58. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/messages.py +0 -0
  59. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/model.py +0 -0
  60. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/namespace.py +0 -0
  61. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/offshore.py +0 -0
  62. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/ontology.py +0 -0
  63. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/py.typed +0 -0
  64. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/rdf.py +0 -0
  65. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Address.yaml +0 -0
  66. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Airplane.yaml +0 -0
  67. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Article.yaml +0 -0
  68. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Assessment.yaml +0 -0
  69. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Asset.yaml +0 -0
  70. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Associate.yaml +0 -0
  71. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Audio.yaml +0 -0
  72. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Call.yaml +0 -0
  73. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/CallForTenders.yaml +0 -0
  74. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Company.yaml +0 -0
  75. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Contract.yaml +0 -0
  76. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/ContractAward.yaml +0 -0
  77. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/CourtCase.yaml +0 -0
  78. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/CourtCaseParty.yaml +0 -0
  79. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/CryptoWallet.yaml +0 -0
  80. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Debt.yaml +0 -0
  81. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Directorship.yaml +0 -0
  82. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Document.yaml +0 -0
  83. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Documentation.yml +0 -0
  84. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/EconomicActivity.yaml +0 -0
  85. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Email.yaml +0 -0
  86. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Employment.yaml +0 -0
  87. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Event.yaml +0 -0
  88. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Family.yaml +0 -0
  89. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Folder.yaml +0 -0
  90. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/HyperText.yaml +0 -0
  91. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Identification.yaml +0 -0
  92. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Image.yaml +0 -0
  93. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Interest.yaml +0 -0
  94. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Interval.yaml +0 -0
  95. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/License.yaml +0 -0
  96. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Membership.yaml +0 -0
  97. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Mention.yaml +0 -0
  98. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Message.yaml +0 -0
  99. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Note.yaml +0 -0
  100. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Occupancy.yaml +0 -0
  101. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Organization.yaml +0 -0
  102. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Ownership.yaml +0 -0
  103. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Package.yaml +0 -0
  104. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Page.yaml +0 -0
  105. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Pages.yaml +0 -0
  106. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Passport.yaml +0 -0
  107. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Payment.yaml +0 -0
  108. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/PlainText.yaml +0 -0
  109. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Position.yaml +0 -0
  110. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Post.yaml +0 -0
  111. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Project.yaml +0 -0
  112. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/ProjectParticipant.yaml +0 -0
  113. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/PublicBody.yaml +0 -0
  114. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/RealEstate.yaml +0 -0
  115. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Representation.yaml +0 -0
  116. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Sanction.yaml +0 -0
  117. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Similar.yaml +0 -0
  118. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Succession.yaml +0 -0
  119. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Table.yaml +0 -0
  120. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/TaxRoll.yaml +0 -0
  121. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Trip.yaml +0 -0
  122. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/UnknownLink.yaml +0 -0
  123. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/UserAccount.yaml +0 -0
  124. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Value.yaml +0 -0
  125. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Vehicle.yaml +0 -0
  126. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Vessel.yaml +0 -0
  127. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Video.yaml +0 -0
  128. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema/Workbook.yaml +0 -0
  129. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/schema.py +0 -0
  130. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/translations/ar/LC_MESSAGES/followthemoney.mo +0 -0
  131. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/translations/ar/LC_MESSAGES/followthemoney.po +0 -0
  132. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/translations/bs/LC_MESSAGES/followthemoney.mo +0 -0
  133. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/translations/bs/LC_MESSAGES/followthemoney.po +0 -0
  134. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/translations/de/LC_MESSAGES/followthemoney.mo +0 -0
  135. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/translations/de/LC_MESSAGES/followthemoney.po +0 -0
  136. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/translations/es/LC_MESSAGES/followthemoney.mo +0 -0
  137. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/translations/es/LC_MESSAGES/followthemoney.po +0 -0
  138. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/translations/fr/LC_MESSAGES/followthemoney.mo +0 -0
  139. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/translations/fr/LC_MESSAGES/followthemoney.po +0 -0
  140. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/translations/fr/followthemoney.po +0 -0
  141. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/translations/messages.pot +0 -0
  142. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/translations/nb/LC_MESSAGES/followthemoney.mo +0 -0
  143. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/translations/nb/LC_MESSAGES/followthemoney.po +0 -0
  144. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/translations/nl/LC_MESSAGES/followthemoney.mo +0 -0
  145. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/translations/nl/LC_MESSAGES/followthemoney.po +0 -0
  146. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/translations/pt_BR/LC_MESSAGES/followthemoney.mo +0 -0
  147. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/translations/pt_BR/LC_MESSAGES/followthemoney.po +0 -0
  148. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/translations/ru/LC_MESSAGES/followthemoney.mo +0 -0
  149. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/translations/ru/LC_MESSAGES/followthemoney.po +0 -0
  150. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/translations/ru/followthemoney.po +0 -0
  151. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/translations/tr/LC_MESSAGES/followthemoney.mo +0 -0
  152. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/translations/tr/LC_MESSAGES/followthemoney.po +0 -0
  153. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/types/address.py +0 -0
  154. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/types/checksum.py +0 -0
  155. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/types/country.py +0 -0
  156. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/types/gender.py +0 -0
  157. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/types/json.py +0 -0
  158. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/types/number.py +0 -0
  159. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/types/registry.py +0 -0
  160. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/types/string.py +0 -0
  161. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney/util.py +0 -0
  162. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney.egg-info/SOURCES.txt +0 -0
  163. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney.egg-info/dependency_links.txt +0 -0
  164. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney.egg-info/entry_points.txt +0 -0
  165. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney.egg-info/namespace_packages.txt +0 -0
  166. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney.egg-info/not-zip-safe +0 -0
  167. {followthemoney-3.5.9 → followthemoney-3.6.1}/followthemoney.egg-info/top_level.txt +0 -0
  168. {followthemoney-3.5.9 → followthemoney-3.6.1}/setup.cfg +0 -0
@@ -0,0 +1,94 @@
1
+ Metadata-Version: 2.1
2
+ Name: followthemoney
3
+ Version: 3.6.1
4
+ Summary: UNKNOWN
5
+ Home-page: https://followthemoney.tech/
6
+ Author: Organized Crime and Corruption Reporting Project
7
+ Author-email: data@occrp.org
8
+ License: MIT
9
+ Platform: UNKNOWN
10
+ Classifier: Intended Audience :: Developers
11
+ Classifier: Operating System :: OS Independent
12
+ Classifier: Programming Language :: Python
13
+ Classifier: Programming Language :: Python :: 3.9
14
+ Description-Content-Type: text/markdown
15
+ Provides-Extra: dev
16
+ License-File: LICENSE
17
+
18
+ # Follow the Money
19
+
20
+ [![ftm-build](https://github.com/alephdata/followthemoney/actions/workflows/build.yml/badge.svg)](https://github.com/alephdata/followthemoney/actions/workflows/build.yml)
21
+
22
+ This repository contains a pragmatic data model for the entities most
23
+ commonly used in investigative reporting: people, companies, assets,
24
+ payments, court cases, etc.
25
+
26
+ The purpose of this is not to model reality in an ideal data model, but
27
+ rather to have a working data structure for researchers.
28
+
29
+ `followthemoney` also contains code used to validate and normalize many
30
+ of the elements of data, and to map tabular data into the model.
31
+
32
+ ## Documentation
33
+
34
+ For a general introduction to `followthemoney`, check the high-level introduction:
35
+
36
+ * https://followthemoney.tech
37
+
38
+ Part of this package is a command-line tool that can be used to process and
39
+ transform data in various ways. You can find a tutorial here:
40
+
41
+ * https://followthemoney.tech/docs/cli/
42
+
43
+ Besides the introductions, there is also a full reference documentation for the
44
+ library and the contained ontology:
45
+
46
+ * https://followthemoney.tech/explorer/
47
+
48
+ There's also a number of viewers for the RDF schema definitions generated
49
+ from FollowTheMoney, e.g.:
50
+
51
+ * [LODE documentation](http://150.146.207.114/lode/extract?url=https%3A%2F%2Falephdata.github.io%2Ffollowthemoney%2Fns%2Fftm.xml&owlapi=true&imported=true&lang=en)
52
+ * [WebVOWL](https://service.tib.eu/webvowl/#iri=https://alephdata.github.io/followthemoney/ns/ftm.xml)
53
+ * RDF/OWL specification in [XML](https://alephdata.github.io/followthemoney/ns/ftm.xml).
54
+
55
+ ## Development environment
56
+
57
+ For local development with a virtualenv:
58
+
59
+ ```bash
60
+ python3 -mvenv .env
61
+ source .env/bin/activate
62
+ pip install -e ".[dev]"
63
+ ```
64
+
65
+ Now you can run the tests with
66
+
67
+ ```bash
68
+ make test
69
+ ```
70
+
71
+ ## Releasing
72
+
73
+ We release a lot of version of `followthemoney` because even small changes
74
+ to the code base require a pypi release to begin being used in `aleph`. To
75
+ this end, here's the steps for making a release:
76
+
77
+ ```bash
78
+ git pull --rebase
79
+ make build
80
+ make test
81
+ git add . && git commit -m "Updating translation files"
82
+ bumpversion patch
83
+ git push --atomic origin main $(git describe --tags --abbrev=0)
84
+ ```
85
+
86
+ This will create a new patch release and upload a distribution of it. If
87
+ the changes are more significant, you can run `bumpversion` with the `minor`
88
+ or `major` arguments.
89
+
90
+ When the schema is updated, please update the docs, ideally including the
91
+ diagrams. For the RDF namespace and JavaScript version of the model,
92
+ run `make generate`.
93
+
94
+
@@ -3,7 +3,7 @@ import os
3
3
  from followthemoney.model import Model
4
4
  from followthemoney.util import set_model_locale
5
5
 
6
- __version__ = "3.5.9"
6
+ __version__ = "3.6.1"
7
7
 
8
8
 
9
9
  model_path = os.path.dirname(__file__)
@@ -1,14 +1,14 @@
1
1
  import sys
2
- import json
3
2
  import click
3
+ import orjson
4
4
  import logging
5
5
  from pathlib import Path
6
- from typing import Optional, TextIO
6
+ from typing import Optional, BinaryIO, List, Any, Dict
7
7
  from banal import ensure_list
8
8
 
9
9
  from followthemoney import model
10
10
  from followthemoney.namespace import Namespace
11
- from followthemoney.cli.util import InPath, OutPath, path_entities, read_entities
11
+ from followthemoney.cli.util import InPath, OutPath, path_entities
12
12
  from followthemoney.cli.util import path_writer, write_entity
13
13
  from followthemoney.proxy import EntityProxy
14
14
 
@@ -20,9 +20,10 @@ def cli() -> None:
20
20
 
21
21
 
22
22
  @cli.command("dump-model", help="Export the current schema model")
23
- @click.option("-o", "--outfile", type=click.File("w"), default="-")
24
- def dump_model(outfile: TextIO) -> None:
25
- outfile.write(json.dumps(model.to_dict(), indent=2, sort_keys=True))
23
+ @click.option("-o", "--outfile", type=click.File("wb"), default="-")
24
+ def dump_model(outfile: BinaryIO) -> None:
25
+ f = orjson.OPT_INDENT_2 | orjson.OPT_SORT_KEYS
26
+ outfile.write(orjson.dumps(model.to_dict(), option=f))
26
27
 
27
28
 
28
29
  @cli.command("validate", help="Re-parse and validate the given data")
@@ -34,7 +35,7 @@ def validate(infile: Path, outfile: Path) -> None:
34
35
  for entity in path_entities(infile, EntityProxy, cleaned=False):
35
36
  clean = model.make_entity(entity.schema)
36
37
  clean.id = entity.id
37
- for (prop, value) in entity.itervalues():
38
+ for prop, value in entity.itervalues():
38
39
  clean.add(prop, value)
39
40
  write_entity(outfh, clean)
40
41
  except BrokenPipeError:
@@ -46,12 +47,14 @@ def validate(infile: Path, outfile: Path) -> None:
46
47
  @click.option("-o", "--outfile", type=OutPath, default="-") # noqa
47
48
  def import_vis(infile: Path, outfile: Path) -> None:
48
49
  with path_writer(outfile) as outfh:
49
- with open(infile, "r") as infh:
50
- data = json.load(infh)
50
+ with open(infile, "rb") as infh:
51
+ data: Dict[str, Any] = orjson.loads(infh.read())
51
52
  if "entities" in data:
52
- entities = data.get("entities", data)
53
- if "layout" in data:
53
+ entities: List[Dict[str, Any]] = data.get("entities", data)
54
+ elif "layout" in data:
54
55
  entities = data.get("layout", {}).get("entities", data)
56
+ else:
57
+ raise click.ClickException("No entities found in VIS file")
55
58
  for entity_data in ensure_list(entities):
56
59
  entity = EntityProxy.from_dict(model, entity_data)
57
60
  write_entity(outfh, entity)
@@ -75,10 +78,11 @@ def sign(infile: Path, outfile: Path, signature: Optional[str]) -> None:
75
78
  @cli.command(help="Format a stream of entities to make it readable")
76
79
  @click.option("-i", "--infile", type=InPath, default="-") # noqa
77
80
  def pretty(infile: Path) -> None:
78
- stdout = click.get_text_stream("stdout")
81
+ stdout = click.get_binary_stream("stdout")
79
82
  try:
83
+ f = orjson.OPT_INDENT_2 | orjson.OPT_APPEND_NEWLINE
80
84
  for entity in path_entities(infile, EntityProxy):
81
- data = json.dumps(entity.to_dict(), indent=2)
82
- stdout.write(data + "\n")
85
+ data = orjson.dumps(entity.to_dict(), option=f)
86
+ stdout.write(data)
83
87
  except BrokenPipeError:
84
88
  raise click.Abort()
@@ -1,11 +1,11 @@
1
1
  import logging
2
2
  from io import BytesIO
3
3
  from typing import Dict, List, Optional
4
- from openpyxl import Workbook # type: ignore
5
- from openpyxl.cell import WriteOnlyCell # type: ignore
6
- from openpyxl.styles import Font, PatternFill # type: ignore
7
- from openpyxl.worksheet.worksheet import Worksheet # type: ignore
8
- from openpyxl.utils.exceptions import IllegalCharacterError # type: ignore
4
+ from openpyxl import Workbook
5
+ from openpyxl.cell import WriteOnlyCell
6
+ from openpyxl.styles import Font, PatternFill
7
+ from openpyxl.worksheet.worksheet import Worksheet
8
+ from openpyxl.utils.exceptions import IllegalCharacterError
9
9
 
10
10
  from followthemoney.export.common import Exporter
11
11
  from followthemoney.proxy import E
@@ -25,7 +25,7 @@ class ExcelWriter(object):
25
25
  self.workbook = Workbook(write_only=True)
26
26
 
27
27
  def make_sheet(self, title: str, headers: List[str]) -> Worksheet:
28
- sheet = self.workbook.create_sheet(title=title)
28
+ sheet: Worksheet = self.workbook.create_sheet(title=title)
29
29
  sheet.freeze_panes = "A2"
30
30
  sheet.sheet_properties.filterMode = True
31
31
  cells = []
@@ -1,3 +1,4 @@
1
+ import logging
1
2
  from hashlib import sha1
2
3
  from warnings import warn
3
4
  from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set
@@ -15,6 +16,8 @@ if TYPE_CHECKING:
15
16
  from followthemoney.model import Model
16
17
  from followthemoney.mapping.query import QueryMapping
17
18
 
19
+ log = logging.getLogger(__name__)
20
+
18
21
 
19
22
  class EntityMapping(object):
20
23
 
@@ -112,16 +115,24 @@ class EntityMapping(object):
112
115
  # from that accessible to phone and address parsers.
113
116
  for prop in self.properties:
114
117
  if prop.prop.type == registry.country:
115
- prop.map(proxy, record, entities)
118
+ discarded_values = prop.map(proxy, record, entities)
119
+ for value in discarded_values:
120
+ log.warn(f"[{self.name}] Discarded unclean value \"{value}\" for property \"{prop.prop.qname}\".")
116
121
 
117
122
  for prop in self.properties:
118
123
  if prop.prop.type != registry.country:
119
- prop.map(proxy, record, entities)
124
+ discarded_values = prop.map(proxy, record, entities)
125
+ for value in discarded_values:
126
+ log.warn(f"[{self.name}] Discarding unclean value \"{value}\" for property \"{prop.prop.qname}\".")
120
127
 
121
128
  # Generate the ID at the end to avoid self-reference checks on empty
122
129
  # keys.
123
130
  proxy.id = self.compute_key(record)
124
131
  if proxy.id is None:
132
+ if self.id_column:
133
+ log.warn(f"[{self.name}] Skipping entity because no ID could be computed. Make sure that there are no empty values in the \"{self.id_column}\" column.")
134
+ if self.keys:
135
+ log.warn(f"[{self.name}] Skipping entity because no ID could be computed. Make sure that there are no empty values in key columns.")
125
136
  return None
126
137
 
127
138
  for prop in self.properties:
@@ -130,6 +141,7 @@ class EntityMapping(object):
130
141
  # the mapping, not in the model. Basically it means: if
131
142
  # this row of source data doesn't have that field, then do
132
143
  # not map it again.
144
+ log.warn(f"[{self.name}] Skipping entity because required property \"{prop.prop.name}\" is empty.")
133
145
  return None
134
146
  return proxy
135
147
 
@@ -112,13 +112,13 @@ class PropertyMapping(object):
112
112
 
113
113
  def map(
114
114
  self, proxy: EntityProxy, record: Record, entities: Dict[str, EntityProxy]
115
- ) -> None:
115
+ ) -> List[str]:
116
116
  if self.entity is not None:
117
117
  entity = entities.get(self.entity)
118
118
  if entity is not None:
119
119
  proxy.unsafe_add(self.prop, entity.id, cleaned=True)
120
120
  inline_names(proxy, entity)
121
- return None
121
+ return []
122
122
 
123
123
  # clean the values returned by the query, or by using literals, or
124
124
  # formats.
@@ -133,5 +133,17 @@ class PropertyMapping(object):
133
133
  splote.extend(value.split(self.split))
134
134
  values = splote
135
135
 
136
+ discarded_values: List[str] = []
137
+
136
138
  for value in values:
137
- proxy.unsafe_add(self.prop, value, fuzzy=self.fuzzy, format=self.format)
139
+ added_value = proxy.unsafe_add(
140
+ prop=self.prop,
141
+ value=value,
142
+ fuzzy=self.fuzzy,
143
+ format=self.format,
144
+ )
145
+
146
+ if value is not None and added_value is None:
147
+ discarded_values.append(value)
148
+
149
+ return discarded_values
@@ -55,7 +55,7 @@ class SQLSource(Source):
55
55
  if database is None:
56
56
  raise InvalidMapping("No database in SQL mapping!")
57
57
  self.database_uri = cast(str, os.path.expandvars(database))
58
- self.engine = create_engine(self.database_uri, poolclass=NullPool)
58
+ self.engine = create_engine(self.database_uri, poolclass=NullPool)
59
59
  self.meta = MetaData()
60
60
 
61
61
  tables = keys_values(data, "table", "tables")
@@ -27,6 +27,7 @@ class PropertyDict(TypedDict, total=False):
27
27
  # stub: Optional[bool]
28
28
  rdf: Optional[str]
29
29
  range: Optional[str]
30
+ format: Optional[str]
30
31
 
31
32
 
32
33
  class PropertySpec(PropertyDict):
@@ -58,6 +59,7 @@ class Property:
58
59
  "matchable",
59
60
  "deprecated",
60
61
  "_range",
62
+ "format",
61
63
  "range",
62
64
  "stub",
63
65
  "_reverse",
@@ -113,6 +115,11 @@ class Property:
113
115
  self._range = data.get("range")
114
116
  self.range: Optional["Schema"] = None
115
117
 
118
+ #: If the property is of type ``identifier``, a more narrow definition of the
119
+ #: identifier format can be provided. For example, LEI, INN or IBAN codes
120
+ #: can be automatically validated.
121
+ self.format: Optional[str] = data.get("format")
122
+
116
123
  #: When a property points to another schema, a reverse property is added for
117
124
  #: various administrative reasons. These properties are, however, not real
118
125
  #: and cannot be written to. That's why they are marked as stubs and adding
@@ -169,6 +176,8 @@ class Property:
169
176
  if self.stub:
170
177
  return gettext("Property cannot be written")
171
178
  val = get_entity_id(val)
179
+ if val is None:
180
+ continue
172
181
  if not self.type.validate(val):
173
182
  return gettext("Invalid value")
174
183
  if val is not None:
@@ -203,6 +212,8 @@ class Property:
203
212
  data["range"] = self.range.name
204
213
  if self.reverse is not None:
205
214
  data["reverse"] = self.reverse.name
215
+ if self.format is not None:
216
+ data["format"] = self.format
206
217
  return data
207
218
 
208
219
  def __repr__(self) -> str:
@@ -194,6 +194,7 @@ class EntityProxy(object):
194
194
 
195
195
  for value in value_list(values):
196
196
  if not cleaned:
197
+ format = format or prop.format
197
198
  value = prop.type.clean(value, proxy=self, fuzzy=fuzzy, format=format)
198
199
  self.unsafe_add(prop, value, cleaned=True)
199
200
  return None
@@ -205,26 +206,32 @@ class EntityProxy(object):
205
206
  cleaned: bool = False,
206
207
  fuzzy: bool = False,
207
208
  format: Optional[str] = None,
208
- ) -> None:
209
+ ) -> Optional[str]:
209
210
  """A version of `add()` to be used only in type-checking code. This accepts
210
211
  only a single value, and performs input cleaning on the premise that the
211
- value is already valid unicode."""
212
+ value is already valid unicode. Returns the value that has been added."""
212
213
  if not cleaned and value is not None:
214
+ format = format or prop.format
213
215
  value = prop.type.clean_text(value, fuzzy=fuzzy, format=format, proxy=self)
214
- if value is not None:
215
- # Somewhat hacky: limit the maximum size of any particular
216
- # field to avoid overloading upstream aleph/elasticsearch.
217
- value_size = len(value)
218
- if prop.type.max_size is not None:
219
- if self._size + value_size > prop.type.max_size:
220
- # msg = "[%s] too large. Rejecting additional values."
221
- # log.warning(msg, prop.name)
222
- return None
223
- self._size += value_size
224
- self._properties.setdefault(prop.name, list())
225
- if value not in self._properties[prop.name]:
226
- self._properties[prop.name].append(value)
227
- return None
216
+
217
+ if value is None:
218
+ return None
219
+
220
+ # Somewhat hacky: limit the maximum size of any particular
221
+ # field to avoid overloading upstream aleph/elasticsearch.
222
+ value_size = len(value)
223
+ if prop.type.max_size is not None:
224
+ if self._size + value_size > prop.type.max_size:
225
+ # msg = "[%s] too large. Rejecting additional values."
226
+ # log.warning(msg, prop.name)
227
+ return None
228
+ self._size += value_size
229
+ self._properties.setdefault(prop.name, list())
230
+
231
+ if value not in self._properties[prop.name]:
232
+ self._properties[prop.name].append(value)
233
+
234
+ return value
228
235
 
229
236
  def set(
230
237
  self,
@@ -424,9 +431,12 @@ class EntityProxy(object):
424
431
  dictionary can be used to make a new proxy, and it is commonly written to disk
425
432
  or a database."""
426
433
  data = dict(self.context)
427
- data.update(
428
- {"id": self.id, "schema": self.schema.name, "properties": self.properties}
429
- )
434
+ extra = {
435
+ "id": self.id,
436
+ "schema": self.schema.name,
437
+ "properties": self.properties,
438
+ }
439
+ data.update(extra)
430
440
  return data
431
441
 
432
442
  def to_full_dict(self, matchable: bool = False) -> Dict[str, Any]:
@@ -32,6 +32,8 @@ Analyzable:
32
32
  ibanMentioned:
33
33
  label: "Detected IBANs"
34
34
  hidden: true
35
+ # type: identifier
36
+ # format: iban
35
37
  type: iban
36
38
  ipMentioned:
37
39
  label: "Detected IP addresses"
@@ -28,10 +28,13 @@ BankAccount:
28
28
  type: identifier
29
29
  iban:
30
30
  label: IBAN
31
+ # type: identifier
32
+ # format: iban
31
33
  type: iban
32
34
  bic:
33
35
  label: Bank Identifier Code
34
36
  type: identifier
37
+ format: bic
35
38
  bank:
36
39
  label: Bank
37
40
  type: entity
@@ -107,15 +107,18 @@ LegalEntity:
107
107
  label: "INN"
108
108
  description: "Russian company ID"
109
109
  type: identifier
110
+ # format: inn
110
111
  ogrnCode:
111
112
  label: "OGRN"
112
113
  description: "Major State Registration Number"
113
114
  type: identifier
115
+ # format: ogrn
114
116
  leiCode:
115
117
  # cf. https://www.gleif.org/en/about-lei/introducing-the-legal-entity-identifier-lei
116
118
  label: "LEI"
117
119
  description: "Legal Entity Identifier"
118
120
  type: identifier
121
+ format: lei
119
122
  dunsCode:
120
123
  label: "D-U-N-S"
121
124
  description: "Dun & Bradstreet identifier"
@@ -124,6 +127,7 @@ LegalEntity:
124
127
  label: "SWIFT/BIC"
125
128
  description: "Bank identifier code"
126
129
  type: identifier
130
+ format: bic
127
131
  parent:
128
132
  label: "Parent company"
129
133
  description: "If this entity is a subsidiary, another entity (company or organisation) is its parent"
@@ -26,6 +26,10 @@ Person:
26
26
  title:
27
27
  label: Title
28
28
  rdf: http://xmlns.com/foaf/0.1/title
29
+ # The `firstName`, `lastName`, `secondName` etc. properties intentionally do not use
30
+ # the `name` property type. Many FtM tools (including Aleph) use name properties to
31
+ # compare/match entities, but matching entites just on e.g. a first name would lead to
32
+ # too many false positives.
29
33
  firstName:
30
34
  label: First name
31
35
  rdf: http://xmlns.com/foaf/0.1/givenName
@@ -23,6 +23,7 @@ Security:
23
23
  label: ISIN
24
24
  description: International Securities Identification Number
25
25
  type: identifier
26
+ format: isin
26
27
  registrationNumber:
27
28
  label: Registration number
28
29
  type: identifier
@@ -32,6 +33,7 @@ Security:
32
33
  figiCode:
33
34
  label: Financial Instrument Global Identifier
34
35
  type: identifier
36
+ format: figi
35
37
  issuer:
36
38
  label: "Issuer"
37
39
  type: entity
@@ -57,6 +57,7 @@ Thing:
57
57
  wikidataId:
58
58
  label: Wikidata ID
59
59
  type: identifier
60
+ format: qid
60
61
  keywords:
61
62
  label: Keywords
62
63
  topics:
@@ -3,7 +3,6 @@ from followthemoney.types.url import UrlType
3
3
  from followthemoney.types.name import NameType
4
4
  from followthemoney.types.email import EmailType
5
5
  from followthemoney.types.ip import IpType
6
- from followthemoney.types.iban import IbanType
7
6
  from followthemoney.types.address import AddressType
8
7
  from followthemoney.types.date import DateType
9
8
  from followthemoney.types.phone import PhoneType
@@ -12,6 +11,7 @@ from followthemoney.types.language import LanguageType
12
11
  from followthemoney.types.mimetype import MimeType
13
12
  from followthemoney.types.checksum import ChecksumType
14
13
  from followthemoney.types.identifier import IdentifierType
14
+ from followthemoney.types.iban import IbanType
15
15
  from followthemoney.types.entity import EntityType
16
16
  from followthemoney.types.topic import TopicType
17
17
  from followthemoney.types.gender import GenderType
@@ -27,7 +27,6 @@ registry.add(UrlType)
27
27
  registry.add(NameType)
28
28
  registry.add(EmailType)
29
29
  registry.add(IpType)
30
- registry.add(IbanType)
31
30
  registry.add(AddressType)
32
31
  registry.add(DateType)
33
32
  registry.add(PhoneType)
@@ -36,6 +35,7 @@ registry.add(LanguageType)
36
35
  registry.add(MimeType)
37
36
  registry.add(ChecksumType)
38
37
  registry.add(IdentifierType)
38
+ registry.add(IbanType) # TODO: remove
39
39
  registry.add(EntityType)
40
40
  registry.add(TopicType)
41
41
  registry.add(GenderType)
@@ -1,3 +1,4 @@
1
+ from inspect import cleandoc
1
2
  from itertools import product
2
3
  from babel.core import Locale
3
4
  from banal import ensure_list
@@ -64,12 +65,17 @@ class PropertyType(object):
64
65
 
65
66
  @property
66
67
  def docs(self) -> Optional[str]:
67
- return self.__doc__
68
+ if not self.__doc__:
69
+ return None
70
+
71
+ return cleandoc(self.__doc__)
68
72
 
69
- def validate(self, value: str) -> bool:
73
+ def validate(
74
+ self, value: str, fuzzy: bool = False, format: Optional[str] = None
75
+ ) -> bool:
70
76
  """Returns a boolean to indicate if the given value is a valid instance of
71
77
  the type."""
72
- cleaned = self.clean(value)
78
+ cleaned = self.clean(value, fuzzy=fuzzy, format=format)
73
79
  return cleaned is not None
74
80
 
75
81
  def clean(
@@ -141,7 +147,7 @@ class PropertyType(object):
141
147
  ) -> float:
142
148
  """Compare two sets of values and select the highest-scored result."""
143
149
  results = []
144
- for (l, r) in product(ensure_list(left), ensure_list(right)):
150
+ for l, r in product(ensure_list(left), ensure_list(right)):
145
151
  results.append(self.compare(l, r))
146
152
  if not len(results):
147
153
  return 0.0
@@ -229,7 +235,9 @@ class EnumType(PropertyType):
229
235
  self._names[locale] = self._locale_names(locale)
230
236
  return self._names[locale]
231
237
 
232
- def validate(self, value: str) -> bool:
238
+ def validate(
239
+ self, value: str, fuzzy: bool = False, format: Optional[str] = None
240
+ ) -> bool:
233
241
  """Make sure that the given code value is one of the supported set."""
234
242
  if value is None:
235
243
  return False
@@ -27,9 +27,14 @@ class DateType(PropertyType):
27
27
  plural = _("Dates")
28
28
  matchable = True
29
29
 
30
- def validate(self, value: str) -> bool:
30
+ def validate(
31
+ self, value: str, fuzzy: bool = False, format: Optional[str] = None
32
+ ) -> bool:
31
33
  """Check if a thing is a valid date."""
32
- prefix = parse(value)
34
+ if format is not None:
35
+ prefix = parse_format(value, format)
36
+ else:
37
+ prefix = parse(value)
33
38
  return prefix.precision != Precision.EMPTY
34
39
 
35
40
  def clean_text(
@@ -36,7 +36,9 @@ class EmailType(PropertyType):
36
36
  # except:
37
37
  # return False
38
38
 
39
- def validate(self, value: str) -> bool:
39
+ def validate(
40
+ self, value: str, fuzzy: bool = False, format: Optional[str] = None
41
+ ) -> bool:
40
42
  """Check to see if this is a valid email address."""
41
43
  # TODO: adopt email.utils.parseaddr
42
44
  email = sanitize_text(value)
@@ -29,7 +29,9 @@ class EntityType(PropertyType):
29
29
  matchable = True
30
30
  pivot = True
31
31
 
32
- def validate(self, value: str) -> bool:
32
+ def validate(
33
+ self, value: str, fuzzy: bool = False, format: Optional[str] = None
34
+ ) -> bool:
33
35
  text = sanitize_text(value)
34
36
  if text is None:
35
37
  return False
@@ -1,6 +1,5 @@
1
- from typing import Optional, TYPE_CHECKING, cast
2
- from stdnum import iban # type: ignore
3
- from stdnum.exceptions import ValidationError # type: ignore
1
+ from typing import Optional, TYPE_CHECKING
2
+ from rigour.ids import IBAN
4
3
 
5
4
  from followthemoney.types.common import PropertyType
6
5
  from followthemoney.rdf import URIRef, Identifier
@@ -26,12 +25,11 @@ class IbanType(PropertyType):
26
25
  matchable = True
27
26
  pivot = True
28
27
 
29
- def validate(self, value: str) -> bool:
28
+ def validate(self, value: str, fuzzy: bool = False, format: Optional[str] = None) -> bool:
30
29
  text = sanitize_text(value)
31
- try:
32
- return cast(bool, iban.validate(text))
33
- except ValidationError:
30
+ if text is None:
34
31
  return False
32
+ return IBAN.is_valid(text)
35
33
 
36
34
  def clean_text(
37
35
  self,
@@ -42,7 +40,7 @@ class IbanType(PropertyType):
42
40
  ) -> Optional[str]:
43
41
  """Create a more clean, but still user-facing version of an
44
42
  instance of the type."""
45
- return text.replace(" ", "").upper()
43
+ return IBAN.normalize(text)
46
44
 
47
45
  def country_hint(self, value: str) -> str:
48
46
  return value[:2].lower()
@@ -54,4 +52,4 @@ class IbanType(PropertyType):
54
52
  return f"iban:{value.upper()}"
55
53
 
56
54
  def caption(self, value: str) -> str:
57
- return cast(str, iban.format(value))
55
+ return IBAN.format(value)