followthemoney 1.3.7__py3-none-any.whl → 3.8.1__py3-none-any.whl

followthemoney/types/string.py

@@ -0,0 +1,43 @@
+from followthemoney.types.common import PropertyType
+from followthemoney.util import defer as _
+from followthemoney.util import MEGABYTE
+
+
+class StringType(PropertyType):
+    """A simple string property with no additional semantics."""
+
+    name = "string"
+    label = _("Label")
+    plural = _("Labels")
+    matchable = False
+    max_length = 1024
+
+    def node_id(self, value: str) -> None:
+        return None
+
+
+class TextType(StringType):
+    """Longer text fragments, such as descriptions or document text. Unlike
+    string properties, it might make sense to treat properties of this type as
+    full-text search material."""
+
+    name = "text"
+    label = _("Text")
+    plural = _("Texts")
+    total_size = 30 * MEGABYTE
+    max_length = 65000
+
+
+class HTMLType(StringType):
+    """Properties that contain raw hypertext markup (HTML).
+
+    User interfaces rendering properties of this type need to take extreme
+    care not to allow attacks such as cross-site scripting. It is recommended
+    to perform server-side sanitisation, or to not render this property at all.
+    """
+
+    name = "html"
+    label = _("HTML")
+    plural = _("HTMLs")
+    total_size = 30 * MEGABYTE
+    max_length = 65000

followthemoney/types/topic.py

@@ -0,0 +1,94 @@
+from babel.core import Locale
+
+from followthemoney.types.common import EnumType, EnumValues
+from followthemoney.rdf import URIRef, Identifier
+from followthemoney.util import gettext, defer as _
+
+
+class TopicType(EnumType):
+    """Topics define a controlled vocabulary of terms applicable to some
+    entities, such as companies and people. They describe categories of
+    journalistic interest which may apply to the given entity, for example
+    if a given person is a criminal or a politician.
+
+    Besides the informative value, topics are ultimately supposed to bear
+    fruits in the context of graph-based data analysis, where they would
+    enable queries such as _find all paths between a government procurement
+    award and a politician_."""
+
+    name = "topic"
+    group = "topics"
+    label = _("Topic")
+    plural = _("Topics")
+    matchable = False
+    max_length = 64
+
+    _TOPICS = {
+        "crime": _("Crime"),
+        "crime.fraud": _("Fraud"),
+        "crime.cyber": _("Cybercrime"),
+        "crime.fin": _("Financial crime"),
+        "crime.env": _("Environmental violations"),
+        "crime.theft": _("Theft"),
+        "crime.war": _("War crimes"),
+        "crime.boss": _("Criminal leadership"),
+        "crime.terror": _("Terrorism"),
+        "crime.traffick": _("Trafficking"),
+        "crime.traffick.drug": _("Drug trafficking"),
+        "crime.traffick.human": _("Human trafficking"),
+        "wanted": _("Wanted"),
+        "corp.offshore": _("Offshore"),
+        "corp.shell": _("Shell company"),
+        "corp.public": _("Public listed company"),
+        "corp.disqual": _("Disqualified"),
+        "gov": _("Government"),
+        "gov.national": _("National government"),
+        "gov.state": _("State government"),
+        "gov.muni": _("Municipal government"),
+        "gov.soe": _("State-owned enterprise"),
+        "gov.igo": _("Intergovernmental organization"),
+        "gov.head": _("Head of government or state"),
+        "gov.admin": _("Civil service"),
+        "gov.executive": _("Executive branch of government"),
+        "gov.legislative": _("Legislative branch of government"),
+        "gov.judicial": _("Judicial branch of government"),
+        "gov.security": _("Security services"),
+        "gov.financial": _("Central banking and financial integrity"),
+        "fin": _("Financial services"),
+        "fin.bank": _("Bank"),
+        "fin.fund": _("Fund"),
+        "fin.adivsor": _("Financial advisor"),
+        "reg.action": _("Regulator action"),
+        "reg.warn": _("Regulator warning"),
+        "role.pep": _("Politician"),
+        "role.pol": _("Non-PEP"),
+        "role.rca": _("Close Associate"),
+        "role.judge": _("Judge"),
+        "role.civil": _("Civil servant"),
+        "role.diplo": _("Diplomat"),
+        "role.lawyer": _("Lawyer"),
+        "role.acct": _("Accountant"),
+        "role.spy": _("Spy"),
+        "role.oligarch": _("Oligarch"),
+        "role.journo": _("Journalist"),
+        "role.act": _("Activist"),
+        "role.lobby": _("Lobbyist"),
+        "pol.party": _("Political party"),
+        "pol.union": _("Union"),
+        "rel": _("Religion"),
+        "mil": _("Military"),
+        "asset.frozen": _("Frozen asset"),
+        "sanction": _("Sanctioned entity"),
+        "sanction.linked": _("Sanction-linked entity"),
+        "sanction.counter": _("Counter-sanctioned entity"),
+        "export.control": _("Export controlled"),
+        "export.risk": _("Trade risk"),
+        "debarment": _("Debarred entity"),
+        "poi": _("Person of interest"),
+    }
+
+    def _locale_names(self, locale: Locale) -> EnumValues:
+        return {k: gettext(v) for (k, v) in self._TOPICS.items()}
+
+    def rdf(self, value: str) -> Identifier:
+        return URIRef(f"ftm:topic:{value}")

followthemoney/types/url.py

@@ -1,27 +1,49 @@
-from rdflib import URIRef
-from urlnormalizer import normalize_url, is_valid_url
+from typing import Optional, TYPE_CHECKING
+from rigour.urls import clean_url, compare_urls
 
 from followthemoney.types.common import PropertyType
+from followthemoney.rdf import URIRef, Identifier
+from followthemoney.util import dampen, defer as _
+
+if TYPE_CHECKING:
+    from followthemoney.proxy import EntityProxy
 
 
 class UrlType(PropertyType):
-    name = 'url'
-    group = 'urls'
-    prefix = 'url'
+    """A uniform resource locator (URL). This will perform some normalisation
+    on the URL so that it's sure to be using valid encoding/quoting, and to
+    make sure the URL has a schema (e.g. `http`, `https`, ...)."""
+
+    SCHEMES = ("http", "https", "ftp", "mailto")
+    DEFAULT_SCHEME = "http"
 
-    def validate(self, url, **kwargs):
-        """Check if `url` is a valid URL."""
-        return is_valid_url(url)
+    name = "url"
+    group = "urls"
+    label = _("URL")
+    plural = _("URLs")
+    matchable = True
+    pivot = True
+    max_length = 4096
 
-    def clean_text(self, url, **kwargs):
-        """Perform intensive care on URLs, see `urlnormalizer`."""
-        try:
-            return normalize_url(url)
-        except UnicodeDecodeError:
-            return None
+    def clean_text(
+        self,
+        text: str,
+        fuzzy: bool = False,
+        format: Optional[str] = None,
+        proxy: Optional["EntityProxy"] = None,
+    ) -> Optional[str]:
+        """Perform intensive care on URLs to make sure they have a scheme
+        and a host name. If no scheme is given HTTP is assumed."""
+        return clean_url(text)
 
-    def specificity(self, value):
-        return 1
+    def compare(self, left: str, right: str) -> float:
+        return compare_urls(left, right)
 
-    def rdf(self, value):
+    def _specificity(self, value: str) -> float:
+        return dampen(10, 120, value)
+
+    def rdf(self, value: str) -> Identifier:
         return URIRef(value)
+
+    def node_id(self, value: str) -> Optional[str]:
+        return f"url:{value}"

followthemoney/util.py

@@ -1,63 +1,157 @@
 import os
-from threading import local
-from normality import stringify
+import logging
+from hashlib import sha1
 from babel import Locale
 from gettext import translation
-from rdflib import Namespace
-from banal import is_mapping, is_sequence
-from banal import unique_list, ensure_list
 
-NAMESPACE = Namespace('https://w3id.org/ftm#')
-DEFAULT_LOCALE = 'en'
-i18n_path = os.path.join(os.path.dirname(__file__), 'translations')
+from threading import local
+from typing import cast, Dict, Any, List, Optional, TypeVar, Union, Sequence
+from normality import stringify
+from normality.cleaning import compose_nfc
+from normality.cleaning import remove_unsafe_chars
+from normality.encoding import DEFAULT_ENCODING
+from banal import is_mapping, unique_list, ensure_list
+
+MEGABYTE = 1024 * 1024
+DEFAULT_LOCALE = "en"
+ENTITY_ID_LEN = 200
+
+T = TypeVar("T")
+K = TypeVar("K")
+V = TypeVar("V")
+
+PathLike = Union[str, os.PathLike[str]]
+i18n_path = os.path.join(os.path.dirname(__file__), "translations")
 state = local()
+log = logging.getLogger(__name__)
+
 
+def gettext(*args: Optional[str], **kwargs: Dict[str, str]) -> str:
+    if not hasattr(state, "translation"):
+        set_model_locale(Locale.parse(DEFAULT_LOCALE))
+    return cast(str, state.translation.gettext(*args, **kwargs))
 
-def gettext(*args, **kwargs):
-    if not hasattr(state, 'translation'):
-        set_model_locale(DEFAULT_LOCALE)
-    return state.translation.gettext(*args, **kwargs)
 
+def defer(text: str) -> str:
+    return text
 
-def set_model_locale(locale):
+
+def set_model_locale(locale: Locale) -> None:
     state.locale = locale
-    state.translation = translation('followthemoney', i18n_path, [locale],
-                                    fallback=True)
+    state.translation = translation(
+        "followthemoney", i18n_path, [str(locale)], fallback=True
+    )
+
+
+def get_locale() -> Locale:
+    if not hasattr(state, "locale"):
+        return Locale.parse(DEFAULT_LOCALE)
+    return Locale.parse(state.locale)
+
+
+def get_env_list(name: str, default: List[str] = []) -> List[str]:
+    value = stringify(os.environ.get(name))
+    if value is not None:
+        values = value.split(":")
+        if len(values):
+            return values
+    return default
+
+
+def sanitize_text(text: Any, encoding: str = DEFAULT_ENCODING) -> Optional[str]:
+    text = stringify(text, encoding_default=encoding)
+    if text is None:
+        return None
+    try:
+        text = compose_nfc(text)
+    except (SystemError, Exception) as ex:
+        log.warning("Cannot NFC text: %s", ex)
+        return None
+    text = remove_unsafe_chars(text)
+    if text is None:
+        return None
+    byte_text = text.encode(DEFAULT_ENCODING, "replace")
+    return cast(str, byte_text.decode(DEFAULT_ENCODING, "replace"))
 
 
-def get_locale():
-    if not hasattr(state, 'locale'):
-        return Locale(DEFAULT_LOCALE)
-    return Locale(state.locale)
+def value_list(value: Union[T, Sequence[T]]) -> List[T]:
+    if not isinstance(value, (str, bytes)):
+        try:
+            return [v for v in cast(Sequence[T], value)]
+        except TypeError:
+            pass
+    return [cast(T, value)]
 
 
-def key_bytes(key):
+def key_bytes(key: Any) -> bytes:
     """Convert the given data to a value appropriate for hashing."""
-    key = stringify(key) or ''
-    return key.encode('utf-8')
-
-
-def merge_data(old, new):
-    """Extend the values of the new doc with extra values from the old."""
-    if is_sequence(old) or is_sequence(new):
-        new = ensure_list(new)
-        new.extend(ensure_list(old))
-        return unique_list(new)
-    if is_mapping(old) or is_mapping(new):
-        old = old if is_mapping(old) else {}
-        new = new if is_mapping(new) else {}
-        keys = set(new.keys())
-        keys.update(old.keys())
-        combined = {}
-        for key in keys:
-            value = merge_data(old.get(key), new.get(key))
-            if value is not None:
-                combined[key] = value
-        return combined
-    return new or old
-
-
-def dampen(short, long, text):
+    if isinstance(key, bytes):
+        return key
+    text = stringify(key)
+    if text is None:
+        return b""
+    return text.encode("utf-8")
+
+
+def join_text(*parts: Any, sep: str = " ") -> Optional[str]:
+    """Join all the non-null arguments using sep."""
+    texts: List[str] = []
+    for part in parts:
+        text = stringify(part)
+        if text is not None:
+            texts.append(text)
+    if not len(texts):
+        return None
+    return sep.join(texts)
+
+
+def get_entity_id(obj: Any) -> Optional[str]:
+    """Given an entity-ish object, try to get the ID."""
+    if is_mapping(obj):
+        obj = obj.get("id")
+    else:
+        try:
+            obj = obj.id
+        except AttributeError:
+            pass
+    return stringify(obj)
+
+
+def make_entity_id(*parts: Any, key_prefix: Optional[str] = None) -> Optional[str]:
+    digest = sha1()
+    if key_prefix:
+        digest.update(key_bytes(key_prefix))
+    base = digest.digest()
+    for part in parts:
+        digest.update(key_bytes(part))
+    if digest.digest() == base:
+        return None
+    return digest.hexdigest()
+
+
+def merge_context(left: Dict[K, V], right: Dict[K, V]) -> Dict[K, List[V]]:
+    """When merging two entities, make lists of all the duplicate context
+    keys."""
+    combined = {}
+    keys = [*left.keys(), *right.keys()]
+    for key in set(keys):
+        if key in ("caption",):
+            continue
+        lval: List[V] = [i for i in ensure_list(left.get(key)) if i is not None]
+        rval: List[V] = [i for i in ensure_list(right.get(key)) if i is not None]
+        combined[key] = unique_list([*lval, *rval])
+    return combined
+
+
+def dampen(short: int, long: int, text: str) -> float:
     length = len(text) - short
     baseline = max(1.0, (long - short))
     return max(0, min(1.0, (length / baseline)))
+
+
+def shortest(*texts: str) -> str:
+    return min(texts, key=len)
+
+
+def longest(*texts: str) -> str:
+    return max(texts, key=len)

followthemoney-3.8.1.dist-info/METADATA

@@ -0,0 +1,153 @@
+Metadata-Version: 2.4
+Name: followthemoney
+Version: 3.8.1
+Summary: A data model for anti corruption data modeling and analysis.
+Project-URL: Documentation, https://followthemoney.tech/
+Project-URL: Repository, https://github.com/alephdata/followthemoney.git
+Project-URL: Issues, https://github.com/alephdata/followthemoney/issues
+Author-email: Organized Crime and Corruption Reporting Project <data@occrp.org>, OpenSanctions <info@opensanctions.org>
+License: MIT License
+        
+        Copyright (c) 2017-2024 Journalism Development Network, Inc.
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.10
+Requires-Dist: babel<3.0.0,>=2.14.0
+Requires-Dist: banal<1.1.0,>=1.0.6
+Requires-Dist: click<9.0.0,>=8.0
+Requires-Dist: countrynames<2.0.0,>=1.13.0
+Requires-Dist: fingerprints<2.0.0,>=1.0.1
+Requires-Dist: networkx<3.5,>=2.5
+Requires-Dist: normality<3.0.0,>=2.4.0
+Requires-Dist: openpyxl<4.0.0,>=3.0.5
+Requires-Dist: orjson<4.0,>=3.7
+Requires-Dist: phonenumbers<9.0.0,>=8.12.22
+Requires-Dist: prefixdate<1.0.0,>=0.4.0
+Requires-Dist: python-stdnum<2.0.0,>=1.16
+Requires-Dist: pytz>=2021.1
+Requires-Dist: pyyaml<7.0.0,>=5.0.0
+Requires-Dist: rdflib<7.2.0,>=6.2.0
+Requires-Dist: requests<3.0.0,>=2.21.0
+Requires-Dist: rigour<1.0.0,>=0.9.3
+Requires-Dist: sqlalchemy2-stubs
+Requires-Dist: sqlalchemy<3.0.0,>=1.4.49
+Requires-Dist: stringcase<2.0.0,>=1.2.0
+Requires-Dist: types-pyyaml
+Provides-Extra: dev
+Requires-Dist: build; extra == 'dev'
+Requires-Dist: bump2version; extra == 'dev'
+Requires-Dist: coverage>=4.1; extra == 'dev'
+Requires-Dist: flake8>=2.6.0; extra == 'dev'
+Requires-Dist: mypy; extra == 'dev'
+Requires-Dist: pip>=10.0.0; extra == 'dev'
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: pytest-cov; extra == 'dev'
+Requires-Dist: recommonmark>=0.4.0; extra == 'dev'
+Requires-Dist: responses>=0.9.0; extra == 'dev'
+Requires-Dist: transifex-client; extra == 'dev'
+Requires-Dist: twine; extra == 'dev'
+Requires-Dist: types-openpyxl; extra == 'dev'
+Requires-Dist: types-pyyaml; extra == 'dev'
+Requires-Dist: types-requests; extra == 'dev'
+Requires-Dist: types-setuptools; extra == 'dev'
+Requires-Dist: wheel>=0.29.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# Follow the Money
+
+[![ftm-build](https://github.com/alephdata/followthemoney/actions/workflows/build.yml/badge.svg)](https://github.com/alephdata/followthemoney/actions/workflows/build.yml)
+
+This repository contains a pragmatic data model for the entities most
+commonly used in investigative reporting: people, companies, assets,
+payments, court cases, etc.
+
+The purpose of this is not to model reality in an ideal data model, but
+rather to have a working data structure for researchers.
+
+`followthemoney` also contains code used to validate and normalize many
+of the elements of data, and to map tabular data into the model.
+
+## Documentation
+
+For a general introduction to `followthemoney`, check the high-level introduction:
+
+* https://followthemoney.tech
+
+Part of this package is a command-line tool that can be used to process and
+transform data in various ways. You can find a tutorial here:
+
+* https://followthemoney.tech/docs/cli/
+
+Besides the introductions, there is also a full reference documentation for the
+library and the contained ontology: 
+
+* https://followthemoney.tech/explorer/
+
+There's also a number of viewers for the RDF schema definitions generated
+from FollowTheMoney, e.g.:
+
+* [LODE documentation](http://150.146.207.114/lode/extract?url=https%3A%2F%2Falephdata.github.io%2Ffollowthemoney%2Fns%2Fftm.xml&owlapi=true&imported=true&lang=en)
+* [WebVOWL](https://service.tib.eu/webvowl/#iri=https://alephdata.github.io/followthemoney/ns/ftm.xml)
+* RDF/OWL specification in [XML](https://alephdata.github.io/followthemoney/ns/ftm.xml).
+
+## Development environment
+
+For local development with a virtualenv:
+
+```bash
+python3 -mvenv .env
+source .env/bin/activate
+pip install -e ".[dev]"
+```
+
+Now you can run the tests with
+
+```bash
+make test
+```
+
+## Releasing
+
+We release a lot of version of `followthemoney` because even small changes
+to the code base require a pypi release to begin being used in `aleph`. To
+this end, here's the steps for making a release:
+
+```bash
+git pull --rebase
+make build
+make test
+git add . && git commit -m "Updating translation files"
+bumpversion patch
+git push --atomic origin main $(git describe --tags --abbrev=0)
+```
+
+This will create a new patch release and upload a distribution of it. If
+the changes are more significant, you can run `bumpversion` with the `minor`
+or `major` arguments.
+
+When the schema is updated, please update the docs, ideally including the
+diagrams. For the RDF namespace and JavaScript version of the model, 
+run `make generate`.