nomenklatura-mpt 4.1.9__py3-none-any.whl

nomenklatura/enrich/__init__.py

@@ -0,0 +1,94 @@
+import logging
+from requests import Session
+from importlib import import_module
+from typing import Iterable, Generator, Optional, Type, cast
+from followthemoney import DS, SE
+
+from nomenklatura.cache import Cache
+from nomenklatura.matching import DefaultAlgorithm
+from nomenklatura.enrich.common import Enricher, EnricherConfig
+from nomenklatura.enrich.common import EnrichmentAbort, EnrichmentException
+from nomenklatura.judgement import Judgement
+from nomenklatura.matching.types import ScoringConfig
+from nomenklatura.resolver import Resolver
+
+log = logging.getLogger(__name__)
+__all__ = [
+    "Enricher",
+    "EnrichmentAbort",
+    "EnrichmentException",
+    "make_enricher",
+    "enrich",
+    "match",
+]
+
+
+def make_enricher(
+    dataset: DS,
+    cache: Cache,
+    config: EnricherConfig,
+    http_session: Optional[Session] = None,
+) -> Enricher[DS]:
+    enricher_type = config.pop("type")
+    if ":" not in enricher_type:
+        raise RuntimeError("Invalid import path: %r" % enricher_type)
+    module_name, clazz_name = enricher_type.split(":", 1)
+    module = import_module(module_name)
+    clazz = getattr(module, clazz_name)
+    if clazz is None or not issubclass(clazz, Enricher):
+        raise RuntimeError("Invalid enricher: %r" % enricher_type)
+    enr_clazz = cast(Type[Enricher[DS]], clazz)
+    return enr_clazz(dataset, cache, config, session=http_session)
+
+
+# nk match -i entities.json -o entities-with-matches.json -r resolver.json
+# then:
+# nk dedupe -i entities-with-matches.json -r resolver.json
+def match(
+    enricher: Enricher[DS],
+    resolver: Resolver[SE],
+    entities: Iterable[SE],
+    config: Optional[ScoringConfig] = None,
+) -> Generator[SE, None, None]:
+    if config is None:
+        config = ScoringConfig.defaults()
+    for entity in entities:
+        yield entity
+        try:
+            for match in enricher.match_wrapped(entity):
+                if entity.id is None or match.id is None:
+                    continue
+                if not resolver.check_candidate(entity.id, match.id):
+                    continue
+                if not entity.schema.can_match(match.schema):
+                    continue
+                result = DefaultAlgorithm.compare(entity, match, config)
+                log.info("Match [%s]: %.2f -> %s", entity, result.score, match)
+                resolver.suggest(entity.id, match.id, result.score)
+                match.datasets.add(enricher.dataset.name)
+                match = resolver.apply(match)
+                yield match
+        except EnrichmentException:
+            log.exception("Failed to match: %r" % entity)
+
+
+# nk enrich -i entities.json -r resolver.json -o combined.json
+def enrich(
+    enricher: Enricher[DS], resolver: Resolver[SE], entities: Iterable[SE]
+) -> Generator[SE, None, None]:
+    for entity in entities:
+        try:
+            for match in enricher.match_wrapped(entity):
+                if entity.id is None or match.id is None:
+                    continue
+                judgement = resolver.get_judgement(match.id, entity.id)
+                if judgement != Judgement.POSITIVE:
+                    continue
+
+                log.info("Enrich [%s]: %r", entity, match)
+                for adjacent in enricher.expand_wrapped(entity, match):
+                    adjacent.datasets.add(enricher.dataset.name)
+                    adjacent = resolver.apply(adjacent)
+                    yield adjacent
+        except EnrichmentException:
+            log.exception("Failed to enrich: %r" % entity)

nomenklatura/enrich/aleph.py

@@ -0,0 +1,141 @@
+import os
+import uuid
+import logging
+from banal import is_mapping, ensure_list, hash_data
+from typing import Any, Dict, cast, Generator, Optional
+from urllib.parse import urljoin
+from functools import cached_property
+from followthemoney.exc import InvalidData
+from followthemoney.namespace import Namespace
+from followthemoney import DS, SE
+from requests import Session
+from rigour.urls import build_url
+
+from nomenklatura.cache import Cache
+from nomenklatura.enrich.common import Enricher, EnricherConfig
+
+log = logging.getLogger(__name__)
+
+
+class AlephEnricher(Enricher[DS]):
+    def __init__(
+        self,
+        dataset: DS,
+        cache: Cache,
+        config: EnricherConfig,
+        session: Optional[Session] = None,
+    ):
+        super().__init__(dataset, cache, config, session)
+        self._host: str = os.environ.get("ALEPH_HOST", "https://aleph.occrp.org/")
+        self._host = self.get_config_expand("host") or self._host
+        self._base_url: str = urljoin(self._host, "/api/2/")
+        self._collection: Optional[str] = self.get_config_expand("collection")
+        self._ns: Optional[Namespace] = None
+        if self.get_config_bool("strip_namespace"):
+            self._ns = Namespace()
+        self._api_key: Optional[str] = os.environ.get("ALEPH_API_KEY")
+        self._api_key = self.get_config_expand("api_key") or self._api_key
+        if self._api_key is not None:
+            self.session.headers["Authorization"] = f"ApiKey {self._api_key}"
+        self.session.headers["X-Aleph-Session"] = str(uuid.uuid4())
+
+    @cached_property
+    def collection_id(self) -> Optional[str]:
+        if self._collection is None:
+            return None
+        url = urljoin(self._base_url, "collections")
+        url = build_url(url, {"filter:foreign_id": self._collection})
+        res = self.session.get(url)
+        res.raise_for_status()
+        response = res.json()
+        for result in response.get("results", []):
+            return cast(str, result["id"])
+        return None
+
+    def load_aleph_entity(self, entity: SE, data: Dict[str, Any]) -> Optional[SE]:
+        data["referents"] = [data["id"]]
+        try:
+            proxy = super().load_entity(entity, data)
+        except InvalidData:
+            log.warning("Server model mismatch: %s" % data.get("schema"))
+            return None
+        links = data.get("links", {})
+        proxy.add("alephUrl", links.get("self"), quiet=True, cleaned=True)
+        collection = data.get("collection", {})
+        proxy.add("publisher", collection.get("label"), quiet=True, cleaned=True)
+        # clinks = collection.get("links", {})
+        # entity.add("publisherUrl", clinks.get("ui"), quiet=True, cleaned=True)
+        return proxy
+
+    def convert_nested(
+        self, entity: SE, data: Dict[str, Any]
+    ) -> Generator[SE, None, None]:
+        proxy = self.load_aleph_entity(entity, data)
+        if proxy is not None:
+            if self._ns is not None:
+                entity = self._ns.apply(entity)
+            yield proxy
+        properties = data.get("properties", {})
+        for _, values in properties.items():
+            for value in ensure_list(values):
+                if is_mapping(value):
+                    proxy = self.load_aleph_entity(entity, value)
+                    if proxy is not None:
+                        yield proxy
+
+    # def enrich_entity(self, entity):
+    #     url = self.api._make_url("match")
+    #     for page in range(10):
+    #         data = self.post_match(url, entity)
+    #         for res in data.get("results", []):
+    #             proxy = self.convert_entity(res)
+    #             yield self.make_match(entity, proxy)
+
+    #         url = data.get("next")
+    #         if url is None:
+    #             break
+
+    # def expand_entity(self, entity):
+    #     for url in entity.get("alephUrl", quiet=True):
+    #         data = self.get_api(url)
+    #         yield from self.convert_nested(data)
+
+    #         _, entity_id = url.rsplit("/", 1)
+    #         filters = (("entities", entity_id),)
+    #         search_api = self.api._make_url("entities", filters=filters)
+    #         while True:
+    #             res = self.get_api(search_api)
+    #             for data in ensure_list(res.get("results")):
+    #                 yield from self.convert_nested(data)
+
+    #             search_api = res.get("next")
+    #             if search_api is None:
+    #                 break
+
+    def match(self, entity: SE) -> Generator[SE, None, None]:
+        if not entity.schema.matchable:
+            return
+        url = urljoin(self._base_url, "match")
+        if self.collection_id is not None:
+            url = build_url(url, {"collection_ids": self.collection_id})
+        query = {
+            "schema": entity.schema.name,
+            "properties": entity.properties,
+        }
+        cache_id = entity.id or hash_data(query)
+        cache_key = f"{url}:{cache_id}"
+        response = self.http_post_json_cached(url, cache_key, query)
+        for result in response.get("results", []):
+            proxy = self.load_aleph_entity(entity, result)
+            if proxy is not None:
+                if self._ns is not None:
+                    entity = self._ns.apply(entity)
+                yield proxy
+
+    def expand(self, entity: SE, match: SE) -> Generator[SE, None, None]:
+        url = urljoin(self._base_url, f"entities/{match.id}")
+        for aleph_url in match.get("alephUrl", quiet=True):
+            if aleph_url.startswith(self._base_url):
+                url = aleph_url.replace("/entities/", "/api/2/entities/")
+        response = self.http_get_json_cached(url)
+        yield from self.convert_nested(match, response)

nomenklatura/enrich/common.py

@@ -0,0 +1,219 @@
+import os
+import json
+import logging
+import traceback
+from banal import as_bool
+from typing import Union, Any, Dict, Optional, Generator, Generic
+from abc import ABC, abstractmethod
+from requests import Session
+from requests.exceptions import RequestException, ChunkedEncodingError
+from followthemoney import DS, registry
+from followthemoney import StatementEntity, SE
+from followthemoney.types.topic import TopicType
+from rigour.urls import build_url, ParamsType
+
+from nomenklatura import __version__
+from nomenklatura.cache import Cache
+from nomenklatura.util import HeadersType
+
+EnricherConfig = Dict[str, Any]
+log = logging.getLogger(__name__)
+
+
+class EnrichmentException(Exception):
+    pass
+
+
+class EnrichmentAbort(Exception):
+    pass
+
+
+class BaseEnricher(Generic[DS]):
+    def __init__(self, dataset: DS, cache: Cache, config: EnricherConfig):
+        self.dataset = dataset
+        self.cache = cache
+        self.config = config
+        self.cache_days = int(config.pop("cache_days", 90))
+        self._filter_schemata = config.pop("schemata", [])
+        self._filter_topics = config.pop("topics", [])
+
+    def get_config_expand(
+        self, name: str, default: Optional[str] = None
+    ) -> Optional[str]:
+        value = self.config.get(name, default)
+        if value is None:
+            return None
+        return str(os.path.expandvars(value))
+
+    def get_config_int(self, name: str, default: Union[int, str]) -> int:
+        return int(self.config.get(name, default))
+
+    def get_config_bool(self, name: str, default: Union[bool, str] = False) -> int:
+        return as_bool(self.config.get(name, default))
+
+    def _filter_entity(self, entity: StatementEntity) -> bool:
+        """Check if the given entity should be filtered out. Filters
+        can be applied by schema or by topic."""
+        if len(self._filter_schemata):
+            if entity.schema.name not in self._filter_schemata:
+                return False
+        _filter_topics = set(self._filter_topics)
+        if "all" in _filter_topics:
+            assert isinstance(registry.topic, TopicType)
+            _filter_topics.update(registry.topic.names.keys())
+        if len(_filter_topics):
+            topics = set(entity.get_type_values(registry.topic))
+            if not len(topics.intersection(_filter_topics)):
+                return False
+        return True
+
+
+class Enricher(BaseEnricher[DS], ABC):
+    def __init__(
+        self,
+        dataset: DS,
+        cache: Cache,
+        config: EnricherConfig,
+        session: Optional[Session] = None,
+    ):
+        super().__init__(dataset, cache, config)
+        self._session: Optional[Session] = session
+
+    @property
+    def session(self) -> Session:
+        if self._session is None:
+            self._session = Session()
+            self._session.headers["User-Agent"] = f"nomenklatura/{__version__}"
+        return self._session
+
+    def http_get_cached(
+        self,
+        url: str,
+        params: ParamsType = None,
+        hidden: ParamsType = None,
+        cache_days: Optional[int] = None,
+    ) -> str:
+        url = build_url(url, params=params)
+        cache_days_ = self.cache_days if cache_days is None else cache_days
+        response = self.cache.get(url, max_age=cache_days_)
+        if response is None:
+            log.debug("HTTP GET: %s", url)
+            hidden_url = build_url(url, params=hidden)
+            try:
+                resp = self.session.get(hidden_url)
+                resp.raise_for_status()
+            except RequestException as rex:
+                if rex.response is not None and rex.response.status_code in (401, 403):
+                    raise EnrichmentAbort("Authorization failure: %s" % url) from rex
+                msg = "HTTP fetch failed [%s]: %s" % (url, rex)
+                log.info(f"{msg}\n{traceback.format_exc()}")
+                raise EnrichmentException(msg) from rex
+            response = resp.text
+            if cache_days_ > 0:
+                self.cache.set(url, response)
+        return response
+
+    def http_remove_cache(self, url: str, params: ParamsType = None) -> None:
+        url = build_url(url, params=params)
+        self.cache.delete(url)
+
+    def http_get_json_cached(
+        self,
+        url: str,
+        params: ParamsType = None,
+        hidden: ParamsType = None,
+        cache_days: Optional[int] = None,
+    ) -> Any:
+        res = self.http_get_cached(url, params, hidden=hidden, cache_days=cache_days)
+        return json.loads(res)
+
+    def http_post_json_cached(
+        self,
+        url: str,
+        cache_key: str,
+        json: Any = None,
+        data: Any = None,
+        headers: HeadersType = None,
+        cache_days: Optional[int] = None,
+        retry_chunked_encoding_error: int = 1,
+    ) -> Any:
+        cache_days_ = self.cache_days if cache_days is None else cache_days
+        resp_data = self.cache.get_json(cache_key, max_age=cache_days_)
+        if resp_data is None:
+            try:
+                resp = self.session.post(url, json=json, data=data, headers=headers)
+                resp.raise_for_status()
+            except ChunkedEncodingError as rex:
+                # Due to https://github.com/urllib3/urllib3/issues/2751#issuecomment-2567630065,
+                # urllib3's Retry strategy will not retry on chunked encoding errors.
+                # Since urllib won't retry it, retry it here.
+                # urllib does close the connection.
+                if (
+                    "Response ended prematurely" in str(rex)
+                    and retry_chunked_encoding_error > 0
+                ):
+                    log.info("Retrying due to chunked encoding error: %s", rex)
+                    return self.http_post_json_cached(
+                        url,
+                        cache_key,
+                        json=json,
+                        data=data,
+                        headers=headers,
+                        cache_days=cache_days,
+                        retry_chunked_encoding_error=retry_chunked_encoding_error - 1,
+                    )
+
+                msg = "HTTP POST failed [%s]: %s" % (url, rex)
+                raise EnrichmentException(msg) from rex
+            except RequestException as rex:
+                if rex.response is not None and rex.response.status_code in (401, 403):
+                    raise EnrichmentAbort("Authorization failure: %s" % url) from rex
+
+                msg = "HTTP POST failed [%s]: %s" % (url, rex)
+                log.info(f"{msg}\n{traceback.format_exc()}")
+                raise EnrichmentException(msg) from rex
+            resp_data = resp.json()
+            if cache_days_ > 0:
+                self.cache.set_json(cache_key, resp_data)
+        return resp_data
+
+    def _make_data_entity(
+        self, entity: SE, data: Dict[str, Any], cleaned: bool = True
+    ) -> SE:
+        """Create an entity which is of the same sub-type of SE as the given
+        query entity."""
+        return type(entity).from_data(self.dataset, data, cleaned=cleaned)
+
+    def load_entity(self, entity: SE, data: Dict[str, Any]) -> SE:
+        proxy = self._make_data_entity(entity, data, cleaned=False)
+        for prop in proxy.iterprops():
+            if prop.stub:
+                proxy.pop(prop)
+        return proxy
+
+    def make_entity(self, entity: SE, schema: str) -> SE:
+        """Create a new entity of the given schema."""
+        return self._make_data_entity(entity, {"schema": schema})
+
+    def match_wrapped(self, entity: SE) -> Generator[SE, None, None]:
+        if not self._filter_entity(entity):
+            return
+        yield from self.match(entity)
+
+    def expand_wrapped(self, entity: SE, match: SE) -> Generator[SE, None, None]:
+        if not self._filter_entity(entity):
+            return
+        yield from self.expand(entity, match)
+
+    @abstractmethod
+    def match(self, entity: SE) -> Generator[SE, None, None]:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def expand(self, entity: SE, match: SE) -> Generator[SE, None, None]:
+        raise NotImplementedError()
+
+    def close(self) -> None:
+        self.cache.close()
+        if self._session is not None:
+            self._session.close()

nomenklatura/enrich/nominatim.py

@@ -0,0 +1,72 @@
+import logging
+from normality import squash_spaces
+from typing import Any, Dict, Iterable, Generator, Optional
+
+from requests import Session
+from followthemoney import DS, SE
+from followthemoney import StatementEntity
+
+from nomenklatura.cache import Cache
+from nomenklatura.enrich.common import Enricher, EnricherConfig
+
+
+log = logging.getLogger(__name__)
+NOMINATIM = "https://nominatim.openstreetmap.org/search.php"
+
+
+class NominatimEnricher(Enricher[DS]):
+    def __init__(
+        self,
+        dataset: DS,
+        cache: Cache,
+        config: EnricherConfig,
+        session: Optional[Session] = None,
+    ):
+        super().__init__(dataset, cache, config, session)
+        self.cache.preload(f"{NOMINATIM}%")
+
+    def search_nominatim(self, address: StatementEntity) -> Iterable[Dict[str, Any]]:
+        for full in address.get("full"):
+            full_norm = squash_spaces(full)
+            if len(full_norm) < 5:
+                log.warning("Skipping tiny address: %s", full)
+                continue
+            params = {
+                "q": full_norm,
+                "countrycodes": address.get("country"),
+                "format": "jsonv2",
+                "accept-language": "en",
+                "addressdetails": 1,
+            }
+            results = self.http_get_json_cached(NOMINATIM, params)
+            log.info("OpenStreetMap geocoded [%s]: %d results", full, len(results))
+            for result in results:
+                yield result
+                # FIXME: only best result for now.
+                return
+
+    def match(self, entity: SE) -> Generator[SE, None, None]:
+        if not entity.schema.is_a("Address"):
+            return
+
+        for result in self.search_nominatim(entity):
+            # pprint(result)
+            addr = self.make_entity(entity, "Address")
+            osm_type = result.get("osm_type")
+            osm_id = result.get("osm_id")
+            if osm_id is None or osm_type is None:
+                continue
+            addr.id = f"osm-{osm_type}-{osm_id}"
+            addr.add("full", result["display_name"])
+            # addr.add("latitude", result.get("lat"))
+            # addr.add("longitude", result.get("lon"))
+            addr_data: Dict[str, str] = result.get("address", {})
+            addr.add("country", addr_data.get("country"))
+            addr.add("country", addr_data.get("country_code"))
+            addr.add("city", addr_data.get("city"))
+            addr.add("state", addr_data.get("state"))
+            addr.add("postalCode", addr_data.get("postcode"))
+            yield addr
+
+    def expand(self, entity: SE, match: SE) -> Generator[SE, None, None]:
+        yield match