nomenklatura-mpt 4.1.9__py3-none-any.whl

nomenklatura/enrich/opencorporates.py

@@ -0,0 +1,233 @@
+import json
+import logging
+from normality import slugify_text
+from typing import cast, Any, Dict, Generator, Optional
+from urllib.parse import urlparse
+from banal import ensure_dict
+from followthemoney import registry, DS, SE
+from requests import Session
+from requests.exceptions import RequestException
+from rigour.urls import build_url, ParamsType
+
+from nomenklatura.cache import Cache
+from nomenklatura.enrich.common import Enricher, EnricherConfig
+from nomenklatura.enrich.common import EnrichmentAbort, EnrichmentException
+
+
+log = logging.getLogger(__name__)
+
+
+def parse_date(raw: Any) -> Optional[str]:
+    return registry.date.clean(raw)
+
+
+class OpenCorporatesEnricher(Enricher[DS]):
+    COMPANY_SEARCH_API = "https://api.opencorporates.com/v0.4/companies/search"
+    OFFICER_SEARCH_API = "https://api.opencorporates.com/v0.4/officers/search"
+    UI_PART = "://opencorporates.com/"
+    API_PART = "://api.opencorporates.com/v0.4/"
+
+    def __init__(
+        self,
+        dataset: DS,
+        cache: Cache,
+        config: EnricherConfig,
+        session: Optional[Session] = None,
+    ):
+        super().__init__(dataset, cache, config, session)
+        token_var = "${OPENCORPORATES_API_TOKEN}"
+        self.api_token: Optional[str] = self.get_config_expand("api_token", token_var)
+        self.quota_exceeded = False
+        if self.api_token == token_var:
+            self.api_token = None
+        if self.api_token is None:
+            log.warning("OpenCorporates has no API token (%s)" % token_var)
+        self.cache.preload(f"{self.COMPANY_SEARCH_API}%")
+
+    def oc_get_cached(self, url: str, params: ParamsType = None) -> Optional[Any]:
+        url = build_url(url, params=params)
+        response = self.cache.get(url, max_age=self.cache_days)
+        if response is None:
+            if self.quota_exceeded:
+                return None
+            hidden_url = build_url(url, params={"api_token": self.api_token})
+            try:
+                resp = self.session.get(hidden_url)
+                resp.raise_for_status()
+            except RequestException as rex:
+                if rex.response is not None:
+                    if rex.response.status_code in (403, 429):
+                        log.info("OpenCorporates quota exceeded; using only cache now.")
+                        self.quota_exceeded = True
+                        return None
+                    elif rex.response.status_code == 401:
+                        raise EnrichmentAbort(
+                            "Authorization failure: %s" % url
+                        ) from rex
+                msg = "HTTP fetch failed [%s]: %s" % (url, rex)
+                raise EnrichmentException(msg) from rex
+            response = resp.text
+            self.cache.set(url, response)
+        return json.loads(response)
+
+    def match(self, entity: SE) -> Generator[SE, None, None]:
+        if not entity.schema.matchable:
+            return
+        if entity.has("opencorporatesUrl"):
+            # TODO: fetch entity here when we start to expand with content!
+            return
+
+        if entity.schema.name in ["Company", "Organization", "LegalEntity"]:
+            yield from self.search_companies(entity)
+        if entity.schema.name in ["Person", "LegalEntity", "Company", "Organization"]:
+            # yield from self.search_officers(entity)
+            pass
+
+    def expand(self, entity: SE, match: SE) -> Generator[SE, None, None]:
+        clone = self.make_entity(match, match.schema.name)
+        clone.id = match.id
+        clone.add("opencorporatesUrl", match.get("opencorporatesUrl"))
+        yield clone
+
+    # def expand_entity(self, entity):
+    #     for url in entity.get("opencorporatesUrl", quiet=True):
+    #         url = self.make_url(url)
+    #         data = self.get_api(url).get("results", {})
+    #         if "company" in data:
+    #             yield from self.expand_company(entity, data)
+    #         if "officer" in data:
+    #             yield from self.expand_officer(data, officer=entity)
+
+    def make_entity_id(self, url: str) -> str:
+        parsed = urlparse(url)
+        path = slugify_text(parsed.path, sep="-")
+        assert path is not None, "Invalid OpenCorporates URL: %s" % url
+        return f"oc-{path}"
+
+    def jurisdiction_to_country(self, juris: Optional[Any]) -> Optional[str]:
+        if juris is None:
+            return None
+        return str(juris).split("_", 1)[0]
+
+    def company_entity(
+        self, ref: SE, data: Dict[str, Any], entity: Optional[SE] = None
+    ) -> SE:
+        if "company" in data:
+            data = ensure_dict(data.get("company", data))
+        oc_url = cast(Optional[str], data.get("opencorporates_url"))
+        if oc_url is None:
+            raise ValueError("Company has no URL: %r" % data)
+        if entity is None:
+            entity = self.make_entity(ref, "Company")
+            entity.id = self.make_entity_id(oc_url)
+        entity.add("name", data.get("name"))
+
+        # TODO: make this an adjacent object?
+        address: Dict[str, Any] = ensure_dict(data.get("registered_address"))
+        entity.add("country", address.get("country"))
+
+        juris = self.jurisdiction_to_country(data.get("jurisdiction_code"))
+        entity.add("jurisdiction", juris)
+        entity.add("alias", data.get("alternative_names"))
+        entity.add("address", data.get("registered_address_in_full"))
+        entity.add("sourceUrl", data.get("registry_url"))
+        entity.add("legalForm", data.get("company_type"))
+        inc_date = data.get("incorporation_date")
+        entity.add("incorporationDate", parse_date(inc_date))
+        dis_date = data.get("dissolution_date")
+        entity.add("dissolutionDate", parse_date(dis_date))
+        entity.add("status", data.get("current_status"))
+        entity.add("registrationNumber", data.get("company_number"))
+        entity.add("opencorporatesUrl", oc_url)
+        source = data.get("source", {})
+        entity.add("publisher", source.get("publisher"))
+        entity.add("publisherUrl", source.get("url"))
+        entity.add("retrievedAt", parse_date(source.get("retrieved_at")))
+        for code in data.get("industry_codes", []):
+            code = code.get("industry_code", code)
+            entity.add("sector", code.get("description"))
+        for previous in data.get("previous_names", []):
+            entity.add("previousName", previous.get("company_name"))
+        for alias in data.get("alternative_names", []):
+            entity.add("alias", alias.get("company_name"))
+        return entity
+
+    # def officer_entity(self, data, entity=None):
+    #     if "officer" in data:
+    #         data = ensure_dict(data.get("officer", data))
+    #     person = data.get("occupation") or data.get("date_of_birth")
+    #     schema = "Person" if person else "LegalEntity"
+    #     entity = model.make_entity(schema)
+    #     entity.make_id(data.get("opencorporates_url"))
+    #     entity.add("name", data.get("name"))
+    #     entity.add("country", data.get("nationality"))
+    #     entity.add("jurisdiction", data.get("jurisdiction_code"))
+    #     entity.add("address", data.get("address"))
+    #     entity.add("birthDate", data.get("date_of_birth"), quiet=True)
+    #     entity.add("position", data.get("occupation"), quiet=True)
+    #     entity.add("opencorporatesUrl", data.get("opencorporates_url"))
+    #     source = data.get("source", {})
+    #     entity.add("publisher", source.get("publisher"))
+    #     entity.add("publisherUrl", source.get("url"))
+    #     entity.add("retrievedAt", source.get("retrieved_at"))
+    #     return entity
+
+    def search_companies(self, entity: SE) -> Generator[SE, None, None]:
+        countries = entity.get_type_values(registry.country)
+        params = {"q": entity.caption, "sparse": True, "country_codes": countries}
+        for page in range(1, 9):
+            params["page"] = page
+            results = self.oc_get_cached(self.COMPANY_SEARCH_API, params=params)
+            if results is None:
+                break
+
+            # print(results)
+            for company in results.get("results", {}).get("companies", []):
+                proxy = self.company_entity(entity, company)
+                yield proxy
+            if page >= results.get("total_pages", 0):
+                break
+
+    # def search_officers(self, entity):
+    #     params = self.get_query(entity)
+    #     for page in range(1, 9):
+    #         params["page"] = page
+    #         url = self.make_url(self.OFFICER_SEARCH_API, params)
+    #         results = self.get_api(url)
+    #         officers = results.get("results", {}).get("officers")
+    #         for officer in ensure_list(officers):
+    #             proxy = self.officer_entity(officer)
+    #             yield self.make_match(entity, proxy)
+    #         if page >= results.get("total_pages", 0):
+    #             break
+
+    # def enrich_entity(self, entity):
+    #     schema = entity.schema.name
+
+    #     if schema in ["Person", "LegalEntity", "Company", "Organization"]:
+    #         yield from self.search_officers(entity)
+
+    # def expand_company(self, entity, data):
+    #     data = ensure_dict(data.get("company", data))
+    #     entity = self.company_entity(data, entity=entity)
+    #     for officer in ensure_list(data.get("officers")):
+    #         yield from self.expand_officer(officer, company=entity)
+    #     yield entity
+
+    # def expand_officer(self, data, entity=None, company=None):
+    #     data = ensure_dict(data.get("officer", data))
+    #     entity = self.officer_entity(data, entity=entity)
+    #     yield entity
+
+    #     company = self.company_entity(data.get("company"), entity=company)
+    #     yield company
+
+    #     if company.id and entity.id:
+    #         directorship = model.make_entity("Directorship")
+    #         directorship.make_id(data.get("opencorporates_url"), "Directorship")
+    #         directorship.add("director", entity)
+    #         directorship.add("startDate", data.get("start_date"))
+    #         directorship.add("endDate", data.get("end_date"))
+    #         directorship.add("organization", company)
+    #         directorship.add("role", data.get("position"))
+    #         yield directorship

nomenklatura/enrich/openfigi.py

@@ -0,0 +1,124 @@
+import os
+import logging
+from typing import Generator, Dict, Optional
+from followthemoney.util import make_entity_id
+from followthemoney import DS, SE
+from requests import Session
+
+from nomenklatura.cache import Cache
+from nomenklatura.enrich.common import Enricher, EnricherConfig
+
+log = logging.getLogger(__name__)
+
+
+class OpenFIGIEnricher(Enricher[DS]):
+    """Uses the `OpenFIGI` search API to look up FIGIs by company name."""
+
+    SEARCH_URL = "https://api.openfigi.com/v3/search"
+    MAPPING_URL = "https://api.openfigi.com/v3/mapping"
+
+    def __init__(
+        self,
+        dataset: DS,
+        cache: Cache,
+        config: EnricherConfig,
+        session: Optional[Session] = None,
+    ):
+        super().__init__(dataset, cache, config, session)
+        api_key_var = "${OPENFIGI_API_KEY}"
+        self.api_key: Optional[str] = self.get_config_expand("api_key", api_key_var)
+        if self.api_key == api_key_var:
+            self.api_key = None
+        if self.api_key is None:
+            log.warning("PermID has no API token (%s)" % api_key_var)
+
+        api_key = os.environ.get("OPENFIGI_API_KEY")
+        if api_key is not None:
+            self.session.headers["X-OPENFIGI-APIKEY"] = api_key
+
+    def make_company_id(self, name: str) -> str:
+        return f"figi-company-{make_entity_id(name)}"
+
+    def make_security_id(self, figi: str) -> str:
+        return f"figi-{figi}"
+
+    def search(self, query: str) -> Generator[Dict[str, str], None, None]:
+        body = {"query": query}
+        next = None
+
+        while True:
+            if next is not None:
+                body["start"] = next
+
+            log.info(f"Searching {query!r}, offset={next}")
+            cache_key = f"{self.SEARCH_URL}:{query}:{next}"
+            resp = self.http_post_json_cached(self.SEARCH_URL, cache_key, json=body)
+            if "data" in resp:
+                yield from resp["data"]
+
+            next = resp.get("next", None)
+            if next is None:
+                break
+
+    def match_organization(self, entity: SE) -> Generator[SE, None, None]:
+        for name in entity.get("name"):
+            for match in self.search(name):
+                match_name = match.get("name", None)
+                if match_name is None:
+                    continue
+                other = self.make_entity(entity, "Company")
+                other.id = self.make_company_id(match_name)
+                other.add("name", match_name)
+                other.add("topics", "corp.public")
+                yield other
+
+    def match_security(self, entity: SE) -> Generator[SE, None, None]:
+        for isin in entity.get("isin"):
+            cache_key = f"{self.MAPPING_URL}:ISIN:{isin}"
+            query = [{"idType": "ID_ISIN", "idValue": isin}]
+            resp = self.http_post_json_cached(self.MAPPING_URL, cache_key, json=query)
+            for section in resp:
+                for item in section.get("data", []):
+                    figi = item["figi"]
+                    if figi != item.get("compositeFIGI", figi):
+                        continue
+                    security = self.make_entity(entity, "Security")
+                    # security.id = self.make_security_id(item["figi"])
+                    security.id = entity.id
+                    security.add("isin", isin)
+                    security.add("figiCode", item["figi"])
+                    security.add("ticker", item["ticker"])
+                    security.add("type", item["securityType"])
+                    yield security
+
+    def match(self, entity: SE) -> Generator[SE, None, None]:
+        if entity.schema.is_a("Organization"):
+            yield from self.match_organization(entity)
+        if entity.schema.is_a("Security"):
+            yield from self.match_security(entity)
+
+    def expand(self, entity: SE, match: SE) -> Generator[SE, None, None]:
+        if match.schema.is_a("Security"):
+            yield match
+        if match.schema.is_a("Organization"):
+            name = match.first("name")
+            if name is None:
+                return
+            yield match
+            for item in self.search(name):
+                # Only emit the securities which match the name of the positive match
+                # to the company exactly. Skip everything else.
+                if item["name"] != name:
+                    continue
+
+                figi = item["figi"]
+                security = self.make_entity(match, "Security")
+                security.id = self.make_security_id(figi)
+                security.add("figiCode", figi)
+                security.add("issuer", match)
+                security.add("ticker", item["ticker"])
+                security.add("type", item["securityType"])
+                # if item["exchCode"] is not None:
+                #     security.add("notes", f'exchange {item["exchCode"]}')
+                security.add("description", item["securityDescription"])
+                yield security

nomenklatura/enrich/permid.py

@@ -0,0 +1,201 @@
+import csv
+import io
+import json
+import logging
+from functools import lru_cache
+from itertools import product
+from typing import cast, Set, Generator, Optional, Dict, Any
+from urllib.parse import urljoin
+
+from followthemoney import StatementEntity, registry, DS, SE
+from lxml import etree
+from requests import Session
+
+from nomenklatura.cache import Cache
+from nomenklatura.enrich.common import Enricher, EnricherConfig
+from nomenklatura.enrich.common import EnrichmentAbort
+from nomenklatura.matching.compat import fingerprint_name
+
+log = logging.getLogger(__name__)
+
+GN = "{http://www.geonames.org/ontology#}"
+STATUS = {
+    "tr-org:statusActive": "Active",
+    "tr-org:statusInActive": "Inactive",
+}
+
+
+class PermIDEnricher(Enricher[DS]):
+    MATCHING_API = "https://api-eit.refinitiv.com/permid/match"
+
+    def __init__(
+        self,
+        dataset: DS,
+        cache: Cache,
+        config: EnricherConfig,
+        session: Optional[Session] = None,
+    ):
+        super().__init__(dataset, cache, config, session)
+        token_var = "${PERMID_API_TOKEN}"
+        self.api_token: Optional[str] = self.get_config_expand("api_token", token_var)
+        if self.api_token == token_var:
+            self.api_token = None
+        if self.api_token is None:
+            log.warning("PermID has no API token (%s)" % token_var)
+        self.quota_exceeded = False
+
+    def entity_to_queries(self, entity: StatementEntity) -> bytes:
+        names = entity.get_type_values(registry.name, matchable=True)
+        countries = entity.get("jurisdiction", quiet=True)
+        if not len(countries):
+            countries = entity.get_type_values(registry.country, matchable=True)
+        country_set = {c.upper()[:2] for c in countries}
+        if len(country_set) == 0:
+            country_set.add("")
+        if len(names) * len(country_set) < 999:
+            country_set.add("")
+        if len(names) * len(country_set) < 999:
+            fp = fingerprint_name(entity.caption)
+            if fp is not None and fp not in names:
+                names.append(fp)
+        for name in entity.get("name", quiet=True):
+            if len(names) * len(country_set) >= 999:
+                break
+            fp = fingerprint_name(entity.caption)
+            if fp is not None and fp not in names:
+                names.append(fp)
+        sio = io.StringIO()
+        writer = csv.writer(sio, dialect=csv.unix_dialect, delimiter=",")
+        # LocalID,Standard Identifier,Name,Country,Street,City,PostalCode,State,Website
+        writer.writerow(["LocalID", "Standard Identifier", "Name", "Country"])
+        lei_code = entity.first("leiCode", quiet=True)
+        if lei_code is not None:
+            lei_code = f"LEI:{lei_code}"
+        else:
+            lei_code = ""
+        for name, country in list(product(names, country_set))[:999]:
+            writer.writerow([entity.id, lei_code, name, country])
+        sio.seek(0)
+        return sio.getvalue().encode("utf-8")
+
+    @lru_cache(maxsize=1000)
+    def fetch_placename(self, value: Optional[str]) -> Optional[str]:
+        if value is None:
+            return None
+        if not value.startswith("http://sws.geonames.org/"):
+            raise ValueError("Not a GeoNames URL: %s" % value)
+        url = urljoin(value, "about.rdf")
+        res = self.http_get_cached(url, cache_days=120)
+        try:
+            doc = etree.fromstring(res.encode("utf=8"))
+        except Exception:
+            log.warn("Invalid GeoNames response: %s", url)
+            self.http_remove_cache(url)
+            return None
+        for code in doc.findall(".//%scountryCode" % GN):
+            return code.text
+        for name in doc.findall(".//%sname" % GN):
+            return name.text
+        return value
+
+    def fetch_permid(self, url: str) -> Optional[Dict[str, Any]]:
+        params = {"format": "json-ld"}
+        hidden = {"access-token": self.api_token}
+        res_raw = self.http_get_cached(url, params=params, hidden=hidden, cache_days=90)
+        try:
+            return cast(Dict[str, Any], json.loads(res_raw))
+        except Exception:
+            log.info("Invalid response from PermID: %s", url)
+            self.http_remove_cache(url, params=params)
+            return None
+
+    def fetch_perm_org(self, entity: SE, url: str) -> Optional[SE]:
+        res = self.fetch_permid(url)
+        if res is None:
+            return None
+        res.pop("@id", None)
+        res.pop("@type", None)
+        res.pop("@context", None)
+        res.pop("hasPrimaryIndustryGroup", None)
+
+        perm_id = res.pop("tr-common:hasPermId", url.rsplit("-", 1)[-1])
+        lei_code = res.pop("tr-org:hasLEI", None)
+        match = self.make_entity(entity, "Company")
+        match.id = f"lei-{lei_code}" if lei_code is not None else f"permid-{perm_id}"
+        match.add("sourceUrl", url)
+        match.add("leiCode", lei_code)
+        match.add("permId", perm_id)
+        match.add("name", res.pop("vcard:organization-name", None))
+        match.add("website", res.pop("hasURL", None))
+        match.add("country", self.fetch_placename(res.pop("isDomiciledIn", None)))
+        incorporated = self.fetch_placename(res.pop("isIncorporatedIn", None))
+        match.add("jurisdiction", incorporated)
+        inc_date = res.pop("hasLatestOrganizationFoundedDate", None)
+        match.add("incorporationDate", inc_date)
+
+        hq_addr = res.pop("mdaas:HeadquartersAddress", None)
+        reg_addr = res.pop("mdaas:RegisteredAddress", None)
+        for addr in (hq_addr, reg_addr):
+            if addr is not None:
+                addr = ", ".join(addr.split("\n"))
+                addr = addr.replace(",,", ",").strip().strip(",")
+                match.add("address", addr)
+        status_uri = res.pop("hasActivityStatus", None)
+        status = STATUS.get(status_uri)
+        if status is None:
+            log.warning("Unknown status: %s" % status_uri)
+        match.add("status", status)
+        match.add("phone", res.pop("tr-org:hasHeadquartersPhoneNumber", None))
+        match.add("phone", res.pop("tr-org:hasRegisteredPhoneNumber", None))
+        res.pop("tr-org:hasHeadquartersFaxNumber", None)
+        res.pop("tr-org:hasRegisteredFaxNumber", None)
+
+        quote = res.pop("hasOrganizationPrimaryQuote", None)
+        if quote is not None:
+            quote_res = self.fetch_permid(quote)
+            if quote_res is not None:
+                match.add("ticker", quote_res.pop("tr-fin:hasExchangeTicker", None))
+                match.add("ricCode", quote_res.pop("tr-fin:hasRic", None))
+                match.add("topics", "corp.public")
+        return match
+
+    def match(self, entity: SE) -> Generator[SE, None, None]:
+        if self.quota_exceeded:
+            return
+        if not entity.schema.is_a("Organization"):
+            return
+        try:
+            for permid in entity.get("permId", quiet=True):
+                permid_url = f"https://permid.org/1-{permid}"
+                match = self.fetch_perm_org(entity, permid_url)
+                if match is not None:
+                    yield match
+            headers = {
+                "x-openmatch-numberOfMatchesPerRecord": "4",
+                "X-AG-Access-Token": self.api_token,
+                "x-openmatch-dataType": "Organization",
+            }
+            cache_key = f"permid:{entity.id}"
+            query = self.entity_to_queries(entity)
+            res = self.http_post_json_cached(
+                self.MATCHING_API,
+                cache_key,
+                data=query,
+                headers=headers,
+                cache_days=self.cache_days,
+            )
+            seen_matches: Set[str] = set()
+            for result in res.get("outputContentResponse", []):
+                match_permid_url = result.get("Match OpenPermID")
+                if match_permid_url is None or match_permid_url in seen_matches:
+                    continue
+                seen_matches.add(match_permid_url)
+                match = self.fetch_perm_org(entity, match_permid_url)
+                if match is not None:
+                    yield match
+        except EnrichmentAbort as exc:
+            self.quota_exceeded = True
+            log.warning("PermID quota exceeded: %s", exc)
+
+    def expand(self, entity: SE, match: SE) -> Generator[SE, None, None]:
+        yield match