nomenklatura-mpt 4.1.9__py3-none-any.whl

nomenklatura/wikidata/client.py

@@ -0,0 +1,122 @@
+import json
+import logging
+from functools import lru_cache
+from typing import Any, List, Optional, Dict
+from requests import Session
+from normality import squash_spaces
+from rigour.urls import build_url
+from nomenklatura.cache import Cache
+from nomenklatura.wikidata.lang import LangText
+from nomenklatura.wikidata.model import Item
+from nomenklatura.wikidata.query import SparqlResponse
+
+log = logging.getLogger(__name__)
+
+
+class WikidataClient(object):
+    WD_API = "https://www.wikidata.org/w/api.php"
+    QUERY_API = "https://query.wikidata.org/sparql"
+    QUERY_HEADERS = {
+        "Accept": "application/sparql-results+json",
+    }
+    CACHE_SHORT = 1
+    CACHE_MEDIUM = CACHE_SHORT * 7
+    CACHE_LONG = CACHE_SHORT * 30
+
+    LABEL_PREFIX = "wd:lb:"
+    LABEL_CACHE_DAYS = 100
+
+    def __init__(
+        self, cache: Cache, session: Optional[Session] = None, cache_days: int = 14
+    ) -> None:
+        self.cache = cache
+        self.session = session or Session()
+        self.cache_days = cache_days
+        # self.cache.preload(f"{self.LABEL_PREFIX}%")
+
+    @lru_cache(maxsize=1000)
+    def fetch_item(self, qid: str) -> Optional[Item]:
+        # https://www.mediawiki.org/wiki/Wikibase/API
+        # https://www.wikidata.org/w/api.php?action=help&modules=wbgetentities
+        params = {"format": "json", "ids": qid, "action": "wbgetentities"}
+        url = build_url(self.WD_API, params=params)
+        raw = self.cache.get(url, max_age=self.cache_days)
+        if raw is None:
+            res = self.session.get(url)
+            res.raise_for_status()
+            raw = res.text
+            self.cache.set(url, raw)
+        data = json.loads(raw)
+        entity = data.get("entities", {}).get(qid)
+        if entity is None:
+            return None
+        return Item(self, entity)
+
+    @lru_cache(maxsize=100000)
+    def get_label(self, qid: str) -> LangText:
+        cache_key = f"{self.LABEL_PREFIX}{qid}"
+        cached = self.cache.get_json(cache_key, max_age=self.LABEL_CACHE_DAYS)
+        if cached is not None:
+            return LangText.parse(cached)
+        params = {
+            "format": "json",
+            "ids": qid,
+            "action": "wbgetentities",
+            "props": "labels",
+        }
+        url = build_url(self.WD_API, params=params)
+        res = self.session.get(url)
+        res.raise_for_status()
+        data: Dict[str, Any] = res.json()
+        entity = data.get("entities", {}).get(qid)
+        if entity is None:
+            return LangText(None)
+        labels = LangText.from_dict(entity.get("labels", {}))
+        label = LangText.pick(labels)
+        if label is None:
+            label = LangText(qid)
+        label.original = qid
+        self.cache.set_json(cache_key, label.pack())
+        return label
+
+    def query(self, query_text: str) -> SparqlResponse:
+        """Query the Wikidata SPARQL endpoint."""
+        clean_text = squash_spaces(query_text)
+        if len(clean_text) == 0:
+            raise RuntimeError("Invalid query: %r" % query_text)
+        params = {"query": clean_text}
+        url = build_url(self.QUERY_API, params=params)
+        raw = self.cache.get(url, max_age=self.cache_days)
+        if raw is None:
+            res = self.session.get(url, headers=self.QUERY_HEADERS)
+            res.raise_for_status()
+            raw = res.text
+            self.cache.set(url, raw)
+        try:
+            data = json.loads(raw)
+        except json.JSONDecodeError as err:
+            self.cache.delete(url)
+            log.exception("Failed to parse JSON: %s", err)
+            return SparqlResponse(clean_text, {})
+        return SparqlResponse(clean_text, data)
+
+    @lru_cache(maxsize=30000)
+    def _type_props(self, qid: str) -> List[str]:
+        item = self.fetch_item(qid)
+        if item is None:
+            return []
+        types: List[str] = []
+        for claim in item.claims:
+            # historical countries are always historical:
+            ended = claim.qualifiers.get("P582") is not None and claim.qid != "Q3024240"
+            if ended or claim.qid is None:
+                continue
+            if claim.property in ("P31", "P279"):
+                types.append(claim.qid)
+        return types
+
+    def __repr__(self) -> str:
+        return "<WikidataClient()>"
+
+    def __hash__(self) -> int:
+        return 42

nomenklatura/wikidata/lang.py

@@ -0,0 +1,94 @@
+import logging
+from rigour.langs import PREFERRED_LANGS
+from typing import Callable, Dict, Iterable, List, Optional, Any, Set
+from followthemoney import registry, StatementEntity
+from normality.cleaning import remove_unsafe_chars
+
+log = logging.getLogger(__name__)
+
+
+class LangText(object):
+    __slots__ = ["text", "lang", "original"]
+
+    def __init__(
+        self,
+        text: Optional[str],
+        lang: Optional[str] = None,
+        original: Optional[str] = None,
+    ) -> None:
+        if text is None or len(text.strip()) == 0:
+            text = None
+        if text is not None:
+            text = remove_unsafe_chars(text)
+        self.text = text
+        self.lang: Optional[str] = None
+        if lang is not None:
+            self.lang = registry.language.clean_text(lang)
+        if lang is not None and self.lang is None:
+            # Language is given, but it is not one supported by the FtM ecosystem:
+            self.text = None
+        self.original = original or self.text
+
+    def apply(
+        self,
+        entity: StatementEntity,
+        prop: str,
+        clean: Optional[Callable[[str], Optional[str]]] = None,
+    ) -> None:
+        if self.text is None:
+            return
+        clean_text = self.text if clean is None else clean(self.text)
+        if clean_text is None or clean_text.strip() == "":
+            return
+        entity.add(prop, clean_text, lang=self.lang, original_value=self.original)
+
+    def pack(self) -> Dict[str, Optional[str]]:
+        data = {"t": self.text, "l": self.lang}
+        if self.original is not None and self.original != self.text:
+            data["o"] = self.original
+        return data
+
+    @classmethod
+    def parse(cls, data: Dict[str, Optional[str]]) -> "LangText":
+        return LangText(data["t"], data["l"], original=data.get("o"))
+
+    @classmethod
+    def pick(cls, texts: Iterable["LangText"]) -> Optional["LangText"]:
+        for lang in PREFERRED_LANGS:
+            for lt in texts:
+                if lt.lang == lang:
+                    return lt
+        for lt in texts:
+            return lt
+        return None
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, List[Dict[str, str]]]) -> Set["LangText"]:
+        langs: Set[LangText] = set()
+        for objs in data.values():
+            if not isinstance(objs, list):
+                objs = [objs]
+            for obj in objs:
+                value = obj["value"]
+                if value is None:
+                    continue
+                lang = obj["language"]
+                lt = LangText(value, lang, original=value)
+                if lt.text is None:
+                    continue
+                langs.add(lt)
+        return langs
+
+    def __str__(self) -> str:
+        if self.text is None:
+            return ""
+        return self.text
+
+    def __hash__(self) -> int:
+        return hash((self.text, self.lang, self.original))
+
+    def __eq__(self, other: Any) -> bool:
+        return hash(self) == hash(other)
+
+    def __repr__(self) -> str:
+        return f"<LangText({self.text!r}, {self.lang!r}, {self.original!r})>"

nomenklatura/wikidata/model.py

@@ -0,0 +1,139 @@
+from normality import stringify
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set
+
+from nomenklatura.wikidata.value import snak_value_to_string
+from nomenklatura.wikidata.lang import LangText
+
+if TYPE_CHECKING:
+    from nomenklatura.wikidata.client import WikidataClient
+
+
+class Snak(object):
+    """Some Notation About Knowledge (TM)."""
+
+    def __init__(self, client: "WikidataClient", data: Dict[str, Any]):
+        self.client = client
+        datavalue = data.pop("datavalue", {})
+        self.value_type: str = datavalue.pop("type", None)
+        self._value = datavalue.pop("value", None)
+        data.pop("hash", None)
+        self.type = data.pop("datatype", None)
+        self.property: Optional[str] = data.pop("property", None)
+        self.snaktype = data.pop("snaktype", None)
+        # self._data = data
+
+    @property
+    def property_label(self) -> LangText:
+        return self.client.get_label(self.property)
+
+    @property
+    def qid(self) -> Optional[str]:
+        if self.value_type == "wikibase-entityid":
+            return stringify(self._value.get("id"))
+        return None
+
+    @property
+    def text(self) -> LangText:
+        return snak_value_to_string(self.client, self.value_type, self._value)
+
+    def __repr__(self) -> str:
+        return f"<Snak({self.qid}, {self.property}, {self.value_type})>"
+
+
+class Reference(object):
+    def __init__(self, client: "WikidataClient", data: Dict[str, Any]) -> None:
+        self.snaks: Dict[str, List[Snak]] = {}
+        for prop, snak_data in data.pop("snaks", {}).items():
+            self.snaks[prop] = [Snak(client, s) for s in snak_data]
+
+    def get(self, prop: str) -> List[Snak]:
+        return self.snaks.get(prop, [])
+
+
+class Claim(Snak):
+    def __init__(
+        self, client: "WikidataClient", data: Dict[str, Any], prop: str
+    ) -> None:
+        self.id = data.pop("id")
+        self.rank = data.pop("rank")
+        super().__init__(client, data.pop("mainsnak"))
+        self.qualifiers: Dict[str, List[Snak]] = {}
+        for prop, snaks in data.pop("qualifiers", {}).items():
+            self.qualifiers[prop] = [Snak(client, s) for s in snaks]
+
+        self.references = [Reference(client, r) for r in data.pop("references", [])]
+        self.property = self.property or prop
+
+    def get_qualifier(self, prop: str) -> List[Snak]:
+        return self.qualifiers.get(prop, [])
+
+    @property
+    def is_ended(self) -> bool:
+        snak = self.qualifiers.get("P582")
+        if snak is not None and len(snak) > 0:
+            return True
+        return False
+
+    def __repr__(self) -> str:
+        return f"<Claim({self.qid}, {self.property}, {self.value_type})>"
+
+    def __hash__(self) -> int:
+        return hash((self.qid, self.property, self.id))
+
+
+class Item(object):
+    """A wikidata item (or entity)."""
+
+    def __init__(self, client: "WikidataClient", data: Dict[str, Any]) -> None:
+        self.client = client
+        self.id: str = data.pop("id")
+        self.modified: Optional[str] = data.pop("modified", None)
+
+        self.labels: Set[LangText] = LangText.from_dict(data.pop("labels", {}))
+        self.aliases: Set[LangText] = LangText.from_dict(data.pop("aliases", {}))
+
+        descriptions = LangText.from_dict(data.pop("descriptions", {}))
+        self.description = LangText.pick(descriptions)
+
+        self.claims: List[Claim] = []
+        claims: Dict[str, List[Dict[str, Any]]] = data.pop("claims", {})
+        for prop, values in claims.items():
+            for value in values:
+                self.claims.append(Claim(client, value, prop))
+
+        # TODO: get back to this later:
+        data.pop("sitelinks", None)
+
+    @property
+    def label(self) -> Optional[LangText]:
+        label = LangText.pick(self.labels)
+        if label is not None:
+            return label
+        return LangText.pick(self.aliases)
+
+    def is_instance(self, qid: str) -> bool:
+        for claim in self.claims:
+            if claim.property == "P31" and claim.qid == qid:
+                return True
+        return False
+
+    def _types(self, path: List[str]) -> Set[str]:
+        qid = path[-1]
+        types = set([qid])
+        if len(path) > 6:
+            return types
+        for type_ in self.client._type_props(qid):
+            if type_ not in path:
+                types.update(self._types(path + [type_]))
+        return types
+
+    @property
+    def types(self) -> Set[str]:
+        """Get all the `instance of` and `subclass of` types for an item."""
+        return self._types([self.id])
+
+    def __repr__(self) -> str:
+        return f"<Item({self.id})>"
+
+    def __hash__(self) -> int:
+        return hash(self.id)

nomenklatura/wikidata/props.py

@@ -0,0 +1,70 @@
+# Query: https://w.wiki/4Z73
+PROPS_FAMILY = {
+    "P7": "sibling",
+    "P9": "sibling",
+    "P22": "parent",
+    "P26": "spouse",
+    "P25": "parent",
+    "P40": "child",
+    "P43": "stepparent",
+    "P44": "stepparent",
+    "P451": "unmarried partner",
+    "P1038": "relative",
+    "P1290": "godparent",
+    "P3373": "sibling",
+    "P3448": "stepparent",
+    "P8810": "unspecified parent",
+}
+
+PROPS_ASSOCIATION = {
+    "P1327": "partner in business or sport",
+    "P3342": "significant person",
+}
+
+# https://www.wikidata.org/wiki/Wikidata:List_of_properties/human
+PROPS_DIRECT = {
+    "P1477": "alias",  # birth name
+    "P1813": "alias",  # short name
+    "P2561": "alias",  # name
+    "P1559": "alias",  # name in native language
+    "P2562": "alias",  # married name
+    "P511": "title",
+    "P735": "firstName",
+    "P734": "lastName",
+    "P1950": "lastName",
+    "P21": "gender",
+    "P39": "position",
+    "P140": "religion",
+    "P106": "topics",
+    "P569": "birthDate",
+    "P5056": "fatherName",
+    "P570": "deathDate",
+    "P19": "birthPlace",
+    "P856": "website",
+    "P512": "education",
+    "P69": "education",
+    "P27": "citizenship",
+    "P742": "weakAlias",
+    "P172": "ethnicity",
+    "P973": "sourceUrl",
+    "P1278": "leiCode",
+    "P17": "country",
+    "P571": "incorporationDate",
+    "P1454": "legalForm",
+}
+
+PROPS_QUALIFIED = (
+    "position",
+    "education",
+)
+
+PROPS_TOPICS = {
+    "Q82955": "role.pep",
+    "Q193391": "role.diplo",
+    # "Q392651": "role.spy",
+    "Q14886050": "crime.terror",
+    "Q16533": "role.judge",
+    "Q17276321": "role.pep",  # member of the state duma
+    "Q189290": "mil",  # military officer
+    "Q47064": "mil",  # military personnel
+}

nomenklatura/wikidata/qualified.py

@@ -0,0 +1,49 @@
+from typing import Set
+from followthemoney.helpers import dates_years
+
+from nomenklatura.wikidata.model import Claim
+from nomenklatura.wikidata.lang import LangText
+
+
+def post_summary(
+    position: LangText,
+    start_dates: Set[str],
+    end_dates: Set[str],
+    dates: Set[str],
+) -> LangText:
+    """Make a string summary for a Post object."""
+    start = min(dates_years(start_dates), default="")
+    end = min(dates_years(end_dates), default="")
+    date_range = None
+    if len(start) or len(end):
+        date_range = f"{start}-{end}"
+    dates_ = dates_years(dates)
+    if date_range is None and len(dates_):
+        date_range = ", ".join(sorted(dates_))
+
+    label = position.text
+    if date_range:
+        label = f"{label} ({date_range})"
+    original = position.text or position.original
+    return LangText(label, position.lang, original=original)
+
+
+def qualify_value(value: LangText, claim: Claim) -> LangText:
+    if value.text is None:
+        return value
+    starts: Set[str] = set()
+    for qual in claim.get_qualifier("P580"):
+        if qual.text.text is not None:
+            starts.add(qual.text.text)
+
+    ends: Set[str] = set()
+    for qual in claim.get_qualifier("P582"):
+        if qual.text.text is not None:
+            ends.add(qual.text.text)
+
+    dates: Set[str] = set()
+    for qual in claim.get_qualifier("P585"):
+        if qual.text.text is not None:
+            dates.add(qual.text.text)
+
+    return post_summary(value, starts, ends, dates)

nomenklatura/wikidata/query.py

@@ -0,0 +1,66 @@
+from typing import Dict, Any, List, Optional
+
+
+class SparqlValue(object):
+    WD_PREFIX = "http://www.wikidata.org/entity/"
+
+    __slots__ = ["type", "value", "lang"]
+
+    def __init__(self, data: Dict[str, Any]) -> None:
+        self.type: str = data["type"]
+        self.value: str = data["value"]
+        if self.type == "uri" and self.value.startswith(self.WD_PREFIX):
+            self.value = self.value[len(self.WD_PREFIX) :]
+        self.lang: Optional[str] = data.get("xml:lang")
+
+    def __str__(self) -> str:
+        return self.value
+
+    def __repr__(self) -> str:
+        return f"<SV({self.type!r}, {self.value!r})>"
+
+    def __hash__(self) -> int:
+        return hash(repr(self))
+
+
+class SparqlBinding(object):
+    def __init__(self, response: "SparqlResponse", data: Dict[str, Any]) -> None:
+        self.response = response
+        self.values: Dict[str, SparqlValue] = {}
+        for var, value in data.items():
+            self.values[var] = SparqlValue(value)
+
+    def wrapped(self, var: str) -> Optional[SparqlValue]:
+        if var not in self.response.vars:
+            raise KeyError("No such var: %s (in: %r)" % (var, self.response.vars))
+        value = self.values.get(var)
+        if value is None:
+            return None
+        return value
+
+    def plain(self, var: str) -> Optional[str]:
+        if var not in self.response.vars:
+            raise KeyError("No such var: %s (in: %r)" % (var, self.response.vars))
+        if var in self.values:
+            value = self.wrapped(var)
+            if value is not None:
+                return str(value)
+        return None
+
+    def __repr__(self) -> str:
+        return f"<SparqlBinding({self.values!r})>"
+
+
+class SparqlResponse(object):
+    def __init__(self, query: str, response: Dict[str, Any]) -> None:
+        self.query = query
+        self.vars: List[str] = response["head"]["vars"]
+        self.results: List[SparqlBinding] = []
+        for bind in response["results"]["bindings"]:
+            self.results.append(SparqlBinding(self, bind))
+
+    def __len__(self) -> int:
+        return len(self.results)
+
+    def __repr__(self) -> str:
+        return f"<SparqlResponse({self.vars!r}, {len(self)})>"

nomenklatura/wikidata/value.py

@@ -0,0 +1,87 @@
+import logging
+from prefixdate import Precision
+from typing import TYPE_CHECKING, Set, cast, Any, Dict, Optional
+from rigour.ids.wikidata import is_qid
+from rigour.text.cleaning import remove_emoji, remove_bracketed_text
+from rigour.names import is_name
+# from rigour.text.distance import is_levenshtein_plausible
+
+from nomenklatura.wikidata.lang import LangText
+
+if TYPE_CHECKING:
+    from nomenklatura.wikidata.client import WikidataClient
+
+
+log = logging.getLogger(__name__)
+PRECISION = {
+    11: Precision.DAY,
+    10: Precision.MONTH,
+    9: Precision.YEAR,
+}
+
+
+def snak_value_to_string(
+    client: "WikidataClient", value_type: Optional[str], value: Dict[str, Any]
+) -> LangText:
+    if value_type is None:
+        return LangText(None)
+    elif value_type == "time":
+        raw_time = cast(Optional[str], value.get("time"))
+        if raw_time is None:
+            return LangText(None)
+        time = raw_time.strip("+")
+        prec_id = cast(int, value.get("precision"))
+        prec = PRECISION.get(prec_id, Precision.DAY)
+        time = time[: prec.value]
+
+        # Remove Jan 01, because it seems to be in input failure pattern
+        # with Wikidata (probably from bots that don't get "precision").
+        if time.endswith("-01-01"):
+            time = time[:4]
+
+        # Date limit in FtM. These will be removed by the death filter:
+        time = max("1001", time)
+        return LangText(time, original=raw_time)
+    elif value_type == "wikibase-entityid":
+        qid = value.get("id")
+        return client.get_label(qid)
+    elif value_type == "monolingualtext":
+        text = value.get("text")
+        if isinstance(text, str):
+            return LangText(text, lang=value.get("language"))
+    elif value_type == "quantity":
+        # Resolve unit name and make into string:
+        raw_amount = cast(str, value.get("amount", ""))
+        amount = raw_amount.lstrip("+")
+        unit = value.get("unit", "")
+        unit = unit.split("/")[-1]
+        if is_qid(unit):
+            unit = client.get_label(unit)
+            amount = f"{amount} {unit}"
+        return LangText(amount, original=raw_amount)
+    elif isinstance(value, str):
+        return LangText(value)
+    else:
+        log.warning("Unhandled value [%s]: %s", value_type, value)
+    return LangText(None)
+
+
+def clean_name(name: str) -> Optional[str]:
+    """Clean a name for storage, try to throw out dangerous user inputs."""
+    if not is_name(name):
+        return None
+    clean_name = remove_bracketed_text(name)
+    if not is_name(clean_name):
+        clean_name = name
+    return remove_emoji(clean_name)
+
+
+def is_alias_strong(alias: str, names: Set[str]) -> bool:
+    """Check if an alias is a plausible nickname for a person, ie. shows some
+    similarity to the actual name."""
+    if " " not in alias:
+        return False
+    # for name in names:
+    #     if is_levenshtein_plausible(alias, name, max_edits=None, max_percent=0.7):
+    #         return True
+    return True