personal_knowledge_library 3.0.0__py3-none-any.whl → 3.1.1__py3-none-any.whl

knowledge/public/client.py

@@ -0,0 +1,428 @@
+# -*- coding: utf-8 -*-
+# Copyright © 2023-present Wacom. All rights reserved.
+import multiprocessing
+from collections import deque
+from multiprocessing import Pool
+from typing import Union, Any, Dict, List, Tuple, Set, Optional, Callable
+
+import requests
+from requests import Response
+from requests.adapters import HTTPAdapter
+from urllib3 import Retry
+
+from knowledge import logger
+from knowledge.base.entity import (
+    LanguageCode,
+)
+from knowledge.public.cache import WikidataCache
+from knowledge.public.helper import (
+    __waiting_request__,
+    __waiting_multi_request__,
+    WikiDataAPIException,
+    WIKIDATA_SPARQL_URL,
+    WIKIDATA_SEARCH_URL,
+    API_LIMIT,
+)
+from knowledge.public.wikidata import WikidataClass, WikidataThing, WikidataSearchResult, WikidataProperty
+
+# Constants
+QUALIFIERS_TAG: str = "QUALIFIERS"
+LITERALS_TAG: str = "LITERALS"
+# Cache for wikidata objects
+wikidata_cache: WikidataCache = WikidataCache()
+
+
+def chunks(lst: List[str], chunk_size: int):
+    """
+    Yield successive n-sized chunks from lst.Yield successive n-sized chunks from lst.
+    Parameters
+    ----------
+    lst: List[str]
+        Full length.
+    chunk_size: int
+        Chunk size.
+
+    """
+    for i in range(0, len(lst), chunk_size):
+        yield lst[i : i + chunk_size]
+
+
+class WikiDataAPIClient:
+    """
+    WikiDataAPIClient
+    -----------------
+    Utility class for the WikiData.
+
+    """
+
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def sparql_query(query_string: str, wikidata_sparql_url: str = WIKIDATA_SPARQL_URL, max_retries: int = 3) -> dict:
+        """Send a SPARQL query and return the JSON formatted result.
+
+        Parameters
+        -----------
+        query_string: str
+          SPARQL query string
+        wikidata_sparql_url: str
+          Wikidata SPARQL endpoint to use
+        max_retries: int
+            Maximum number of retries
+        """
+        # Define the retry policy
+        retry_policy: Retry = Retry(
+            total=max_retries,  # maximum number of retries
+            backoff_factor=1,  # factor by which to multiply the delay between retries
+            status_forcelist=[429, 500, 502, 503, 504],  # HTTP status codes to retry on
+            respect_retry_after_header=True,  # respect the Retry-After header
+        )
+
+        # Create a session and mount the retry adapter
+        with requests.Session() as session:
+            retry_adapter = HTTPAdapter(max_retries=retry_policy)
+            session.mount("https://", retry_adapter)
+
+            # Make a request using the session
+            response: Response = session.get(
+                wikidata_sparql_url, params={"query": query_string, "format": "json"}, timeout=10000
+            )
+            if response.ok:
+                return response.json()
+
+            raise WikiDataAPIException(
+                f"Failed to query entities. " f"Response code:={response.status_code}, Exception:= {response.content}."
+            )
+
+    @staticmethod
+    def superclasses(qid: str) -> Dict[str, WikidataClass]:
+        """
+        Returns the Wikidata class with all its superclasses for the given QID.
+
+        Parameters
+        ----------
+        qid: str
+            Wikidata QID (e.g., 'Q146' for house cat).
+
+        Returns
+        -------
+        classes: Dict[str, WikidataClass]
+            A dictionary of WikidataClass objects, where the keys are QIDs and the values are the corresponding
+        """
+        # Fetch superclasses
+        query = f"""
+        SELECT DISTINCT ?class ?classLabel ?superclass ?superclassLabel
+        WHERE
+        {{
+            wd:{qid} wdt:P279* ?class.
+            ?class wdt:P279 ?superclass.
+            SERVICE wikibase:label {{bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". }}
+        }}
+        """
+        try:
+            reply: Dict[str, Any] = WikiDataAPIClient.sparql_query(query)
+            wikidata_classes: Dict[str, WikidataClass] = {}
+            cycle_detector: Set[Tuple[str, str]] = set()
+            adjacency_list: Dict[str, Set[str]] = {}
+
+            if "results" in reply:
+                for b in reply["results"]["bindings"]:
+                    superclass_qid = b["superclass"]["value"].rsplit("/", 1)[-1]
+                    class_qid = b["class"]["value"].rsplit("/", 1)[-1]
+                    superclass_label = b["superclassLabel"]["value"]
+                    class_label = b["classLabel"]["value"]
+                    wikidata_classes.setdefault(class_qid, WikidataClass(class_qid, class_label))
+                    wikidata_classes.setdefault(superclass_qid, WikidataClass(superclass_qid, superclass_label))
+                    adjacency_list.setdefault(class_qid, set()).add(superclass_qid)
+        except Exception as e:
+            logger.exception(e)
+            return {qid: WikidataClass(qid, f"Class {qid}")}
+        queue = deque([qid])
+        visited = set()
+
+        while queue:
+            current_qid = queue.popleft()
+            if current_qid in visited:
+                continue
+            visited.add(current_qid)
+
+            if current_qid in adjacency_list:
+                for superclass_qid in adjacency_list[current_qid]:
+                    if (current_qid, superclass_qid) not in cycle_detector:
+                        wikidata_classes[current_qid].superclasses.append(wikidata_classes[superclass_qid])
+                        queue.append(superclass_qid)
+                        cycle_detector.add((current_qid, superclass_qid))
+
+        return wikidata_classes
+
+    @staticmethod
+    def subclasses(qid: str) -> Dict[str, WikidataClass]:
+        """
+        Returns the Wikidata class with all its subclasses for the given QID.
+
+        Parameters
+        ----------
+        qid: str
+            Wikidata QID (e.g., 'Q146' for house cat).
+
+        Returns
+        -------
+        classes: Dict[str, WikidataClass]
+            A dictionary of WikidataClass objects, where the keys are QIDs and the values are the corresponding
+            classes with their subclasses populated.
+        """
+        # Fetch subclasses
+        query = f"""
+            SELECT DISTINCT ?class ?classLabel ?subclass ?subclassLabel
+            WHERE
+            {{
+                ?subclass wdt:P279 wd:{qid}.
+                ?subclass wdt:P279 ?class.
+                SERVICE wikibase:label {{ bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". }}
+            }}
+            LIMIT 1000
+            """
+        try:
+            reply: Dict[str, Any] = WikiDataAPIClient.sparql_query(query)
+            wikidata_classes: Dict[str, WikidataClass] = {}
+            cycle_detector: Set[Tuple[str, str]] = set()
+            adjacency_list: Dict[str, Set[str]] = {}
+
+            if "results" in reply:
+                for b in reply["results"]["bindings"]:
+                    subclass_qid = b["subclass"]["value"].rsplit("/", 1)[-1]
+                    class_qid = b["class"]["value"].rsplit("/", 1)[-1]
+                    subclass_label = b["subclassLabel"]["value"]
+                    class_label = b["classLabel"]["value"]
+
+                    wikidata_classes.setdefault(class_qid, WikidataClass(class_qid, class_label))
+                    wikidata_classes.setdefault(subclass_qid, WikidataClass(subclass_qid, subclass_label))
+
+                    # subclass -> class relationship (reverse of superclass logic)
+                    adjacency_list.setdefault(class_qid, set()).add(subclass_qid)
+        except Exception as e:
+            logger.exception(e)
+            return {qid: WikidataClass(qid, f"Class {qid}")}
+
+        queue = deque([qid])
+        visited = set()
+
+        while queue:
+            current_qid = queue.popleft()
+            if current_qid in visited:
+                continue
+            visited.add(current_qid)
+
+            # Ensure the starting QID is in the dictionary
+            if current_qid not in wikidata_classes:
+                # If not present, we might need to fetch its label separately
+                wikidata_classes[current_qid] = WikidataClass(current_qid, f"Class {current_qid}")
+
+            if current_qid in adjacency_list:
+                for subclass_qid in adjacency_list[current_qid]:
+                    if (current_qid, subclass_qid) not in cycle_detector:
+                        wikidata_classes[current_qid].subclasses.append(wikidata_classes[subclass_qid])
+                        queue.append(subclass_qid)
+                        cycle_detector.add((current_qid, subclass_qid))
+
+        return wikidata_classes
+
+    @staticmethod
+    def search_term(
+        search_term: str, language: LanguageCode, url: str = WIKIDATA_SEARCH_URL
+    ) -> List[WikidataSearchResult]:
+        """
+        Search for a term in the WikiData.
+        Parameters
+        ----------
+        search_term: str
+            The term to search for.
+        language: str
+            The language to search in.
+        url: str
+            The URL of the WikiData search API.
+
+        Returns
+        -------
+        search_results_dict: List[WikidataSearchResult]
+            The search results.
+        """
+        search_results_dict: List[WikidataSearchResult] = []
+        # Define the retry policy
+        retry_policy: Retry = Retry(
+            total=3,  # maximum number of retries
+            backoff_factor=1,  # factor by which to multiply the delay between retries
+            status_forcelist=[429, 500, 502, 503, 504],  # HTTP status codes to retry on
+            respect_retry_after_header=True,  # respect the Retry-After header
+        )
+
+        # Create a session and mount the retry adapter
+        with requests.Session() as session:
+            retry_adapter = HTTPAdapter(max_retries=retry_policy)
+            session.mount("https://", retry_adapter)
+            params: Dict[str, str] = {
+                "action": "wbsearchentities",
+                "format": "json",
+                "language": language,
+                "search": search_term,
+            }
+            # Make a request using the session
+            response: Response = session.get(url, params=params, timeout=200000)
+
+            # Check the response status code
+            if not response.ok:
+                raise WikiDataAPIException(
+                    f"Search request failed with status code : {response.status_code}. " f"URL:= {url}"
+                )
+            search_result_dict_full: Dict[str, Any] = response.json()
+            for search_result_dict in search_result_dict_full["search"]:
+                search_results_dict.append(WikidataSearchResult.from_dict(search_result_dict))
+            return search_results_dict
+
+    @staticmethod
+    def __wikidata_task__(qid: str) -> WikidataThing:
+        """Retrieve a single Wikidata thing.
+
+        Parameters
+        ----------
+        qid: str
+            QID of the entity.
+
+        Returns
+        -------
+        instance: WikidataThing
+            Single wikidata thing
+        """
+        try:
+            if wikidata_cache.qid_in_cache(qid):
+                return wikidata_cache.get_wikidata_object(qid)
+            w_thing = WikidataThing.from_wikidata(__waiting_request__(qid))
+            # Add the thing to the cache
+            wikidata_cache.cache_wikidata_object(w_thing)
+            return w_thing
+        except Exception as e:
+            logger.exception(e)
+            raise WikiDataAPIException(e) from e
+
+    @staticmethod
+    def __wikidata_multiple_task__(qids: List[str]) -> List[WikidataThing]:
+        """Retrieve multiple Wikidata things.
+
+        Parameters
+        ----------
+        qids: List[str]
+            QIDs of the entities.
+
+        Returns
+        -------
+        instances: List[WikidataThing]
+            List of wikidata things
+        """
+        try:
+            results: List[WikidataThing] = []
+            if len(qids) > 0:
+                for e in __waiting_multi_request__(qids):
+                    w_thing = WikidataThing.from_wikidata(e)
+                    results.append(w_thing)
+            return results
+        except Exception as e:
+            logger.exception(e)
+            raise WikiDataAPIException(e) from e
+
+    @staticmethod
+    def retrieve_entity(qid: str) -> WikidataThing:
+        """
+        Retrieve a single Wikidata thing.
+
+        Parameters
+        ----------
+        qid: str
+            QID of the entity.
+
+        Returns
+        -------
+        instance: WikidataThing
+            Single wikidata thing
+        """
+        return WikiDataAPIClient.__wikidata_task__(qid)
+
+    @staticmethod
+    def retrieve_entities(qids: Union[List[str], Set[str]], progress: Optional[Callable[[int, int], None]] = None) \
+            -> List[WikidataThing]:
+        """
+        Retrieve multiple Wikidata things.
+        Parameters
+        ----------
+        qids: List[str]
+            QIDs of the entities.
+        progress: Optional[Callable[[int, int], None]]
+            Optional callback function to report progress.
+
+        Returns
+        -------
+        instances: List[WikidataThing]
+            List of wikidata things.
+        """
+        pulled: List[WikidataThing] = []
+        task_size: int = len(qids)
+        if len(qids) == 0:
+            return []
+        missing_qids: List[str] = []
+        for qid in qids:
+            if not wikidata_cache.qid_in_cache(qid):
+                if qid and qid.startswith("Q") and len(qid) > 1:
+                    missing_qids.append(qid)
+            else:
+                pulled.append(wikidata_cache.get_wikidata_object(qid))
+        ctr: int = len(pulled)
+        if progress:
+            progress(len(pulled), task_size)
+        jobs: List[List[str]] = list(chunks(list(missing_qids), API_LIMIT))
+        num_processes: int = min(len(jobs), multiprocessing.cpu_count())
+        if num_processes > 1:
+            with Pool(processes=num_processes) as pool:
+                # Wikidata thing is not support in multiprocessing
+                for lst in pool.imap_unordered(__waiting_multi_request__, jobs):
+                    for w_dict in lst:
+                        w_thing = WikidataThing.from_wikidata(w_dict)
+                        wikidata_cache.cache_wikidata_object(w_thing)
+                        pulled.append(w_thing)
+                        ctr += 1
+                        if progress:
+                            progress(ctr, task_size)
+        else:
+            results = WikiDataAPIClient.__wikidata_multiple_task__(jobs[0])
+            for w_thing in results:
+                wikidata_cache.cache_wikidata_object(w_thing)
+                ctr += 1
+                if progress:
+                    progress(ctr, task_size)
+            pulled.extend(results)
+        return pulled
+
+    @staticmethod
+    def wikiproperty(pid: str) -> WikidataProperty:
+        """
+        Retrieve a single Wikidata property.
+
+        Parameters
+        ----------
+        pid: str
+            PID of the property.
+
+        Returns
+        -------
+        instance: WikidataProperty
+            Single wikidata property
+        """
+        try:
+            # if wikidata_cache.get_property(pid):
+            #    return wikidata_cache.get_property(pid)
+            w_property = WikidataProperty.from_wikidata(__waiting_request__(pid))
+            # Add the property to the cache
+            wikidata_cache.cache_property(w_property)
+            return w_property
+        except Exception as e:
+            logger.exception(e)
+            raise WikiDataAPIException(e) from e

knowledge/public/helper.py

@@ -13,6 +13,7 @@ from requests.adapters import HTTPAdapter
 from urllib3 import Retry
 
 from knowledge import logger
+from knowledge.public import DEFAULT_MAX_RETRIES, DEFAULT_TIMEOUT, DEFAULT_BACKOFF_FACTOR, STATUS_FORCE_LIST
 
 
 # --------------------------------------- Structures -------------------------------------------------------------------
@@ -49,10 +50,12 @@ QID_TAG: str = "qid"
 PID_TAG: str = "pid"
 LAST_REVID_TAG: str = "lastrevid"
 MODIFIED_TAG: str = "modified"
+SYNC_TIME_TAG: str = "sync"
 WIKIDATA_LANGUAGE_TAG: str = "language"
 LABEL_VALUE_TAG: str = "value"
 LABEL_TAG: str = "label"
 SUPERCLASSES_TAG: str = "superclasses"
+SUBCLASSES_TAG: str = "subclasses"
 CLAIMS_TAG: str = "claims"
 ONTOLOGY_TYPES_TAG: str = "ontology_types"
 REVISION_TAG: str = "revision"
@@ -262,7 +265,13 @@ def wikidate(param: Dict[str, Any]) -> Dict[str, Any]:
     }
 
 
-def __waiting_request__(entity_id: str, base_url: str = WIKIDATA_LDI_URL) -> Dict[str, Any]:
+def __waiting_request__(
+    entity_id: str,
+    base_url: str = WIKIDATA_LDI_URL,
+    timeout: int = DEFAULT_TIMEOUT,
+    max_retries: int = DEFAULT_MAX_RETRIES,
+    backoff_factor: float = DEFAULT_BACKOFF_FACTOR,
+) -> Dict[str, Any]:
     """
     Sena a request with retry policy.
 
@@ -272,18 +281,24 @@ def __waiting_request__(entity_id: str, base_url: str = WIKIDATA_LDI_URL) -> Dic
         Entity QID
     base_url: Base URL
         The base URL
+    timeout:  int
+        Timeout in seconds
+    max_retries: int
+        Maximum number of retries
+    backoff_factor: float
+        Backoff factor for retries.
+
     Returns
     -------
     result_dict: Dict[str, Any]
         Result dict
     """
     url: str = f"{base_url}/{entity_id}.json"
-
     # Define the retry policy
     retry_policy: Retry = Retry(
-        total=3,  # maximum number of retries
-        backoff_factor=1,  # factor by which to multiply the delay between retries
-        status_forcelist=[429, 500, 502, 503, 504],  # HTTP status codes to retry on
+        total=max_retries,  # maximum number of retries
+        backoff_factor=backoff_factor,  # factor by which to multiply the delay between retries
+        status_forcelist=STATUS_FORCE_LIST,  # HTTP status codes to retry on
         respect_retry_after_header=True,  # respect the Retry-After header
     )
 
@@ -293,7 +308,7 @@ def __waiting_request__(entity_id: str, base_url: str = WIKIDATA_LDI_URL) -> Dic
         session.mount("https://", retry_adapter)
 
         # Make a request using the session
-        response: Response = session.get(url)
+        response: Response = session.get(url, timeout=timeout)
 
         # Check the response status code
         if not response.ok:
@@ -311,7 +326,13 @@ def __waiting_request__(entity_id: str, base_url: str = WIKIDATA_LDI_URL) -> Dic
         return entity_dict
 
 
-def __waiting_multi_request__(entity_ids: List[str], base_url: str = MULTIPLE_ENTITIES_API) -> List[Dict[str, Any]]:
+def __waiting_multi_request__(
+    entity_ids: List[str],
+    base_url: str = MULTIPLE_ENTITIES_API,
+    timeout: int = DEFAULT_TIMEOUT,
+    max_retries: int = DEFAULT_MAX_RETRIES,
+    backoff_factor: float = DEFAULT_BACKOFF_FACTOR,
+) -> List[Dict[str, Any]]:
     """
     Sena a request to retrieve multiple entities with retry policy.
 
@@ -321,6 +342,12 @@ def __waiting_multi_request__(entity_ids: List[str], base_url: str = MULTIPLE_EN
         Entity QIDs
     base_url: Base URL
         The base URL
+    timeout:  int
+        Timeout in seconds
+    max_retries: int
+        Maximum number of retries
+    backoff_factor: float
+        Backoff factor for retries.
     Returns
     -------
     result_dict: Dict[str, Any]
@@ -330,6 +357,7 @@ def __waiting_multi_request__(entity_ids: List[str], base_url: str = MULTIPLE_EN
     ValueError - Empty list or to many entities
     """
     checked_entity_ids: List[str] = [e for e in entity_ids if e.startswith("Q")]
+
     if not (0 < len(checked_entity_ids) <= API_LIMIT):
         raise ValueError(
             f"Number of entities must be within [1, {API_LIMIT}]. " f"Number of QIDs: {len(checked_entity_ids)}"
@@ -339,9 +367,9 @@ def __waiting_multi_request__(entity_ids: List[str], base_url: str = MULTIPLE_EN
 
     # Define the retry policy
     retry_policy: Retry = Retry(
-        total=3,  # maximum number of retries
-        backoff_factor=1,  # factor by which to multiply the delay between retries
-        status_forcelist=[429, 500, 502, 503, 504],  # HTTP status codes to retry on
+        total=max_retries,  # maximum number of retries
+        backoff_factor=backoff_factor,  # factor by which to multiply the delay between retries
+        status_forcelist=STATUS_FORCE_LIST,  # HTTP status codes to retry on
         respect_retry_after_header=True,  # respect the Retry-After header
     )
 
@@ -351,7 +379,7 @@ def __waiting_multi_request__(entity_ids: List[str], base_url: str = MULTIPLE_EN
         session.mount("https://", retry_adapter)
 
         # Make a request using the session
-        response: Response = session.get(url)
+        response: Response = session.get(url, timeout=timeout)
 
         # Check the response status code
         if not response.ok:

knowledge/public/relations.py

@@ -7,7 +7,8 @@ from typing import Any, Dict, Set, Tuple, List, Callable, Optional
 from tqdm import tqdm
 
 from knowledge.public.helper import CLAIMS_TAG, PID_TAG, LABEL_TAG, QID_TAG
-from knowledge.public.wikidata import LITERALS_TAG, WikidataThing, WikiDataAPIClient
+from knowledge.public.wikidata import LITERALS_TAG, WikidataThing
+from knowledge.public.client import WikiDataAPIClient
 
 
 def __relations__(thing: Dict[str, Any], wikidata: Set[str]) -> Tuple[str, List[Dict[str, Any]]]: