personal_knowledge_library 3.1.0__tar.gz → 3.1.2__tar.gz

{personal_knowledge_library-3.1.0 → personal_knowledge_library-3.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: personal_knowledge_library
-Version: 3.1.0
+Version: 3.1.2
 Summary: Library to access Wacom's Personal Knowledge graph.
 License: Apache-2.0
 Keywords: semantic-knowledge,knowledge-graph

{personal_knowledge_library-3.1.0 → personal_knowledge_library-3.1.2}/knowledge/__init__.py

@@ -17,7 +17,7 @@ __license__ = "Wacom"
 __maintainer__ = ["Markus Weber"]
 __email__ = "markus.weber@wacom.com"
 __status__ = "beta"
-__version__ = "3.1.0"
+__version__ = "3.1.2"
 
 import loguru
 

{personal_knowledge_library-3.1.0 → personal_knowledge_library-3.1.2}/knowledge/base/ontology.py

@@ -2179,7 +2179,9 @@ class ThingObject(abc.ABC):
         labels.extend([la.__dict__() for la in self.label])
         labels.extend([la.__dict__() for la in self.alias])
         dict_object: Dict[str, Any] = {
-            SOURCE_REFERENCE_ID_TAG: self.source_reference_id if self.source_reference_id is None else reference_id,
+            SOURCE_REFERENCE_ID_TAG: (
+                self.default_source_reference_id() if self.source_reference_id is not None else reference_id
+            ),
             SOURCE_SYSTEM_TAG: self.source_system,
             IMAGE_TAG: self.image,
             LABELS_TAG: labels,

{personal_knowledge_library-3.1.0 → personal_knowledge_library-3.1.2}/knowledge/ontomapping/__init__.py

@@ -543,6 +543,7 @@ def build_configuration(mapping: Dict[str, Any]) -> MappingConfiguration:
         conf.add_property(property_config)
     return conf
 
+
 def register_ontology(rdf_str: str):
     """
     Registers the ontology.
@@ -553,6 +554,7 @@ def register_ontology(rdf_str: str):
     """
     ontology_graph.parse(data=rdf_str, format="xml")
 
+
 def load_configuration(configuration: Path):
     """
     Loads the configuration.

{personal_knowledge_library-3.1.0 → personal_knowledge_library-3.1.2}/knowledge/public/cache.py

@@ -53,7 +53,7 @@ class WikidataCache:
 
     _instance = None  # Singleton instance
 
-    def __init__(self, max_size: int = 10000):
+    def __init__(self, max_size: int = 100000):
         self.max_size = max_size
         self.cache: OrderedDict = OrderedDict()  # Maintain insertion order for LRU eviction
         self.property_cache: OrderedDict = OrderedDict()  # Cache for properties

{personal_knowledge_library-3.1.0 → personal_knowledge_library-3.1.2}/knowledge/public/client.py

@@ -3,7 +3,7 @@
 import multiprocessing
 from collections import deque
 from multiprocessing import Pool
-from typing import Union, Any, Dict, List, Tuple, Set
+from typing import Union, Any, Dict, List, Tuple, Set, Optional, Callable
 
 import requests
 from requests import Response
@@ -321,18 +321,10 @@ class WikiDataAPIClient:
         """
         try:
             results: List[WikidataThing] = []
-            request_qids: List[str] = []
-            for qid in qids:
-                if wikidata_cache.qid_in_cache(qid):
-                    results.append(wikidata_cache.get_wikidata_object(qid))
-                else:
-                    request_qids.append(qid)
-            if len(request_qids) > 0:
-                for e in __waiting_multi_request__(request_qids):
+            if len(qids) > 0:
+                for e in __waiting_multi_request__(qids):
                     w_thing = WikidataThing.from_wikidata(e)
                     results.append(w_thing)
-                    # Add the thing to the cache
-                    wikidata_cache.cache_wikidata_object(w_thing)
             return results
         except Exception as e:
             logger.exception(e)
@@ -356,13 +348,17 @@ class WikiDataAPIClient:
         return WikiDataAPIClient.__wikidata_task__(qid)
 
     @staticmethod
-    def retrieve_entities(qids: Union[List[str], Set[str]]) -> List[WikidataThing]:
+    def retrieve_entities(
+        qids: Union[List[str], Set[str]], progress: Optional[Callable[[int, int], None]] = None
+    ) -> List[WikidataThing]:
         """
         Retrieve multiple Wikidata things.
         Parameters
         ----------
         qids: List[str]
             QIDs of the entities.
+        progress: Optional[Callable[[int, int], None]]
+            Optional callback function to report progress.
 
         Returns
         -------
@@ -370,17 +366,40 @@ class WikiDataAPIClient:
             List of wikidata things.
         """
         pulled: List[WikidataThing] = []
+        task_size: int = len(qids)
         if len(qids) == 0:
             return []
-        jobs: List[List[str]] = list(chunks(list(qids), API_LIMIT))
+        missing_qids: List[str] = []
+        for qid in qids:
+            if not wikidata_cache.qid_in_cache(qid):
+                if qid and qid.startswith("Q") and len(qid) > 1:
+                    missing_qids.append(qid)
+            else:
+                pulled.append(wikidata_cache.get_wikidata_object(qid))
+        ctr: int = len(pulled)
+        if progress:
+            progress(len(pulled), task_size)
+        jobs: List[List[str]] = list(chunks(list(missing_qids), API_LIMIT))
         num_processes: int = min(len(jobs), multiprocessing.cpu_count())
         if num_processes > 1:
             with Pool(processes=num_processes) as pool:
                 # Wikidata thing is not support in multiprocessing
                 for lst in pool.imap_unordered(__waiting_multi_request__, jobs):
-                    pulled.extend([WikidataThing.from_wikidata(e) for e in lst])
+                    for w_dict in lst:
+                        w_thing = WikidataThing.from_wikidata(w_dict)
+                        wikidata_cache.cache_wikidata_object(w_thing)
+                        pulled.append(w_thing)
+                        ctr += 1
+                        if progress:
+                            progress(ctr, task_size)
         else:
-            pulled = WikiDataAPIClient.__wikidata_multiple_task__(jobs[0])
+            results = WikiDataAPIClient.__wikidata_multiple_task__(jobs[0])
+            for w_thing in results:
+                wikidata_cache.cache_wikidata_object(w_thing)
+                ctr += 1
+                if progress:
+                    progress(ctr, task_size)
+            pulled.extend(results)
         return pulled
 
     @staticmethod

{personal_knowledge_library-3.1.0 → personal_knowledge_library-3.1.2}/knowledge/public/wikidata.py

@@ -877,7 +877,7 @@ class WikidataThing:
                             val = {"tabular": data_value["value"]}
                         elif data_type == "entity-schema":
                             val = {"id": data_value["value"]["id"]}
-                        elif data_type == "wikibase-form":
+                        elif data_type in ["wikibase-form", "musical-notation"]:
                             continue
                         else:
                             raise WikiDataAPIException(f"Data type: {data_type} not supported.")

{personal_knowledge_library-3.1.0 → personal_knowledge_library-3.1.2}/knowledge/utils/__init__.py

@@ -2,9 +2,9 @@
 # Copyright © 2024-present Wacom. All rights reserved.
 """ "Utilities"""
 
-__all__ = ["wikipedia", "graph", "wikidata"]
+__all__ = ["import_format", "graph", "wikidata", "wikipedia"]
 
-
-from knowledge.utils import wikipedia
+from knowledge.utils import import_format
 from knowledge.utils import graph
 from knowledge.utils import wikidata
+from knowledge.utils import wikipedia

personal_knowledge_library-3.1.2/knowledge/utils/import_format.py

@@ -0,0 +1,163 @@
+# -*- coding: utf-8 -*-
+# Copyright © 2024-present Wacom. All rights reserved.
+import gzip
+import json
+import logging
+import re
+from json import JSONDecodeError
+from pathlib import Path
+from typing import List, Dict, Any
+
+import loguru
+
+from knowledge.base.ontology import ThingObject, OntologyPropertyReference
+
+logger = loguru.logger
+
+
+def is_http_url(url: str) -> bool:
+    """Check if a string is an HTTP(S) URL.
+    Parameters
+    ----------
+    url: str
+        The URL to check.
+
+    Returns
+    -------
+    bool
+        True if the URL is HTTP(S), False otherwise.
+    """
+    return bool(re.match(r"^(https?://)", url, re.IGNORECASE))
+
+
+def is_local_url(url: str) -> bool:
+    """Check if a string is a local file path or relative URL.
+    Parameters
+    ----------
+    url: str
+        The URL to check.
+
+    Returns
+    -------
+    bool
+        True if the URL is a local file path or relative URL, False otherwise.
+    """
+    return bool(re.match(r"^(file://|/|\.{1,2}/)", url, re.IGNORECASE))
+
+
+def __import_format_to_thing__(line: str) -> ThingObject:
+    """
+    Convert a line of JSON to a ThingObject.
+    Parameters
+    ----------
+    line: str
+        The line of JSON to convert.
+
+    Returns
+    -------
+    entity: ThingObject
+        The ThingObject created from the JSON line.
+
+    Raises
+    ------
+    JSONDecodeError
+        If the line is not valid JSON.
+    """
+    thing_dict: Dict[str, Any] = json.loads(line)
+    entity: ThingObject = ThingObject.from_import_dict(thing_dict)
+    if entity.image:
+        if not is_local_url(entity.image) and not is_http_url(entity.image):
+            path: Path = Path(entity.image)
+            if not path.exists():
+                entity.image = path.absolute().as_uri()
+            else:
+                logger.warning(f"Image path {path} does not exist. Setting to None.")
+                entity.image = None
+    remove_props: List[OntologyPropertyReference] = []
+    # Remove empty properties
+    for obj_prop, value in entity.object_properties.items():
+        if len(value.incoming_relations) == 0 and len(value.outgoing_relations) == 0:
+            remove_props.append(obj_prop)
+    for prop in remove_props:
+        del entity.object_properties[prop]
+    return entity
+
+
+def load_import_format(file_path: Path) -> List[ThingObject]:
+    """
+    Load the import format file.
+    Parameters
+    ----------
+    file_path:  Path
+        The path to the file.
+
+    Returns
+    -------
+    entity_list: List[ThingObject]
+        The list of entities.
+
+    Raises
+    ------
+    FileNotFoundError
+        If the file does not exist or is not a file.
+    """
+    if not file_path.exists():
+        logger.error(f"File {file_path} does not exist.")
+        raise FileNotFoundError(f"File {file_path} does not exist.")
+    if not file_path.is_file():
+        logger.error(f"Path {file_path} is not a file.")
+        raise FileNotFoundError(f"Path {file_path} is not a file.")
+    cached_entities: List[ThingObject] = []
+    if file_path.suffix == ".gz":
+        with gzip.open(file_path, "rb") as f_gz:
+            for line in f_gz.readlines():
+                line = line.decode("utf-8")
+                try:
+                    cached_entities.append(__import_format_to_thing__(line))
+                except JSONDecodeError as e:
+                    logging.error(f"Error decoding JSON: {e}. Line: {line}")
+    else:
+        with file_path.open(encoding="utf8") as f:
+            # Skip the first line
+            for line in f.readlines():
+                try:
+                    cached_entities.append(__import_format_to_thing__(line))
+                except JSONDecodeError as e:
+                    logging.error(f"Error decoding JSON: {e}. Line: {line}")
+    return cached_entities
+
+
+def save_import_format(file_path: Path, entities: List[ThingObject]) -> None:
+    """
+    Save the import format file.
+    Parameters
+    ----------
+    file_path: Path
+        The path to the file.
+    entities: List[ThingObject]
+        The list of entities.
+    """
+    # Create the directory if it does not exist
+    file_path.parent.mkdir(parents=True, exist_ok=True)
+    if file_path.suffix == ".gz":
+        with gzip.open(file_path, "wt", encoding="utf-8") as fp_thing:
+            for entity in entities:
+                fp_thing.write(f"{json.dumps(entity.__import_format_dict__(), ensure_ascii=False)}\n")
+    elif file_path.suffix == ".ndjson":
+        with file_path.open("w", encoding="utf-8") as fp_thing:
+            for entity in entities:
+                fp_thing.write(f"{json.dumps(entity.__import_format_dict__(), ensure_ascii=False)}\n")
+
+
+def append_import_format(file_path: Path, entity: ThingObject) -> None:
+    """
+    Append to the import format file.
+    Parameters
+    ----------
+    file_path: Path
+        The path to the file.
+    entity: ThingObject
+        The entity to append.
+    """
+    with file_path.open("a", encoding="utf-8") as fp_thing:
+        fp_thing.write(f"{json.dumps(entity.__import_format_dict__(), ensure_ascii=False)}\n")

{personal_knowledge_library-3.1.0 → personal_knowledge_library-3.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "personal_knowledge_library"
-version = "3.1.0"
+version = "3.1.2"
 description = "Library to access Wacom's Personal Knowledge graph."
 authors = [{ name = "Markus Weber", email = "markus.weber@wacom.com" }]
 license = "Apache-2.0"