personal_knowledge_library 3.1.0__py3-none-any.whl → 3.1.2__py3-none-any.whl

knowledge/__init__.py

@@ -17,7 +17,7 @@ __license__ = "Wacom"
 __maintainer__ = ["Markus Weber"]
 __email__ = "markus.weber@wacom.com"
 __status__ = "beta"
-__version__ = "3.1.0"
+__version__ = "3.1.2"
 
 import loguru
 

knowledge/base/ontology.py

@@ -2179,7 +2179,9 @@ class ThingObject(abc.ABC):
         labels.extend([la.__dict__() for la in self.label])
         labels.extend([la.__dict__() for la in self.alias])
         dict_object: Dict[str, Any] = {
-            SOURCE_REFERENCE_ID_TAG: self.source_reference_id if self.source_reference_id is None else reference_id,
+            SOURCE_REFERENCE_ID_TAG: (
+                self.default_source_reference_id() if self.source_reference_id is not None else reference_id
+            ),
             SOURCE_SYSTEM_TAG: self.source_system,
             IMAGE_TAG: self.image,
             LABELS_TAG: labels,

knowledge/ontomapping/__init__.py

@@ -543,6 +543,7 @@ def build_configuration(mapping: Dict[str, Any]) -> MappingConfiguration:
         conf.add_property(property_config)
     return conf
 
+
 def register_ontology(rdf_str: str):
     """
     Registers the ontology.
@@ -553,6 +554,7 @@ def register_ontology(rdf_str: str):
     """
     ontology_graph.parse(data=rdf_str, format="xml")
 
+
 def load_configuration(configuration: Path):
     """
     Loads the configuration.

knowledge/public/cache.py

@@ -53,7 +53,7 @@ class WikidataCache:
 
     _instance = None  # Singleton instance
 
-    def __init__(self, max_size: int = 10000):
+    def __init__(self, max_size: int = 100000):
         self.max_size = max_size
         self.cache: OrderedDict = OrderedDict()  # Maintain insertion order for LRU eviction
         self.property_cache: OrderedDict = OrderedDict()  # Cache for properties

knowledge/public/client.py

@@ -3,7 +3,7 @@
 import multiprocessing
 from collections import deque
 from multiprocessing import Pool
-from typing import Union, Any, Dict, List, Tuple, Set
+from typing import Union, Any, Dict, List, Tuple, Set, Optional, Callable
 
 import requests
 from requests import Response
@@ -321,18 +321,10 @@ class WikiDataAPIClient:
         """
         try:
             results: List[WikidataThing] = []
-            request_qids: List[str] = []
-            for qid in qids:
-                if wikidata_cache.qid_in_cache(qid):
-                    results.append(wikidata_cache.get_wikidata_object(qid))
-                else:
-                    request_qids.append(qid)
-            if len(request_qids) > 0:
-                for e in __waiting_multi_request__(request_qids):
+            if len(qids) > 0:
+                for e in __waiting_multi_request__(qids):
                     w_thing = WikidataThing.from_wikidata(e)
                     results.append(w_thing)
-                    # Add the thing to the cache
-                    wikidata_cache.cache_wikidata_object(w_thing)
             return results
         except Exception as e:
             logger.exception(e)
@@ -356,13 +348,17 @@ class WikiDataAPIClient:
         return WikiDataAPIClient.__wikidata_task__(qid)
 
     @staticmethod
-    def retrieve_entities(qids: Union[List[str], Set[str]]) -> List[WikidataThing]:
+    def retrieve_entities(
+        qids: Union[List[str], Set[str]], progress: Optional[Callable[[int, int], None]] = None
+    ) -> List[WikidataThing]:
         """
         Retrieve multiple Wikidata things.
         Parameters
         ----------
         qids: List[str]
             QIDs of the entities.
+        progress: Optional[Callable[[int, int], None]]
+            Optional callback function to report progress.
 
         Returns
         -------
@@ -370,17 +366,40 @@ class WikiDataAPIClient:
             List of wikidata things.
         """
         pulled: List[WikidataThing] = []
+        task_size: int = len(qids)
         if len(qids) == 0:
             return []
-        jobs: List[List[str]] = list(chunks(list(qids), API_LIMIT))
+        missing_qids: List[str] = []
+        for qid in qids:
+            if not wikidata_cache.qid_in_cache(qid):
+                if qid and qid.startswith("Q") and len(qid) > 1:
+                    missing_qids.append(qid)
+            else:
+                pulled.append(wikidata_cache.get_wikidata_object(qid))
+        ctr: int = len(pulled)
+        if progress:
+            progress(len(pulled), task_size)
+        jobs: List[List[str]] = list(chunks(list(missing_qids), API_LIMIT))
         num_processes: int = min(len(jobs), multiprocessing.cpu_count())
         if num_processes > 1:
             with Pool(processes=num_processes) as pool:
                 # Wikidata thing is not support in multiprocessing
                 for lst in pool.imap_unordered(__waiting_multi_request__, jobs):
-                    pulled.extend([WikidataThing.from_wikidata(e) for e in lst])
+                    for w_dict in lst:
+                        w_thing = WikidataThing.from_wikidata(w_dict)
+                        wikidata_cache.cache_wikidata_object(w_thing)
+                        pulled.append(w_thing)
+                        ctr += 1
+                        if progress:
+                            progress(ctr, task_size)
         else:
-            pulled = WikiDataAPIClient.__wikidata_multiple_task__(jobs[0])
+            results = WikiDataAPIClient.__wikidata_multiple_task__(jobs[0])
+            for w_thing in results:
+                wikidata_cache.cache_wikidata_object(w_thing)
+                ctr += 1
+                if progress:
+                    progress(ctr, task_size)
+            pulled.extend(results)
         return pulled
 
     @staticmethod

knowledge/public/wikidata.py

@@ -877,7 +877,7 @@ class WikidataThing:
                             val = {"tabular": data_value["value"]}
                         elif data_type == "entity-schema":
                             val = {"id": data_value["value"]["id"]}
-                        elif data_type == "wikibase-form":
+                        elif data_type in ["wikibase-form", "musical-notation"]:
                             continue
                         else:
                             raise WikiDataAPIException(f"Data type: {data_type} not supported.")

knowledge/utils/__init__.py

@@ -2,9 +2,9 @@
 # Copyright © 2024-present Wacom. All rights reserved.
 """ "Utilities"""
 
-__all__ = ["wikipedia", "graph", "wikidata"]
+__all__ = ["import_format", "graph", "wikidata", "wikipedia"]
 
-
-from knowledge.utils import wikipedia
+from knowledge.utils import import_format
 from knowledge.utils import graph
 from knowledge.utils import wikidata
+from knowledge.utils import wikipedia

knowledge/utils/import_format.py

@@ -0,0 +1,163 @@
+# -*- coding: utf-8 -*-
+# Copyright © 2024-present Wacom. All rights reserved.
+import gzip
+import json
+import logging
+import re
+from json import JSONDecodeError
+from pathlib import Path
+from typing import List, Dict, Any
+
+import loguru
+
+from knowledge.base.ontology import ThingObject, OntologyPropertyReference
+
+logger = loguru.logger
+
+
+def is_http_url(url: str) -> bool:
+    """Check if a string is an HTTP(S) URL.
+    Parameters
+    ----------
+    url: str
+        The URL to check.
+
+    Returns
+    -------
+    bool
+        True if the URL is HTTP(S), False otherwise.
+    """
+    return bool(re.match(r"^(https?://)", url, re.IGNORECASE))
+
+
+def is_local_url(url: str) -> bool:
+    """Check if a string is a local file path or relative URL.
+    Parameters
+    ----------
+    url: str
+        The URL to check.
+
+    Returns
+    -------
+    bool
+        True if the URL is a local file path or relative URL, False otherwise.
+    """
+    return bool(re.match(r"^(file://|/|\.{1,2}/)", url, re.IGNORECASE))
+
+
+def __import_format_to_thing__(line: str) -> ThingObject:
+    """
+    Convert a line of JSON to a ThingObject.
+    Parameters
+    ----------
+    line: str
+        The line of JSON to convert.
+
+    Returns
+    -------
+    entity: ThingObject
+        The ThingObject created from the JSON line.
+
+    Raises
+    ------
+    JSONDecodeError
+        If the line is not valid JSON.
+    """
+    thing_dict: Dict[str, Any] = json.loads(line)
+    entity: ThingObject = ThingObject.from_import_dict(thing_dict)
+    if entity.image:
+        if not is_local_url(entity.image) and not is_http_url(entity.image):
+            path: Path = Path(entity.image)
+            if not path.exists():
+                entity.image = path.absolute().as_uri()
+            else:
+                logger.warning(f"Image path {path} does not exist. Setting to None.")
+                entity.image = None
+    remove_props: List[OntologyPropertyReference] = []
+    # Remove empty properties
+    for obj_prop, value in entity.object_properties.items():
+        if len(value.incoming_relations) == 0 and len(value.outgoing_relations) == 0:
+            remove_props.append(obj_prop)
+    for prop in remove_props:
+        del entity.object_properties[prop]
+    return entity
+
+
+def load_import_format(file_path: Path) -> List[ThingObject]:
+    """
+    Load the import format file.
+    Parameters
+    ----------
+    file_path:  Path
+        The path to the file.
+
+    Returns
+    -------
+    entity_list: List[ThingObject]
+        The list of entities.
+
+    Raises
+    ------
+    FileNotFoundError
+        If the file does not exist or is not a file.
+    """
+    if not file_path.exists():
+        logger.error(f"File {file_path} does not exist.")
+        raise FileNotFoundError(f"File {file_path} does not exist.")
+    if not file_path.is_file():
+        logger.error(f"Path {file_path} is not a file.")
+        raise FileNotFoundError(f"Path {file_path} is not a file.")
+    cached_entities: List[ThingObject] = []
+    if file_path.suffix == ".gz":
+        with gzip.open(file_path, "rb") as f_gz:
+            for line in f_gz.readlines():
+                line = line.decode("utf-8")
+                try:
+                    cached_entities.append(__import_format_to_thing__(line))
+                except JSONDecodeError as e:
+                    logging.error(f"Error decoding JSON: {e}. Line: {line}")
+    else:
+        with file_path.open(encoding="utf8") as f:
+            # Skip the first line
+            for line in f.readlines():
+                try:
+                    cached_entities.append(__import_format_to_thing__(line))
+                except JSONDecodeError as e:
+                    logging.error(f"Error decoding JSON: {e}. Line: {line}")
+    return cached_entities
+
+
+def save_import_format(file_path: Path, entities: List[ThingObject]) -> None:
+    """
+    Save the import format file.
+    Parameters
+    ----------
+    file_path: Path
+        The path to the file.
+    entities: List[ThingObject]
+        The list of entities.
+    """
+    # Create the directory if it does not exist
+    file_path.parent.mkdir(parents=True, exist_ok=True)
+    if file_path.suffix == ".gz":
+        with gzip.open(file_path, "wt", encoding="utf-8") as fp_thing:
+            for entity in entities:
+                fp_thing.write(f"{json.dumps(entity.__import_format_dict__(), ensure_ascii=False)}\n")
+    elif file_path.suffix == ".ndjson":
+        with file_path.open("w", encoding="utf-8") as fp_thing:
+            for entity in entities:
+                fp_thing.write(f"{json.dumps(entity.__import_format_dict__(), ensure_ascii=False)}\n")
+
+
+def append_import_format(file_path: Path, entity: ThingObject) -> None:
+    """
+    Append to the import format file.
+    Parameters
+    ----------
+    file_path: Path
+        The path to the file.
+    entity: ThingObject
+        The entity to append.
+    """
+    with file_path.open("a", encoding="utf-8") as fp_thing:
+        fp_thing.write(f"{json.dumps(entity.__import_format_dict__(), ensure_ascii=False)}\n")

{personal_knowledge_library-3.1.0.dist-info → personal_knowledge_library-3.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: personal_knowledge_library
-Version: 3.1.0
+Version: 3.1.2
 Summary: Library to access Wacom's Personal Knowledge graph.
 License: Apache-2.0
 Keywords: semantic-knowledge,knowledge-graph

{personal_knowledge_library-3.1.0.dist-info → personal_knowledge_library-3.1.2.dist-info}/RECORD

@@ -1,22 +1,22 @@
-knowledge/__init__.py,sha256=jQahttsGtb_GDFJCWfJEJ3J_wJF4v-msV1VN4RUG5Fk,2680
+knowledge/__init__.py,sha256=K8mEnEYsqCb86zEKbKdwpctRUteEBwRxVRYOpeJuB8U,2680
 knowledge/base/__init__.py,sha256=q0NJRQLhfZ8khE-uZCK0SVA38dzbaMomcgL5Olnjtio,895
 knowledge/base/access.py,sha256=BSh-6QbeHOCu55XTxA-p3DwEyRzgtN8TSxtcn6UvmZo,4411
 knowledge/base/entity.py,sha256=b-Ana_H_WI2-AT_n-V_HzUL6W9Ri16DcZFS3q4ziI94,8445
 knowledge/base/language.py,sha256=QHNafhiwg9UPaQS1svzsSz7T2L-VAKlQwd1n5w0HaI8,1500
-knowledge/base/ontology.py,sha256=-v5g_0H-naeGBbYDDRenm0rTvkGFnW9ZWPdYK7xqwZs,95395
+knowledge/base/ontology.py,sha256=LKOMcnUzROK4BD7avMAbvTnjp3N8nzBwt6lLr2qLqkE,95441
 knowledge/base/search.py,sha256=J1PSVpTU2JKF9xSZLZZvAbJfFy1HiMPJRzPjHhR7IQM,14722
 knowledge/base/tenant.py,sha256=f2Z_LjUcjIt5J2_Rse9aQyTzJ0sSyVvCzlm8PP3PqY8,6084
 knowledge/nel/__init__.py,sha256=eTT88LV_KQ-Ku-a4RnTv_TUCNogl357ui4KT1pEKMuQ,330
 knowledge/nel/base.py,sha256=bsUj9PwoZoJWW33RfoYmQHbFhArKH1kMOsDgU3yj0T8,15032
 knowledge/nel/engine.py,sha256=qvQkfsdeYPWJ_5m8mmGFw1tvd699vOQ3IKoBIwALRWk,5347
-knowledge/ontomapping/__init__.py,sha256=9E-rZpxQPfukjDz_0ymrRieYDpv_e3tvpqDfkWzPJy0,18846
+knowledge/ontomapping/__init__.py,sha256=u_t6i5a6pYYnf43Q_S6sG7NRLeGuKmahIHW6TFGHqOk,18848
 knowledge/ontomapping/manager.py,sha256=pXBwRGSTcS731df5vewNv6oMgP18HzvnjZqiUgyFIhI,13361
 knowledge/public/__init__.py,sha256=FrW5sqJGVcIfg5IVpt_g7qlWiIYNGA370jkE5mJDhoo,812
-knowledge/public/cache.py,sha256=nqKIVfY45bHsuf4dOe4aPK-JfRL1pT7NQT_KF1puzLY,14509
-knowledge/public/client.py,sha256=kW4bRXgBYxHMNsyyBtMSNTYKNqReaPih_Que2lgNrG0,14676
+knowledge/public/cache.py,sha256=uKEuW9WN9xPmY-fwWPLSxmdEP6xg-XG3g8fKAmGZgBQ,14510
+knowledge/public/client.py,sha256=7Z-nYhYIni_jKDWJw18CvuUxuOCns74l8XjeEEJXl_Y,15437
 knowledge/public/helper.py,sha256=PDsInMHMHgP8rMvcFk5E47afdE3rC7V3BzPS-DdOsww,13238
 knowledge/public/relations.py,sha256=DrL-rYuwLzaE2H2TOM2gG1TGqHbkcPL4EsRD54LQqLs,4182
-knowledge/public/wikidata.py,sha256=wGM4XH5jOF1R7woML90lq28piu9AW7ADgpUcK2qIJV4,36661
+knowledge/public/wikidata.py,sha256=LwMP2kq2mH5Oi9JhPvG8lN9pZMDlQvxgaZgQKQO3kaM,36683
 knowledge/services/__init__.py,sha256=FQMTLg7RYglLwCtEO0sOrgj55YrzSNqJV-yj3aqttb8,3490
 knowledge/services/asyncio/__init__.py,sha256=dq9yGTxg_hoQb8E1kEtY4AeB8zrdJfw3-XlPoYn2j5U,225
 knowledge/services/asyncio/base.py,sha256=im6J1CWOp1N1tt6WdJ1uU0R-VFpvdYc6lZu8TyPrU5A,16222
@@ -33,11 +33,12 @@ knowledge/services/search.py,sha256=atwvcqagA6yZkzKiad6Yjn3cKu7L5LpKiVJsxKKOY5g,
 knowledge/services/session.py,sha256=y8uTETRMDOBbC30UAlrrtWTN7Zgs2klqHsoMjLO2tq0,14145
 knowledge/services/tenant.py,sha256=4pyQCYlwvREfmRcXRn47irIsFMCia_c-LVvFysDypkI,11691
 knowledge/services/users.py,sha256=uus66ijd1OAN-gJqP95zmhDL31xje3gHBfF4rMnMjM0,16436
-knowledge/utils/__init__.py,sha256=rgLqdgZwjuCnqA4NEY9echbGFaJ5YUhqBEkyVv9R9CM,255
+knowledge/utils/__init__.py,sha256=7unsBV5ge6eIyicKCIc_C4OroD-zkKRMVE0jXcH6EOw,313
 knowledge/utils/graph.py,sha256=kuPZEGhexVN9McoT8JK2ViIrfXTh-nFPv_8XPfG0wlA,12318
+knowledge/utils/import_format.py,sha256=Ho50tGlkNXoWDRFBke6fB_09PFeupTakrjdoHpNqDX4,5037
 knowledge/utils/wikidata.py,sha256=vRH-4AMR-3xywyvmDqbjI4KSw4tF4TEYqUGCJmUMqK8,5512
 knowledge/utils/wikipedia.py,sha256=rBuFqYVM58JCj5ISLxuhYVVl2gOIlPLWx7_CghyQgvE,5052
-personal_knowledge_library-3.1.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-personal_knowledge_library-3.1.0.dist-info/METADATA,sha256=3tiRSrNWLBnrHbaKzj7WC_y6VMgXKBfwLLTOnoHJqsY,57087
-personal_knowledge_library-3.1.0.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-personal_knowledge_library-3.1.0.dist-info/RECORD,,
+personal_knowledge_library-3.1.2.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+personal_knowledge_library-3.1.2.dist-info/METADATA,sha256=c3tQS96PrPj-ulfkQrgJdV49Zdj012MWYLqmTGHu_2g,57087
+personal_knowledge_library-3.1.2.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+personal_knowledge_library-3.1.2.dist-info/RECORD,,

{personal_knowledge_library-3.1.0.dist-info → personal_knowledge_library-3.1.2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.1.1
+Generator: poetry-core 2.1.2
 Root-Is-Purelib: true
 Tag: py3-none-any