personal_knowledge_library 3.0.0__py3-none-any.whl → 3.1.1__py3-none-any.whl

knowledge/__init__.py

@@ -17,7 +17,7 @@ __license__ = "Wacom"
 __maintainer__ = ["Markus Weber"]
 __email__ = "markus.weber@wacom.com"
 __status__ = "beta"
-__version__ = "3.0.0"
+__version__ = "3.1.1"
 
 import loguru
 

knowledge/ontomapping/__init__.py

@@ -10,7 +10,9 @@ import loguru
 from rdflib import Graph, RDFS, URIRef
 
 from knowledge.base.ontology import OntologyClassReference, OntologyPropertyReference, DataPropertyType
-from knowledge.public.wikidata import WikidataClass, WikiDataAPIClient
+from knowledge.public.cache import WikidataCache
+from knowledge.public.wikidata import WikidataClass
+from knowledge.public.client import WikiDataAPIClient
 
 # Classes
 TOPIC_CLASS: str = "wacom:core#Topic"
@@ -25,11 +27,8 @@ CONTEXT_NAME: str = "core"
 CWD: Path = Path(__file__).parent
 ontology_graph: Graph = Graph()
 logger = loguru.logger
-# Mapping configuration
-superclasses_cache: Optional[Dict[str, WikidataClass]] = None
-subclasses_cache: Optional[Dict[str, WikidataClass]] = None
-superclasses_path: Optional[Path] = None
-subclasses_path: Optional[Path] = None
+# Cache
+wikidata_cache: WikidataCache() = WikidataCache()
 
 
 def flatten(hierarchy: WikidataClass) -> Set[str]:
@@ -45,15 +44,24 @@ def flatten(hierarchy: WikidataClass) -> Set[str]:
     -------
     hierarchy: Set[str]
         Hierarchy
-
     """
-    hierarchy_set: Set[str] = {hierarchy.qid}
+    hierarchy_set: Set[str] = set()
     jobs: List[WikidataClass] = [hierarchy]
-    while len(jobs) > 0:
-        job: WikidataClass = jobs.pop()
+
+    while jobs:
+        job = jobs.pop()
+
+        # Skip if already visited
+        if job.qid in hierarchy_set:
+            continue
+
         hierarchy_set.add(job.qid)
+
+        # Add only unvisited subclasses
         for c in job.subclasses:
-            jobs.append(c)
+            if c.qid not in hierarchy_set:
+                jobs.append(c)
+
     return hierarchy_set
 
 
@@ -334,7 +342,7 @@ class MappingConfiguration:
                         properties.append(prop_conf)
         return properties
 
-    def add_class(self, class_configuration: ClassConfiguration, subclasses: Dict[str, List[str]]):
+    def add_class(self, class_configuration: ClassConfiguration):
         """
         Adds a class configuration.
 
@@ -342,30 +350,30 @@ class MappingConfiguration:
         ----------
         class_configuration: ClassConfiguration
             The class configuration
-        subclasses: Dict[str, List[str]]
-            The subclasses
         """
         self.__classes.append(class_configuration)
         class_idx: int = len(self.__classes) - 1
         number_of_classes: int = len(class_configuration.wikidata_classes)
         if number_of_classes > 0:
-            logger.info(f"Adding {number_of_classes} classes for {class_configuration.concept_type.iri}")
-        for cls_idx, c in enumerate(class_configuration.wikidata_classes):
-            if c in subclasses:
-                for subclass in subclasses[c]:
+            logger.debug(f"Adding {number_of_classes} classes for {class_configuration.concept_type.iri}")
+        for _, c in enumerate(class_configuration.wikidata_classes):
+            if wikidata_cache.subclass_in_cache(c):
+                for subclass in wikidata_cache.get_subclass(c).subclasses:
                     if subclass in self.__index:
                         logger.warning(f"Class {subclass} already exists in the index.")
                         class_config: ClassConfiguration = self.__classes[self.__index[subclass]]
-                        logger.warning(f"Class {class_config.concept_type} "
-                                       f"is conflicting with {class_configuration.concept_type}.")
+                        logger.warning(
+                            f"Class {class_config.concept_type} "
+                            f"is conflicting with {class_configuration.concept_type}."
+                        )
                     self.__index[subclass] = class_idx
                 self.__direct_index[c] = class_idx
             else:
                 w_classes: Dict[str, WikidataClass] = WikiDataAPIClient.subclasses(c)
                 for subclass in w_classes.values():
+                    wikidata_cache.cache_subclass(subclass)
                     for cls in flatten(subclass):
                         self.__index[cls] = class_idx
-
         for c in class_configuration.dbpedia_classes:
             self.__index[c] = len(self.__classes) - 1
 
@@ -480,15 +488,13 @@ class MappingConfiguration:
 mapping_configuration: Optional[MappingConfiguration] = None
 
 
-def build_configuration(mapping: Dict[str, Any], subclasses: Dict[str, List[str]]) -> MappingConfiguration:
+def build_configuration(mapping: Dict[str, Any]) -> MappingConfiguration:
     """
     Builds the configuration from the mapping file.
     Parameters
     ----------
     mapping: Dict[str, Any]
         The mapping file
-    subclasses: Dict[str, List[str]]
-        The subclasses
 
     Returns
     -------
@@ -502,7 +508,7 @@ def build_configuration(mapping: Dict[str, Any], subclasses: Dict[str, List[str]
         class_config: ClassConfiguration = ClassConfiguration(c)
         class_config.dbpedia_classes = c_conf[DBPEDIA_TYPES]
         class_config.wikidata_classes = c_conf[WIKIDATA_TYPES]
-        conf.add_class(class_config, subclasses)
+        conf.add_class(class_config)
     dataproperty_count: int = len(mapping["data_properties"])
     logger.debug(f"Adding {dataproperty_count} data properties to the mapping configuration")
     for p, p_conf in mapping["data_properties"].items():
@@ -517,7 +523,6 @@ def build_configuration(mapping: Dict[str, Any], subclasses: Dict[str, List[str]
             for do in p_conf["domains"]:
                 property_config.domains.append(do)
                 property_config.domains.extend(subclasses_of(do))
-
         conf.add_property(property_config)
     object_property_count: int = len(mapping["object_properties"])
     logger.debug(f"Adding {object_property_count} object properties to the mapping configuration")
@@ -538,20 +543,6 @@ def build_configuration(mapping: Dict[str, Any], subclasses: Dict[str, List[str]
         conf.add_property(property_config)
     return conf
 
-
-def update_superclass_cache(path: Path):
-    """
-    Updates the taxonomy cache.
-
-    Parameters
-    ----------
-    path: Path
-        The path to the cache file.
-    """
-    with open(path, "w", encoding="uft-8") as fp_taxonomy_write:
-        fp_taxonomy_write.write(json.dumps(superclasses_cache, indent=2, cls=WikidataClassEncoder))
-
-
 def register_ontology(rdf_str: str):
     """
     Registers the ontology.
@@ -562,41 +553,6 @@ def register_ontology(rdf_str: str):
     """
     ontology_graph.parse(data=rdf_str, format="xml")
 
-
-def superclass_path_from(configuration_path: Path) -> Path:
-    """
-    Returns the path to the superclass cache file.
-
-    Parameters
-    ----------
-    configuration_path: Path
-        The path to the configuration file.
-
-    Returns
-    -------
-    path: Path
-        The path to the superclass cache file.
-    """
-    return configuration_path.parent / "superclasses.json"
-
-
-def subclass_path_from(path: Path):
-    """
-    Returns the path to the subclass cache file.
-
-    Parameters
-    ----------
-    path: Path
-        The path to the configuration file.
-
-    Returns
-    -------
-    subclass_path: Path
-        The path to the subclass cache file.
-    """
-    return path.parent / "subclasses.json"
-
-
 def load_configuration(configuration: Path):
     """
     Loads the configuration.
@@ -606,17 +562,11 @@ def load_configuration(configuration: Path):
     ValueError
         If the configuration file is not found.
     """
-    global mapping_configuration, superclasses_cache, subclasses_cache, superclasses_path, subclasses_path
-    subclasses_path = subclass_path_from(configuration)
-    superclasses_path = superclass_path_from(configuration)
+    global mapping_configuration
     if configuration.exists():
         with configuration.open("r", encoding="utf-8") as fp_configuration:
             configuration = json.loads(fp_configuration.read())
-        subclasses: Dict[str, List[str]] = {}
-        if subclasses_path.exists():
-            with subclasses_path.open("r", encoding="utf-8") as fp_sub:
-                subclasses = json.loads(fp_sub.read())
-        mapping_configuration = build_configuration(configuration, subclasses)
+        mapping_configuration = build_configuration(configuration)
     else:
         raise ValueError(f"Configuration file {configuration} not found.")
 
@@ -633,35 +583,3 @@ def get_mapping_configuration() -> MappingConfiguration:
     if mapping_configuration is None:
         raise ValueError("Please load configuration")
     return mapping_configuration
-
-
-def save_superclasses_cache(path: Path):
-    """
-    Saves the taxonomy cache.
-
-    Parameters
-    ----------
-    path: Path
-        The path to the cache file.
-    """
-    global superclasses_cache
-    if superclasses_cache is None:
-        return
-    with open(path, "w", encoding="utf-8") as fp_taxonomy_write:
-        fp_taxonomy_write.write(json.dumps(superclasses_cache, indent=2, cls=WikidataClassEncoder))
-
-
-def save_subclasses_cache(path: Path):
-    """
-    Saves the taxonomy cache.
-
-    Parameters
-    ----------
-    path: Path
-        The path to the cache file.
-    """
-    global subclasses_cache
-    if subclasses_cache is None:
-        return
-    with open(path, "w", encoding="utf-8") as fp_taxonomy_write:
-        fp_taxonomy_write.write(json.dumps(subclasses_cache, indent=2, cls=WikidataClassEncoder))

knowledge/ontomapping/manager.py

@@ -1,11 +1,13 @@
 # -*- coding: utf-8 -*-
 # Copyright © 2023-present Wacom. All rights reserved.
-import logging
 import time
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import Optional, Any, List, Dict, Tuple, Set
 
+import loguru
+
 from knowledge.base.entity import Label, LanguageCode, Description
+from knowledge.base.language import LOCALE_LANGUAGE_MAPPING, LocaleCode, LANGUAGE_LOCALE_MAPPING, EN_US
 from knowledge.base.ontology import (
     ThingObject,
     DataProperty,
@@ -18,18 +20,19 @@ from knowledge.base.ontology import (
 from knowledge.ontomapping import (
     ClassConfiguration,
     TOPIC_CLASS,
-    superclasses_cache,
     PropertyConfiguration,
     PropertyType,
     get_mapping_configuration,
-    save_superclasses_cache,
-    superclasses_path,
 )
-from knowledge.public.cache import pull_wikidata_object
-from knowledge.public.wikidata import WikidataThing, WikiDataAPIClient, WikidataClass, WikidataProperty
-from knowledge.base.language import LOCALE_LANGUAGE_MAPPING, LocaleCode, LANGUAGE_LOCALE_MAPPING, EN_US
+from knowledge.public.cache import WikidataCache
+from knowledge.public.client import WikiDataAPIClient
+from knowledge.public.wikidata import WikidataThing, WikidataClass, WikidataProperty
 from knowledge.utils.wikipedia import get_wikipedia_summary
 
+logger = loguru.logger
+# Initialize the cache
+cache: WikidataCache = WikidataCache()
+
 
 def flatten(hierarchy: WikidataClass, use_names: bool = False) -> List[str]:
     """
@@ -78,16 +81,10 @@ def wikidata_taxonomy(qid: str) -> Optional[WikidataClass]:
     hierarchy: WikidataClass
         Hierarchy.
     """
-    if superclasses_cache and qid in superclasses_cache:
-        taxonomy: WikidataClass = superclasses_cache[qid]
-        return taxonomy
     hierarchy: Dict[str, WikidataClass] = WikiDataAPIClient.superclasses(qid)
     if qid not in hierarchy:
-        logging.warning(f"Taxonomy for {qid} not found.")
+        logger.warning(f"Taxonomy for {qid} not found.")
         return None
-    if hierarchy:
-        superclasses_cache.update(hierarchy)
-        save_superclasses_cache(superclasses_path)
     return hierarchy.get(qid)
 
 
@@ -116,10 +113,13 @@ def convert_dict(structure: Dict[str, Any], locale: str) -> Optional[str]:
         if structure_type == "quantity" and isinstance(value, dict):
             return value["amount"]
         if structure_type == "wikibase-item" and isinstance(value, dict):
-            wikidata_data: WikidataThing = pull_wikidata_object(value["id"])
+            qid: str = value["id"]
+            if cache.qid_in_cache(qid):
+                wikidata_data: WikidataThing = cache.get_wikidata_object(qid)
+            else:
+                wikidata_data: WikidataThing = WikiDataAPIClient.retrieve_entity(qid)
             if locale in wikidata_data.label:
                 return wikidata_data.label[locale].content
-            return None
         if structure_type == "external-id":
             return value
         if structure_type == "string":
@@ -215,7 +215,7 @@ def wikidata_to_thing(
                             )
                         )
                     except Exception as e:
-                        logging.error(f"Failed to get Wikipedia summary for {title} ({lang}): {e}")
+                        logger.error(f"Failed to get Wikipedia summary for {title} ({lang}): {e}")
     if len(descriptions) == 0:
         descriptions = list(wikidata_thing.description.values())
     t3: float = time.perf_counter()
@@ -228,7 +228,8 @@ def wikidata_to_thing(
     )
     thing.add_data_property(
         DataProperty(
-            content=datetime.utcnow().isoformat(), property_ref=OntologyPropertyReference.parse("wacom:core#lastUpdate")
+            content=datetime.now(timezone.utc).isoformat(),
+            property_ref=OntologyPropertyReference.parse("wacom:core#lastUpdate"),
         )
     )
     t4: float = time.perf_counter()
@@ -291,7 +292,6 @@ def wikidata_to_thing(
                                 "source_concept": thing.concept_type,
                                 "source_classes": class_types,
                                 "property": prop_missing.pid,
-                                "property_label": prop_missing.label,
                                 "target_qid": target_thing.qid,
                                 "target_classes": target_thing.ontology_types,
                             }
@@ -304,7 +304,6 @@ def wikidata_to_thing(
                         "source_concept": thing.concept_type,
                         "source_classes": class_types,
                         "property": prop_missing.pid,
-                        "property_label": prop_missing.label,
                         "target_qid": target_thing.qid,
                         "target_classes": target_thing.ontology_types,
                     }
@@ -312,9 +311,9 @@ def wikidata_to_thing(
     for p, lst in relation_props.items():
         thing.add_relation(ObjectProperty(p, outgoing=lst))
     t7: float = time.perf_counter()
-    logging.debug(
-        f"Wikidata to Thing: {t2 - t1} seconds for labels, {t3 - t2} seconds for descriptions, "
-        f"{t4 - t3} seconds for sources, {t5 - t4} seconds for class types, {t6 - t5} seconds for data "
-        f"properties, {t7 - t6} seconds for object properties"
+    logger.debug(
+        f"Wikidata to Thing: {(t2 - t1) * 1000.:.2f} ms for labels, {(t3 - t2) * 1000.:.2f} ms for descriptions, "
+        f"{(t4 - t3) * 1000.:.2f} ms for sources, {(t5 - t4) * 1000.:.2f} ms for class types, "
+        f"{(t6 - t5) * 1000.:.2f} ms for data properties, {(t7 - t6) * 1000.:.2f} ms for object properties"
     )
     return thing, import_warnings

knowledge/public/__init__.py

@@ -3,20 +3,21 @@
 """Mapping of Wikidata property ids to its string."""
 import json
 from pathlib import Path
-from typing import Dict
+from typing import Dict, List
 
 # OntologyPropertyReference constants
 INSTANCE_OF_PROPERTY: str = "P31"
 IMAGE_PROPERTY: str = "P18"
 
 # Mapping for property names
-PROPERTY_MAPPING: Dict[str, str] = {}
+DEFAULT_TIMEOUT: int = 60
+DEFAULT_TOKEN_REFRESH_TIME: int = 360
+STATUS_FORCE_LIST: List[int] = [429, 500, 502, 503, 504]
+DEFAULT_BACKOFF_FACTOR: float = 0.1
+DEFAULT_MAX_RETRIES: int = 3
 
 CWD: Path = Path(__file__).parent
-CONFIGURATION_FILE: Path = CWD / "../../pkl-cache/property_cache.json"
-if CONFIGURATION_FILE.exists():
-    with CONFIGURATION_FILE.open("r") as f:
-        PROPERTY_MAPPING = json.load(f)
+
 
 from knowledge.public import wikidata
 from knowledge.public import helper
@@ -24,4 +25,4 @@ from knowledge.public import relations
 from knowledge.public import cache
 
 
-__all__ = ["wikidata", "helper", "relations", "cache", "PROPERTY_MAPPING", "INSTANCE_OF_PROPERTY", "IMAGE_PROPERTY"]
+__all__ = ["wikidata", "helper", "relations", "cache", "client", "INSTANCE_OF_PROPERTY", "IMAGE_PROPERTY"]