personal_knowledge_library 3.2.2__tar.gz → 3.3.1__tar.gz

{personal_knowledge_library-3.2.2 → personal_knowledge_library-3.3.1}/PKG-INFO

@@ -1,8 +1,9 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: personal_knowledge_library
-Version: 3.2.2
+Version: 3.3.1
 Summary: Library to access Wacom's Personal Knowledge graph.
-License: Apache-2.0
+License-Expression: Apache-2.0
+License-File: LICENSE
 Keywords: semantic-knowledge,knowledge-graph
 Author: Markus Weber
 Author-email: markus.weber@wacom.com

{personal_knowledge_library-3.2.2 → personal_knowledge_library-3.3.1}/knowledge/__init__.py

@@ -17,7 +17,7 @@ __license__ = "Wacom"
 __maintainer__ = ["Markus Weber"]
 __email__ = "markus.weber@wacom.com"
 __status__ = "beta"
-__version__ = "3.2.2"
+__version__ = "3.3.1"
 
 import loguru
 

{personal_knowledge_library-3.2.2 → personal_knowledge_library-3.3.1}/knowledge/base/ontology.py

@@ -1494,7 +1494,7 @@ class Ontology:
         return f"<Ontology> : classes:= {self.classes}"
 
 
-class ThingObject(abc.ABC):
+class ThingObject(object):
     """
     ThingObject
     -----------

{personal_knowledge_library-3.2.2 → personal_knowledge_library-3.3.1}/knowledge/nel/base.py

@@ -162,6 +162,10 @@ class KnowledgeGraphEntity(NamedEntity):
         List of ontology types (class names)
     entity_type: EntityType
         Type of the entity.
+    tokens: Optional[List[str]] (default:=None)
+        List of tokens used to identify the entity.
+    token_indexes: Optional[List[int]] (default:=None)
+        List of token indexes used to identify the entity.
     """
 
     def __init__(
@@ -175,6 +179,9 @@ class KnowledgeGraphEntity(NamedEntity):
         content_link: str,
         ontology_types: List[str],
         entity_type: EntityType = EntityType.PUBLIC_ENTITY,
+        tokens: Optional[List[str]] = None,
+        token_indexes: Optional[List[int]] = None,
+
     ):
         super().__init__(ref_text, start_idx, end_idx, entity_type)
         self.__source: EntitySource = source
@@ -185,6 +192,8 @@ class KnowledgeGraphEntity(NamedEntity):
         self.__thumbnail: Optional[str] = None
         self.__ontology_types: List[str] = ontology_types
         self.__relevant_type: OntologyClassReference = THING_CLASS
+        self.__tokens: Optional[List[str]] = tokens
+        self.__token_indexes: Optional[List[int]] = token_indexes
 
     @property
     def entity_source(self) -> EntitySource:
@@ -242,6 +251,16 @@ class KnowledgeGraphEntity(NamedEntity):
     def relevant_type(self, value: OntologyClassReference):
         self.__relevant_type = value
 
+    @property
+    def tokens(self) -> Optional[List[str]]:
+        """List of tokens used to identify the entity."""
+        return self.__tokens
+
+    @property
+    def token_indexes(self) -> Optional[List[int]]:
+        """List of token indexes used to identify the entity."""
+        return self.__token_indexes
+
     def __repr__(self):
         return f"{self.ref_text} [{self.start_idx}-{self.end_idx}] -> {self.entity_source} [{self.entity_type}]"
 

{personal_knowledge_library-3.2.2 → personal_knowledge_library-3.3.1}/knowledge/nel/engine.py

@@ -116,6 +116,8 @@ class WacomEntityLinkingEngine(PersonalEntityLinkingProcessor):
                         content_link="",
                         ontology_types=entity_types,
                         entity_type=EntityType.PERSONAL_ENTITY,
+                        tokens=e.get("tokens"),
+                        token_indexes=e.get("tokenIndexes"),
                     )
                     ne.relevant_type = OntologyClassReference.parse(e["type"])
                     named_entities.append(ne)

{personal_knowledge_library-3.2.2 → personal_knowledge_library-3.3.1}/knowledge/services/asyncio/graph.py

@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 # Copyright © 2024-present Wacom. All rights reserved.
 import asyncio
+import gzip
+import json
 import logging
 import os
 import urllib
@@ -35,6 +37,7 @@ from knowledge.base.ontology import (
     OntologyClassReference,
     ObjectProperty,
 )
+from knowledge.base.response import JobStatus, ErrorLogResponse, NewEntityUrisResponse
 from knowledge.nel.base import KnowledgeGraphEntity, EntityType, KnowledgeSource, EntitySource
 from knowledge.services import AUTHORIZATION_HEADER_FLAG, IS_OWNER_PARAM, IndexType
 from knowledge.services import (
@@ -114,6 +117,8 @@ class AsyncWacomKnowledgeService(AsyncServiceAPIClient):
     SEARCH_DESCRIPTION_ENDPOINT: str = "semantic-search/description"
     SEARCH_RELATION_ENDPOINT: str = "semantic-search/relation"
     ONTOLOGY_UPDATE_ENDPOINT: str = "ontology-update"
+    IMPORT_ENTITIES_ENDPOINT: str = "import"
+    IMPORT_ERROR_LOG_ENDPOINT: str = "import/errorlog"
 
     def __init__(
         self,
@@ -588,6 +593,222 @@ class AsyncWacomKnowledgeService(AsyncServiceAPIClient):
         await asyncio.sleep(0.25 if self.use_graceful_shutdown else 0.0)
         return uri
 
+    async def import_entities(
+        self,
+        entities: List[ThingObject],
+        auth_key: Optional[str] = None,
+        timeout: int = DEFAULT_TIMEOUT
+    ) -> str:
+        """Import entities to the graph.
+
+        Parameters
+        ----------
+        entities: List[ThingObject]
+            List of entities to import.
+        auth_key: Optional[str] = None
+            If the auth key is set the logged-in user (if any) will be ignored and the auth key will be used.
+        timeout: int
+            Timeout for the request (default: 60 seconds)
+
+        Returns
+        -------
+        job_id: str
+            ID of the job
+
+        Raises
+        ------
+        WacomServiceException
+            If the graph service returns an error code.
+        """
+        if auth_key is None:
+            auth_key, _ = await self.handle_token()
+        headers: dict = {USER_AGENT_HEADER_FLAG: self.user_agent, AUTHORIZATION_HEADER_FLAG: f"Bearer {auth_key}"}
+        ndjson_lines: List[str] = []
+        for obj in entities:
+            data_dict = obj.__import_format_dict__()
+            ndjson_lines.append(json.dumps(data_dict))  # Convert each dict to a JSON string
+
+        ndjson_content = "\n".join(ndjson_lines)  # Join JSON strings with newline
+
+        # Compress the NDJSON string to a gzip byte array
+        compressed_data: bytes = gzip.compress(ndjson_content.encode("utf-8"))
+        url: str = f"{self.service_base_url}{self.IMPORT_ENTITIES_ENDPOINT}"
+        data: aiohttp.FormData = aiohttp.FormData()
+        data.add_field("file", compressed_data, filename="import.njson.gz", content_type="application/x-gzip")
+        async with AsyncServiceAPIClient.__async_session__() as session:
+            async with session.post(
+                url, headers=headers, data=data, timeout=timeout, verify_ssl=self.verify_calls
+            ) as response:
+                if response.ok:
+                    structure: Dict[str, Any] = await response.json(loads=orjson.loads)
+                    return structure["jobId"]
+                raise await handle_error("Import endpoint returns an error.", response)
+
+    async def import_entities_from_file(
+            self,
+            file_path: Path,
+            auth_key: Optional[str] = None,
+            timeout: int = DEFAULT_TIMEOUT
+    ) -> str:
+        """Import entities from a file to the graph.
+
+        Parameters
+        ----------
+        file_path: Path
+            Path to the file containing entities in NDJSON format.
+        auth_key: Optional[str] = None
+            If the auth key is set the logged-in user (if any) will be ignored and the auth key will be used.
+        timeout: int
+            Timeout for the request (default: 60 seconds)
+
+        Returns
+        -------
+        job_id: str
+            ID of the job
+
+        Raises
+        ------
+        WacomServiceException
+            If the graph service returns an error code.
+        """
+        if not file_path.exists():
+            raise FileNotFoundError(f"The file {file_path} does not exist.")
+        if auth_key is None:
+            auth_key, _ = await self.handle_token()
+        headers: dict = {USER_AGENT_HEADER_FLAG: self.user_agent, AUTHORIZATION_HEADER_FLAG: f"Bearer {auth_key}"}
+        with file_path.open("rb") as file:
+            # Compress the NDJSON string to a gzip byte array
+            compressed_data: bytes = file.read()
+            data: aiohttp.FormData = aiohttp.FormData()
+            data.add_field("file", compressed_data, filename="import.njson.gz",
+                           content_type="application/x-gzip")
+            url: str = f"{self.service_base_url}{self.IMPORT_ENTITIES_ENDPOINT}"
+            async with AsyncServiceAPIClient.__async_session__() as session:
+                async with session.post(
+                        url, headers=headers, data=data, timeout=timeout, verify_ssl=self.verify_calls
+                ) as response:
+                    if response.ok:
+                        structure: Dict[str, Any] = await response.json(loads=orjson.loads)
+                        return structure["jobId"]
+                    raise await handle_error("Import endpoint returns an error.", response)
+
+    async def job_status(
+            self,
+            job_id: str,
+            auth_key: Optional[str] = None,
+            timeout: int = DEFAULT_TIMEOUT
+    ) -> JobStatus:
+        """
+        Retrieve the status of the job.
+
+        Parameters
+        ----------
+        job_id: str
+            ID of the job
+        auth_key: Optional[str] = None
+            If the auth key is set the logged-in user (if any) will be ignored and the auth key will be used.
+        timeout: int
+            Timeout for the request (default: 60 seconds)
+
+        Returns
+        -------
+        job_status: JobStatus
+            Status of the job
+        """
+        if auth_key is None:
+            auth_key, _ = await self.handle_token()
+        headers: dict = {USER_AGENT_HEADER_FLAG: self.user_agent, AUTHORIZATION_HEADER_FLAG: f"Bearer {auth_key}"}
+        url: str = f"{self.service_base_url}{self.IMPORT_ENTITIES_ENDPOINT}/{job_id}"
+        async with AsyncServiceAPIClient.__async_session__() as session:
+            async with session.get(
+                    url, headers=headers, timeout=timeout, verify_ssl=self.verify_calls
+            ) as response:
+                if response.ok:
+                    structure: Dict[str, Any] = await response.json(loads=orjson.loads)
+                    return JobStatus.from_dict(structure)
+                raise await handle_error(f"Retrieving job status for {job_id} failed.", response, headers=headers)
+
+    async def import_error_log(
+            self,
+            job_id: str,
+            auth_key: Optional[str] = None,
+            next_page_id: Optional[str] = None,
+            timeout: int = DEFAULT_TIMEOUT
+    ) -> ErrorLogResponse:
+        """
+        Retrieve the error log of the job.
+
+        Parameters
+        ----------
+        job_id: str
+            ID of the job
+        next_page_id: Optional[str] = None
+            ID of the next page within pagination.
+        auth_key: Optional[str] = None
+            If the auth key is set the logged-in user (if any) will be ignored and the auth key will be used.
+        timeout: int
+            Timeout for the request (default: 60 seconds)
+
+        Returns
+        -------
+        error: ErrorLogResponse
+            Error log of the job
+        """
+        if auth_key is None:
+            auth_key, _ = await self.handle_token()
+        headers: dict = {USER_AGENT_HEADER_FLAG: self.user_agent, AUTHORIZATION_HEADER_FLAG: f"Bearer {auth_key}"}
+        url: str = f"{self.service_base_url}{self.IMPORT_ERROR_LOG_ENDPOINT}/{job_id}"
+        params: Dict[str, str] = {NEXT_PAGE_ID_TAG: next_page_id} if next_page_id else {}
+        async with AsyncServiceAPIClient.__async_session__() as session:
+            async with session.get(
+                    url, headers=headers, params=params, timeout=timeout, verify_ssl=self.verify_calls
+            ) as response:
+                if response.ok:
+                    structure: Dict[str, Any] = await response.json(loads=orjson.loads)
+                    return ErrorLogResponse.from_dict(structure)
+                raise await handle_error(f"Retrieving job status for {job_id} failed.", response)
+
+    async def import_new_uris(
+            self,
+            job_id: str,
+            auth_key: Optional[str] = None,
+            next_page_id: Optional[str] = None,
+            timeout: int = DEFAULT_TIMEOUT
+    ) -> NewEntityUrisResponse:
+        """
+        Retrieve the new entity uris from the job.
+
+        Parameters
+        ----------
+        job_id: str
+            ID of the job
+        next_page_id: Optional[str] = None
+            ID of the next page within pagination.
+        auth_key: Optional[str] = None
+            If the auth key is set the logged-in user (if any) will be ignored and the auth key will be used.
+        timeout: int
+            Timeout for the request (default: 60 seconds)
+
+        Returns
+        -------
+        response: NewEntityUrisResponse
+            New entity uris of the job.
+        """
+        if auth_key is None:
+            auth_key, _ = await self.handle_token()
+        headers: dict = {USER_AGENT_HEADER_FLAG: self.user_agent, AUTHORIZATION_HEADER_FLAG: f"Bearer {auth_key}"}
+        url: str = f"{self.service_base_url}{self.IMPORT_ENTITIES_ENDPOINT}/{job_id}/new-entities"
+        params: Dict[str, str] = {NEXT_PAGE_ID_TAG: next_page_id} if next_page_id else {}
+        async with AsyncServiceAPIClient.__async_session__() as session:
+            async with session.get(
+                    url, headers=headers, params=params, timeout=timeout, verify_ssl=self.verify_calls
+            ) as response:
+                if response.ok:
+                    structure: Dict[str, Any] = await response.json(loads=orjson.loads)
+                    return NewEntityUrisResponse.from_dict(structure)
+                raise await handle_error(f"Retrieving job status for {job_id} failed.", response)
+
+
     async def update_entity(self, entity: ThingObject, auth_key: Optional[str] = None):
         """
         Updates entity in graph.
@@ -1576,6 +1797,8 @@ class AsyncWacomKnowledgeService(AsyncServiceAPIClient):
                             content_link="",
                             ontology_types=entity_types,
                             entity_type=EntityType.PERSONAL_ENTITY,
+                            tokens=e.get("tokens"),
+                            token_indexes=e.get("tokenIndexes"),
                         )
                         ne.relevant_type = OntologyClassReference.parse(e["type"])
                         named_entities.append(ne)

{personal_knowledge_library-3.2.2 → personal_knowledge_library-3.3.1}/knowledge/services/graph.py

@@ -575,6 +575,8 @@ class WacomKnowledgeService(WacomServiceAPIClient):
                     if entity.image is not None and entity.image.startswith("file:"):
                         p = urlparse(entity.image)
                         self.set_entity_image_local(uri, Path(p.path), auth_key=auth_key)
+                    elif entity.image is not None and entity.image.startswith("/"):
+                        self.set_entity_image_local(uri, Path(entity.image), auth_key=auth_key)
                     elif entity.image is not None and entity.image != "":
                         self.set_entity_image_url(uri, entity.image, auth_key=auth_key)
                 except WacomServiceException as _:
@@ -1879,6 +1881,63 @@ class WacomKnowledgeService(WacomServiceAPIClient):
                 return response.json()["jobId"]
             raise handle_error("Import endpoint returns an error.", response)
 
+    def import_entities_from_file(
+        self,
+        file_path: Path,
+        auth_key: Optional[str] = None,
+        timeout: int = DEFAULT_TIMEOUT,
+        max_retries: int = DEFAULT_MAX_RETRIES,
+        backoff_factor: float = DEFAULT_BACKOFF_FACTOR,
+    ) -> str:
+        """Import entities from a file to the graph.
+
+        Parameters
+        ----------
+        file_path: Path
+            Path to the file containing entities in NDJSON format.
+        auth_key: Optional[str] = None
+            If the auth key is set the logged-in user (if any) will be ignored and the auth key will be used.
+        timeout: int
+            Timeout for the request (default: 60 seconds)
+        max_retries: int
+            Maximum number of retries
+        backoff_factor: float
+            A backoff factor to apply between attempts after the second try (most errors are resolved immediately by a
+            second try without a delay)
+
+        Returns
+        -------
+        job_id: str
+            ID of the job
+
+        Raises
+        ------
+        WacomServiceException
+            If the graph service returns an error code.
+        """
+        if not file_path.exists():
+            raise FileNotFoundError(f"The file {file_path} does not exist.")
+        if auth_key is None:
+            auth_key, _ = self.handle_token()
+        headers: dict = {USER_AGENT_HEADER_FLAG: self.user_agent, AUTHORIZATION_HEADER_FLAG: f"Bearer {auth_key}"}
+        with file_path.open("rb") as file:
+            # Compress the NDJSON string to a gzip byte array
+            compressed_data: bytes = file.read()
+            files: List[Tuple[str, Tuple[str, bytes, str]]] = [
+                ("file", ("import.njson.gz", compressed_data, "application/x-gzip"))
+            ]
+            url: str = f"{self.service_base_url}{self.IMPORT_ENTITIES_ENDPOINT}"
+            mount_point: str = "https://" if self.service_url.startswith("https") else "http://"
+            with requests.Session() as session:
+                retries: Retry = Retry(total=max_retries, backoff_factor=backoff_factor, status_forcelist=STATUS_FORCE_LIST)
+                session.mount(mount_point, HTTPAdapter(max_retries=retries))
+                response: Response = session.post(
+                    url, headers=headers, files=files, timeout=timeout, verify=self.verify_calls
+                )
+                if response.ok:
+                    return response.json()["jobId"]
+                raise handle_error("Import endpoint returns an error.", response)
+
     def job_status(
         self,
         job_id: str,

{personal_knowledge_library-3.2.2 → personal_knowledge_library-3.3.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "personal_knowledge_library"
-version = "3.2.2"
+version = "3.3.1"
 description = "Library to access Wacom's Personal Knowledge graph."
 authors = [{ name = "Markus Weber", email = "markus.weber@wacom.com" }]
 license = "Apache-2.0"