isar 1.20.2__py3-none-any.whl → 1.34.13__py3-none-any.whl

isar/storage/blob_storage.py

@@ -1,83 +1,101 @@
 import logging
 from pathlib import Path
-from typing import Union
 
 from azure.core.exceptions import ResourceExistsError
-from azure.storage.blob import BlobClient, BlobServiceClient, ContainerClient
-from injector import inject
+from azure.storage.blob import BlobServiceClient, ContainerClient
 
 from isar.config.keyvault.keyvault_service import Keyvault
 from isar.config.settings import settings
-from isar.storage.storage_interface import StorageException, StorageInterface
+from isar.storage.storage_interface import (
+    BlobStoragePath,
+    StorageException,
+    StorageInterface,
+    StoragePaths,
+)
 from isar.storage.utilities import construct_metadata_file, construct_paths
-from robot_interface.models.inspection.inspection import Inspection
+from robot_interface.models.inspection.inspection import InspectionBlob
 from robot_interface.models.mission.mission import Mission
 
 
 class BlobStorage(StorageInterface):
-    @inject
-    def __init__(
-        self, keyvault: Keyvault, container_name: str = settings.BLOB_CONTAINER
-    ):
-        self.keyvault = keyvault
-        self.storage_connection_string = self.keyvault.get_secret(
-            "AZURE-STORAGE-CONNECTION-STRING"
-        ).value
-        self.container_name = container_name
-
-        self.blob_service_client = self._get_blob_service_client()
-        self.container_client = self._get_container_client(
-            blob_service_client=self.blob_service_client
+    def __init__(self, keyvault: Keyvault) -> None:
+        self.logger = logging.getLogger("uploader")
+
+        self.container_client_data = self._get_container_client(
+            keyvault, "AZURE-STORAGE-CONNECTION-STRING-DATA"
+        )
+        self.container_client_metadata = self._get_container_client(
+            keyvault, "AZURE-STORAGE-CONNECTION-STRING-METADATA"
         )
 
-        self.logger = logging.getLogger("uploader")
+    def _get_container_client(self, keyvault: Keyvault, secret_name: str):
+        storage_connection_string = keyvault.get_secret(secret_name).value
 
-    def store(self, inspection: Inspection, mission: Mission) -> Union[str, dict]:
-        data_path, metadata_path = construct_paths(
+        if storage_connection_string is None:
+            raise RuntimeError(f"{secret_name} from keyvault is None")
+
+        try:
+            blob_service_client = BlobServiceClient.from_connection_string(
+                storage_connection_string
+            )
+        except Exception as e:
+            self.logger.error("Unable to retrieve blob service client. Error: %s", e)
+            raise e
+
+        container_client = blob_service_client.get_container_client(
+            settings.BLOB_CONTAINER
+        )
+
+        if not container_client.exists():
+            raise RuntimeError(
+                "The configured blob container %s does not exist",
+                settings.BLOB_CONTAINER,
+            )
+        return container_client
+
+    def store(
+        self, inspection: InspectionBlob, mission: Mission
+    ) -> StoragePaths[BlobStoragePath]:
+        if inspection.data is None:
+            raise StorageException("Nothing to store. The inspection data is empty")
+
+        data_filename, metadata_filename = construct_paths(
             inspection=inspection, mission=mission
         )
 
         metadata_bytes: bytes = construct_metadata_file(
-            inspection=inspection, mission=mission, filename=data_path.name
+            inspection=inspection, mission=mission, filename=data_filename.name
         )
 
-        self._upload_file(path=metadata_path, data=metadata_bytes)
-        return self._upload_file(path=data_path, data=inspection.data)
+        data_path = self._upload_file(
+            filename=data_filename,
+            data=inspection.data,
+            container_client=self.container_client_data,
+        )
+        metadata_path = self._upload_file(
+            filename=metadata_filename,
+            data=metadata_bytes,
+            container_client=self.container_client_metadata,
+        )
+        return StoragePaths(data_path=data_path, metadata_path=metadata_path)
 
-    def _upload_file(self, path: Path, data: bytes) -> Union[str, dict]:
-        blob_client = self._get_blob_client(path)
+    def _upload_file(
+        self, filename: Path, data: bytes, container_client: ContainerClient
+    ) -> BlobStoragePath:
+        blob_client = container_client.get_blob_client(filename.as_posix())
         try:
-            blob_properties = blob_client.upload_blob(data=data)
+            blob_client.upload_blob(data=data)
         except ResourceExistsError as e:
             self.logger.error(
-                f"Blob {path.as_posix()} already exists in container. Error: {e}"
+                "Blob %s already exists in container. Error: %s", filename.as_posix(), e
             )
             raise StorageException from e
         except Exception as e:
             self.logger.error("An unexpected error occurred while uploading blob")
             raise StorageException from e
 
-        absolute_inspection_path = {
-            "source": "blob",
-            "blob_storage_account_url": settings.BLOB_STORAGE_ACCOUNT_URL,
-            "blob_container": settings.BLOB_CONTAINER,
-            "blob_name": blob_client.blob_name,
-        }
-        return absolute_inspection_path
-
-    def _get_blob_service_client(self) -> BlobServiceClient:
-        try:
-            return BlobServiceClient.from_connection_string(
-                self.storage_connection_string
-            )
-        except Exception as e:
-            self.logger.error(f"Unable to retrieve blob service client. Error: {e}")
-            raise e
-
-    def _get_container_client(
-        self, blob_service_client: BlobServiceClient
-    ) -> ContainerClient:
-        return blob_service_client.get_container_client(self.container_name)
-
-    def _get_blob_client(self, path_to_blob: Path) -> BlobClient:
-        return self.container_client.get_blob_client(path_to_blob.as_posix())
+        return BlobStoragePath(
+            storage_account=settings.BLOB_STORAGE_ACCOUNT,
+            blob_container=settings.BLOB_CONTAINER,
+            blob_name=blob_client.blob_name,
+        )

isar/storage/local_storage.py

@@ -2,9 +2,14 @@ import logging
 from pathlib import Path
 
 from isar.config.settings import settings
-from isar.storage.storage_interface import StorageException, StorageInterface
+from isar.storage.storage_interface import (
+    LocalStoragePath,
+    StorageException,
+    StorageInterface,
+    StoragePaths,
+)
 from isar.storage.utilities import construct_metadata_file, construct_paths
-from robot_interface.models.inspection.inspection import Inspection
+from robot_interface.models.inspection.inspection import InspectionBlob
 from robot_interface.models.mission.mission import Mission
 
 
@@ -13,30 +18,35 @@ class LocalStorage(StorageInterface):
         self.root_folder: Path = Path(settings.LOCAL_STORAGE_PATH)
         self.logger = logging.getLogger("uploader")
 
-    def store(self, inspection: Inspection, mission: Mission) -> str:
-        local_path, local_metadata_path = construct_paths(
+    def store(
+        self, inspection: InspectionBlob, mission: Mission
+    ) -> StoragePaths[LocalStoragePath]:
+        if inspection.data is None:
+            raise StorageException("Nothing to store. The inspection data is empty")
+
+        local_filename, local_metadata_filename = construct_paths(
             inspection=inspection, mission=mission
         )
 
-        absolute_path: Path = self.root_folder.joinpath(local_path)
-        absolute_metadata_path: Path = self.root_folder.joinpath(local_metadata_path)
+        data_path: Path = self.root_folder.joinpath(local_filename)
+        metadata_path: Path = self.root_folder.joinpath(local_metadata_filename)
 
-        absolute_path.parent.mkdir(parents=True, exist_ok=True)
+        data_path.parent.mkdir(parents=True, exist_ok=True)
 
         metadata_bytes: bytes = construct_metadata_file(
-            inspection=inspection, mission=mission, filename=local_path.name
+            inspection=inspection, mission=mission, filename=local_filename.name
         )
         try:
             with (
-                open(absolute_path, "wb") as file,
-                open(absolute_metadata_path, "wb") as metadata_file,
+                open(data_path, "wb") as file,
+                open(metadata_path, "wb") as metadata_file,
             ):
                 file.write(inspection.data)
                 metadata_file.write(metadata_bytes)
         except IOError as e:
             self.logger.warning(
                 f"Failed open/write for one of the following files: \n"
-                f"{absolute_path}\n{absolute_metadata_path}"
+                f"{data_path}\n{metadata_path}"
             )
             raise StorageException from e
         except Exception as e:
@@ -44,4 +54,7 @@ class LocalStorage(StorageInterface):
                 "An unexpected error occurred while writing to local storage"
             )
             raise StorageException from e
-        return str(absolute_path)
+        return StoragePaths(
+            data_path=LocalStoragePath(file_path=data_path),
+            metadata_path=LocalStoragePath(file_path=metadata_path),
+        )

isar/storage/storage_interface.py

@@ -1,25 +1,46 @@
 from abc import ABCMeta, abstractmethod
-from typing import Union
+from pathlib import Path
+from typing import Generic, TypeVar
 
-from robot_interface.models.inspection.inspection import Inspection
+from pydantic import BaseModel
+
+from robot_interface.models.inspection.inspection import InspectionBlob
 from robot_interface.models.mission.mission import Mission
 
 
+class BlobStoragePath(BaseModel):
+    storage_account: str
+    blob_container: str
+    blob_name: str
+
+
+class LocalStoragePath(BaseModel):
+    file_path: Path
+
+
+TPath = TypeVar("TPath", BlobStoragePath, LocalStoragePath)
+
+
+class StoragePaths(BaseModel, Generic[TPath]):
+    data_path: TPath
+    metadata_path: TPath
+
+
 class StorageInterface(metaclass=ABCMeta):
     @abstractmethod
-    def store(self, inspection: Inspection, mission: Mission) -> Union[str, dict]:
+    def store(self, inspection: InspectionBlob, mission: Mission) -> StoragePaths:
         """
         Parameters
         ----------
+        inspection : InspectionBlob
+            The inspection object to be stored.
         mission : Mission
             Mission the inspection is a part of.
-        inspection : Inspection
-            The inspection object to be stored.
 
         Returns
         ----------
-        String
-            Path of the saved inspection
+        StoragePaths
+            Paths to the data and metadata
 
         Raises
         ----------

isar/storage/uploader.py

@@ -1,33 +1,64 @@
 import json
 import logging
 from dataclasses import dataclass
-from datetime import UTC, datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from queue import Empty, Queue
-from typing import List, Union
-
-from injector import inject
+from threading import Event
+from typing import List
 
 from isar.config.settings import settings
-from isar.models.communication.queues import Queues
-from isar.storage.storage_interface import StorageException, StorageInterface
-from robot_interface.models.inspection.inspection import Inspection
+from isar.models.events import Events
+from isar.storage.storage_interface import (
+    BlobStoragePath,
+    LocalStoragePath,
+    StorageException,
+    StorageInterface,
+    StoragePaths,
+)
+from robot_interface.models.inspection.inspection import (
+    Inspection,
+    InspectionBlob,
+    InspectionValue,
+)
 from robot_interface.models.mission.mission import Mission
 from robot_interface.telemetry.mqtt_client import MqttClientInterface
+from robot_interface.telemetry.payloads import (
+    InspectionResultPayload,
+    InspectionValuePayload,
+)
 from robot_interface.utilities.json_service import EnhancedJSONEncoder
 
 
+def has_empty_blob_storage_path(storage_paths: StoragePaths):
+    for path in (storage_paths.data_path, storage_paths.metadata_path):
+        for value in (path.storage_account, path.blob_container, path.blob_name):
+            if not (value and value.strip()):
+                return True
+    return False
+
+
 @dataclass
 class UploaderQueueItem:
     inspection: Inspection
     mission: Mission
+
+
+@dataclass
+class ValueItem(UploaderQueueItem):
+    inspection: InspectionValue
+
+
+@dataclass
+class BlobItem(UploaderQueueItem):
+    inspection: InspectionBlob
     storage_handler: StorageInterface
     _retry_count: int
-    _next_retry_time: datetime = datetime.now(UTC)
+    _next_retry_time: datetime = datetime.now(timezone.utc)
 
     def increment_retry(self, max_wait_time: int) -> None:
         self._retry_count += 1
         seconds_until_retry: int = min(2**self._retry_count, max_wait_time)
-        self._next_retry_time = datetime.now(UTC) + timedelta(
+        self._next_retry_time = datetime.now(timezone.utc) + timedelta(
             seconds=seconds_until_retry
         )
 
@@ -35,17 +66,18 @@ class UploaderQueueItem:
         return self._retry_count
 
     def is_ready_for_upload(self) -> bool:
-        return datetime.now(UTC) >= self._next_retry_time
+        return datetime.now(timezone.utc) >= self._next_retry_time
 
     def seconds_until_retry(self) -> int:
-        return max(0, int((self._next_retry_time - datetime.now(UTC)).total_seconds()))
+        return max(
+            0, int((self._next_retry_time - datetime.now(timezone.utc)).total_seconds())
+        )
 
 
 class Uploader:
-    @inject
     def __init__(
         self,
-        queues: Queues,
+        events: Events,
         storage_handlers: List[StorageInterface],
         mqtt_publisher: MqttClientInterface,
         max_wait_time: int = settings.UPLOAD_FAILURE_MAX_WAIT,
@@ -55,8 +87,8 @@ class Uploader:
 
         Parameters
         ----------
-        queues : Queues
-            Queues used for cross-thread communication.
+        events : Events
+            Events used for cross-thread communication.
         storage_handlers : List[StorageInterface]
             List of handlers for different upload options
         max_wait_time : float
@@ -64,7 +96,7 @@ class Uploader:
         max_retry_attempts : int
             Maximum attempts to retry an upload when it fails
         """
-        self.upload_queue: Queue = queues.upload_queue
+        self.upload_queue: Queue = events.upload_queue
         self.storage_handlers: List[StorageInterface] = storage_handlers
         self.mqtt_publisher = mqtt_publisher
 
@@ -72,11 +104,16 @@ class Uploader:
         self.max_retry_attempts = max_retry_attempts
         self._internal_upload_queue: List[UploaderQueueItem] = []
 
+        self.signal_thread_quitting: Event = Event()
+
         self.logger = logging.getLogger("uploader")
 
+    def stop(self) -> None:
+        self.signal_thread_quitting.set()
+
     def run(self) -> None:
         self.logger.info("Started uploader")
-        while True:
+        while not self.signal_thread_quitting.wait(0):
             inspection: Inspection
             mission: Mission
             try:
@@ -91,76 +128,158 @@ class Uploader:
                     )
                     continue
 
-                # If new item from thread queue, add one per handler to internal queue:
-                for storage_handler in self.storage_handlers:
-                    new_item: UploaderQueueItem = UploaderQueueItem(
-                        inspection, mission, storage_handler, _retry_count=-1
-                    )
+                new_item: UploaderQueueItem
+                if isinstance(inspection, InspectionValue):
+                    new_item = ValueItem(inspection, mission)
                     self._internal_upload_queue.append(new_item)
+
+                elif isinstance(inspection, InspectionBlob):
+                    # If new item from thread queue, add one per handler to internal queue:
+                    for storage_handler in self.storage_handlers:
+                        new_item = BlobItem(
+                            inspection, mission, storage_handler, _retry_count=-1
+                        )
+                        self._internal_upload_queue.append(new_item)
+                else:
+                    self.logger.warning(
+                        f"Unable to add UploaderQueueItem as its type {type(inspection).__name__} is unsupported"
+                    )
             except Empty:
                 continue
+            except Exception as e:
+                self.logger.error(f"Unexpected error in uploader thread: {e}")
+                continue
 
-    def _upload(self, upload_item: UploaderQueueItem) -> Union[str, dict]:
-        inspection_path: Union[str, dict] = ""
+    def _upload(self, item: BlobItem) -> StoragePaths:
+        inspection_paths: StoragePaths
         try:
-            inspection_path = upload_item.storage_handler.store(
-                inspection=upload_item.inspection, mission=upload_item.mission
+            inspection_paths = item.storage_handler.store(
+                inspection=item.inspection, mission=item.mission
             )
             self.logger.info(
-                f"Storage handler: {type(upload_item.storage_handler).__name__} "
-                f"uploaded inspection {str(upload_item.inspection.id)[:8]}"
+                f"Storage handler: {type(item.storage_handler).__name__} "
+                f"uploaded inspection {str(item.inspection.id)[:8]}"
             )
-            self._internal_upload_queue.remove(upload_item)
-        except StorageException:
-            if upload_item.get_retry_count() < self.max_retry_attempts:
-                upload_item.increment_retry(self.max_wait_time)
+            self._internal_upload_queue.remove(item)
+        except StorageException as e:
+            if item.get_retry_count() < self.max_retry_attempts:
+                item.increment_retry(self.max_wait_time)
                 self.logger.warning(
-                    f"Storage handler: {type(upload_item.storage_handler).__name__} "
+                    f"Storage handler: {type(item.storage_handler).__name__} "
                     f"failed to upload inspection: "
-                    f"{str(upload_item.inspection.id)[:8]}. "
-                    f"Retrying in {upload_item.seconds_until_retry()}s."
+                    f"{str(item.inspection.id)[:8]}. "
+                    f"Retrying in {item.seconds_until_retry()}s."
                 )
             else:
-                self._internal_upload_queue.remove(upload_item)
                 self.logger.error(
-                    f"Storage handler: {type(upload_item.storage_handler).__name__} "
+                    f"Storage handler: {type(item.storage_handler).__name__} "
                     f"exceeded max retries to upload inspection: "
-                    f"{str(upload_item.inspection.id)[:8]}. Aborting upload."
+                    f"{str(item.inspection.id)[:8]}. Aborting upload."
                 )
-        return inspection_path
+                self._internal_upload_queue.remove(item)
+            raise e
+        return inspection_paths
 
     def _process_upload_queue(self) -> None:
+        def should_upload(_item):
+            if isinstance(_item, ValueItem):
+                return True
+            if _item.is_ready_for_upload():
+                return True
+            return False
+
         ready_items: List[UploaderQueueItem] = [
-            item for item in self._internal_upload_queue if item.is_ready_for_upload()
+            item for item in self._internal_upload_queue if should_upload(item)
         ]
         for item in ready_items:
-            inspection_path = self._upload(item)
-            self._publish_inspection_result(
-                inspection=item.inspection, inspection_path=inspection_path
+            if isinstance(item, ValueItem):
+                self._publish_inspection_value(item.inspection)
+                self.logger.info(
+                    f"Published value for inspection {str(item.inspection.id)[:8]}"
+                )
+                self._internal_upload_queue.remove(item)
+            elif isinstance(item, BlobItem):
+                try:
+                    inspection_paths = self._upload(item)
+                    if isinstance(inspection_paths.data_path, LocalStoragePath):
+                        self.logger.info("Skipping publishing when using local storage")
+                    elif isinstance(
+                        inspection_paths.data_path, BlobStoragePath
+                    ) and has_empty_blob_storage_path(inspection_paths):
+                        self.logger.warning(
+                            "Skipping publishing: Blob storage paths are empty for inspection %s",
+                            str(item.inspection.id)[:8],
+                        )
+                    else:
+                        self._publish_inspection_result(
+                            inspection=item.inspection,
+                            inspection_paths=inspection_paths,
+                        )
+                except StorageException:
+                    pass
+            else:
+                self.logger.warning(
+                    f"Unable to process upload item as its type {type(item).__name__} is not supported"
+                )
+
+    def _publish_inspection_value(self, inspection: InspectionValue) -> None:
+        if not self.mqtt_publisher:
+            return
+
+        if not isinstance(inspection, InspectionValue):
+            logging.warning(
+                f"Excpected type InspectionValue but got {type(inspection).__name__} instead"
             )
+            return
+
+        payload: InspectionValuePayload = InspectionValuePayload(
+            isar_id=settings.ISAR_ID,
+            robot_name=settings.ROBOT_NAME,
+            inspection_id=inspection.id,
+            installation_code=settings.PLANT_SHORT_NAME,
+            tag_id=inspection.metadata.tag_id,
+            inspection_type=type(inspection).__name__,
+            inspection_description=inspection.metadata.inspection_description,
+            value=inspection.value,
+            unit=inspection.unit,
+            x=inspection.metadata.robot_pose.position.x,
+            y=inspection.metadata.robot_pose.position.y,
+            z=inspection.metadata.robot_pose.position.z,
+            timestamp=inspection.metadata.start_time,
+        )
+        self.mqtt_publisher.publish(
+            topic=settings.TOPIC_ISAR_INSPECTION_VALUE,
+            payload=json.dumps(payload, cls=EnhancedJSONEncoder),
+            qos=1,
+            retain=True,
+        )
 
     def _publish_inspection_result(
-        self, inspection: Inspection, inspection_path: Union[str, dict]
+        self,
+        inspection: InspectionBlob,
+        inspection_paths: StoragePaths[BlobStoragePath],
     ) -> None:
         """Publishes the reference of the inspection result to the MQTT Broker
         along with the analysis type
         """
         if not self.mqtt_publisher:
             return
-        payload: str = json.dumps(
-            {
-                "isar_id": settings.ISAR_ID,
-                "robot_name": settings.ROBOT_NAME,
-                "inspection_id": inspection.id,
-                "inspection_path": inspection_path,
-                "analysis_type": inspection.metadata.analysis_type,
-                "timestamp": datetime.now(UTC),
-            },
-            cls=EnhancedJSONEncoder,
+
+        payload: InspectionResultPayload = InspectionResultPayload(
+            isar_id=settings.ISAR_ID,
+            robot_name=settings.ROBOT_NAME,
+            inspection_id=inspection.id,
+            blob_storage_data_path=inspection_paths.data_path,
+            blob_storage_metadata_path=inspection_paths.metadata_path,
+            installation_code=settings.PLANT_SHORT_NAME,
+            tag_id=inspection.metadata.tag_id,
+            inspection_type=type(inspection).__name__,
+            inspection_description=inspection.metadata.inspection_description,
+            timestamp=inspection.metadata.start_time,
         )
         self.mqtt_publisher.publish(
             topic=settings.TOPIC_ISAR_INSPECTION_RESULT,
-            payload=payload,
+            payload=json.dumps(payload, cls=EnhancedJSONEncoder),
             qos=1,
             retain=True,
         )