isar 1.15.0__py3-none-any.whl → 1.34.9__py3-none-any.whl

isar/storage/uploader.py

@@ -1,33 +1,64 @@
 import json
 import logging
 from dataclasses import dataclass
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from queue import Empty, Queue
+from threading import Event
 from typing import List
 
-from injector import inject
-
 from isar.config.settings import settings
-from isar.models.communication.queues import Queues
+from isar.models.events import Events
+from isar.storage.storage_interface import (
+    BlobStoragePath,
+    LocalStoragePath,
+    StorageException,
+    StorageInterface,
+    StoragePaths,
+)
+from robot_interface.models.inspection.inspection import (
+    Inspection,
+    InspectionBlob,
+    InspectionValue,
+)
 from robot_interface.models.mission.mission import Mission
-from isar.storage.storage_interface import StorageException, StorageInterface
-from robot_interface.models.inspection.inspection import Inspection
 from robot_interface.telemetry.mqtt_client import MqttClientInterface
+from robot_interface.telemetry.payloads import (
+    InspectionResultPayload,
+    InspectionValuePayload,
+)
 from robot_interface.utilities.json_service import EnhancedJSONEncoder
 
 
+def has_empty_blob_storage_path(storage_paths: StoragePaths):
+    for path in (storage_paths.data_path, storage_paths.metadata_path):
+        for value in (path.storage_account, path.blob_container, path.blob_name):
+            if not (value and value.strip()):
+                return True
+    return False
+
+
 @dataclass
 class UploaderQueueItem:
     inspection: Inspection
     mission: Mission
+
+
+@dataclass
+class ValueItem(UploaderQueueItem):
+    inspection: InspectionValue
+
+
+@dataclass
+class BlobItem(UploaderQueueItem):
+    inspection: InspectionBlob
     storage_handler: StorageInterface
     _retry_count: int
-    _next_retry_time: datetime = datetime.utcnow()
+    _next_retry_time: datetime = datetime.now(timezone.utc)
 
     def increment_retry(self, max_wait_time: int) -> None:
         self._retry_count += 1
         seconds_until_retry: int = min(2**self._retry_count, max_wait_time)
-        self._next_retry_time = datetime.utcnow() + timedelta(
+        self._next_retry_time = datetime.now(timezone.utc) + timedelta(
             seconds=seconds_until_retry
         )
 
@@ -35,17 +66,18 @@ class UploaderQueueItem:
         return self._retry_count
 
     def is_ready_for_upload(self) -> bool:
-        return datetime.utcnow() >= self._next_retry_time
+        return datetime.now(timezone.utc) >= self._next_retry_time
 
     def seconds_until_retry(self) -> int:
-        return max(0, int((self._next_retry_time - datetime.utcnow()).total_seconds()))
+        return max(
+            0, int((self._next_retry_time - datetime.now(timezone.utc)).total_seconds())
+        )
 
 
 class Uploader:
-    @inject
     def __init__(
         self,
-        queues: Queues,
+        events: Events,
         storage_handlers: List[StorageInterface],
         mqtt_publisher: MqttClientInterface,
         max_wait_time: int = settings.UPLOAD_FAILURE_MAX_WAIT,
@@ -55,8 +87,8 @@ class Uploader:
 
         Parameters
         ----------
-        queues : Queues
-            Queues used for cross-thread communication.
+        events : Events
+            Events used for cross-thread communication.
         storage_handlers : List[StorageInterface]
             List of handlers for different upload options
         max_wait_time : float
@@ -64,7 +96,7 @@ class Uploader:
         max_retry_attempts : int
             Maximum attempts to retry an upload when it fails
         """
-        self.upload_queue: Queue = queues.upload_queue
+        self.upload_queue: Queue = events.upload_queue
         self.storage_handlers: List[StorageInterface] = storage_handlers
         self.mqtt_publisher = mqtt_publisher
 
@@ -72,11 +104,16 @@ class Uploader:
         self.max_retry_attempts = max_retry_attempts
         self._internal_upload_queue: List[UploaderQueueItem] = []
 
+        self.signal_thread_quitting: Event = Event()
+
         self.logger = logging.getLogger("uploader")
 
+    def stop(self) -> None:
+        self.signal_thread_quitting.set()
+
     def run(self) -> None:
         self.logger.info("Started uploader")
-        while True:
+        while not self.signal_thread_quitting.wait(0):
             inspection: Inspection
             mission: Mission
             try:
@@ -85,72 +122,164 @@ class Uploader:
 
                 inspection, mission = self.upload_queue.get(timeout=1)
 
-                # If new item from thread queue, add one per handler to internal queue:
-                for storage_handler in self.storage_handlers:
-                    new_item: UploaderQueueItem = UploaderQueueItem(
-                        inspection, mission, storage_handler, _retry_count=-1
+                if not mission:
+                    self.logger.warning(
+                        "Failed to upload missing mission from upload queue"
                     )
+                    continue
+
+                new_item: UploaderQueueItem
+                if isinstance(inspection, InspectionValue):
+                    new_item = ValueItem(inspection, mission)
                     self._internal_upload_queue.append(new_item)
+
+                elif isinstance(inspection, InspectionBlob):
+                    # If new item from thread queue, add one per handler to internal queue:
+                    for storage_handler in self.storage_handlers:
+                        new_item = BlobItem(
+                            inspection, mission, storage_handler, _retry_count=-1
+                        )
+                        self._internal_upload_queue.append(new_item)
+                else:
+                    self.logger.warning(
+                        f"Unable to add UploaderQueueItem as its type {type(inspection).__name__} is unsupported"
+                    )
             except Empty:
                 continue
+            except Exception as e:
+                self.logger.error(f"Unexpected error in uploader thread: {e}")
+                continue
 
-    def _upload(self, upload_item: UploaderQueueItem) -> str:
-        inspection_path = ""
+    def _upload(self, item: BlobItem) -> StoragePaths:
+        inspection_paths: StoragePaths
         try:
-            inspection_path = upload_item.storage_handler.store(
-                inspection=upload_item.inspection, mission=upload_item.mission
+            inspection_paths = item.storage_handler.store(
+                inspection=item.inspection, mission=item.mission
             )
             self.logger.info(
-                f"Storage handler: {type(upload_item.storage_handler).__name__} "
-                f"uploaded inspection {str(upload_item.inspection.id)[:8]}"
+                f"Storage handler: {type(item.storage_handler).__name__} "
+                f"uploaded inspection {str(item.inspection.id)[:8]}"
             )
-            self._internal_upload_queue.remove(upload_item)
-        except StorageException:
-            if upload_item.get_retry_count() < self.max_retry_attempts:
-                upload_item.increment_retry(self.max_wait_time)
+            self._internal_upload_queue.remove(item)
+        except StorageException as e:
+            if item.get_retry_count() < self.max_retry_attempts:
+                item.increment_retry(self.max_wait_time)
                 self.logger.warning(
-                    f"Storage handler: {type(upload_item.storage_handler).__name__} "
+                    f"Storage handler: {type(item.storage_handler).__name__} "
                     f"failed to upload inspection: "
-                    f"{str(upload_item.inspection.id)[:8]}. "
-                    f"Retrying in {upload_item.seconds_until_retry()}s."
+                    f"{str(item.inspection.id)[:8]}. "
+                    f"Retrying in {item.seconds_until_retry()}s."
                 )
             else:
-                self._internal_upload_queue.remove(upload_item)
                 self.logger.error(
-                    f"Storage handler: {type(upload_item.storage_handler).__name__} "
+                    f"Storage handler: {type(item.storage_handler).__name__} "
                     f"exceeded max retries to upload inspection: "
-                    f"{str(upload_item.inspection.id)[:8]}. Aborting upload."
+                    f"{str(item.inspection.id)[:8]}. Aborting upload."
                 )
-        return inspection_path
+                self._internal_upload_queue.remove(item)
+            raise e
+        return inspection_paths
 
     def _process_upload_queue(self) -> None:
+        def should_upload(_item):
+            if isinstance(_item, ValueItem):
+                return True
+            if _item.is_ready_for_upload():
+                return True
+            return False
+
         ready_items: List[UploaderQueueItem] = [
-            x for x in self._internal_upload_queue if x.is_ready_for_upload()
+            item for item in self._internal_upload_queue if should_upload(item)
         ]
         for item in ready_items:
-            inspection_path = self._upload(item)
-            self._publish_inspection_path(
-                inspection=item.inspection, inspection_path=inspection_path
+            if isinstance(item, ValueItem):
+                self._publish_inspection_value(item.inspection)
+                self.logger.info(
+                    f"Published value for inspection {str(item.inspection.id)[:8]}"
+                )
+                self._internal_upload_queue.remove(item)
+            elif isinstance(item, BlobItem):
+                try:
+                    inspection_paths = self._upload(item)
+                    if isinstance(inspection_paths.data_path, LocalStoragePath):
+                        self.logger.info("Skipping publishing when using local storage")
+                    elif isinstance(
+                        inspection_paths.data_path, BlobStoragePath
+                    ) and has_empty_blob_storage_path(inspection_paths):
+                        self.logger.warning(
+                            "Skipping publishing: Blob storage paths are empty for inspection %s",
+                            str(item.inspection.id)[:8],
+                        )
+                    else:
+                        self._publish_inspection_result(
+                            inspection=item.inspection,
+                            inspection_paths=inspection_paths,
+                        )
+                except StorageException:
+                    pass
+            else:
+                self.logger.warning(
+                    f"Unable to process upload item as its type {type(item).__name__} is not supported"
+                )
+
+    def _publish_inspection_value(self, inspection: InspectionValue) -> None:
+        if not self.mqtt_publisher:
+            return
+
+        if not isinstance(inspection, InspectionValue):
+            logging.warning(
+                f"Excpected type InspectionValue but got {type(inspection).__name__} instead"
             )
+            return
+
+        payload: InspectionValuePayload = InspectionValuePayload(
+            isar_id=settings.ISAR_ID,
+            robot_name=settings.ROBOT_NAME,
+            inspection_id=inspection.id,
+            installation_code=settings.PLANT_SHORT_NAME,
+            tag_id=inspection.metadata.tag_id,
+            inspection_type=type(inspection).__name__,
+            inspection_description=inspection.metadata.inspection_description,
+            value=inspection.value,
+            unit=inspection.unit,
+            x=inspection.metadata.robot_pose.position.x,
+            y=inspection.metadata.robot_pose.position.y,
+            z=inspection.metadata.robot_pose.position.z,
+            timestamp=inspection.metadata.start_time,
+        )
+        self.mqtt_publisher.publish(
+            topic=settings.TOPIC_ISAR_INSPECTION_VALUE,
+            payload=json.dumps(payload, cls=EnhancedJSONEncoder),
+            qos=1,
+            retain=True,
+        )
 
-    def _publish_inspection_path(
-        self, inspection: Inspection, inspection_path: str
+    def _publish_inspection_result(
+        self,
+        inspection: InspectionBlob,
+        inspection_paths: StoragePaths[BlobStoragePath],
     ) -> None:
-        """Publishes the image url to the MQTT Broker"""
+        """Publishes the reference of the inspection result to the MQTT Broker
+        along with the analysis type
+        """
         if not self.mqtt_publisher:
             return
-        payload: str = json.dumps(
-            {
-                "isar_id": settings.ISAR_ID,
-                "robot_name": settings.ROBOT_NAME,
-                "step_id": inspection.id,
-                "inspection_path": inspection_path,
-                "timestamp": datetime.utcnow(),
-            },
-            cls=EnhancedJSONEncoder,
+
+        payload: InspectionResultPayload = InspectionResultPayload(
+            isar_id=settings.ISAR_ID,
+            robot_name=settings.ROBOT_NAME,
+            inspection_id=inspection.id,
+            blob_storage_data_path=inspection_paths.data_path,
+            blob_storage_metadata_path=inspection_paths.metadata_path,
+            installation_code=settings.PLANT_SHORT_NAME,
+            tag_id=inspection.metadata.tag_id,
+            inspection_type=type(inspection).__name__,
+            inspection_description=inspection.metadata.inspection_description,
+            timestamp=inspection.metadata.start_time,
         )
         self.mqtt_publisher.publish(
             topic=settings.TOPIC_ISAR_INSPECTION_RESULT,
-            payload=payload,
-            retain=False,
+            payload=json.dumps(payload, cls=EnhancedJSONEncoder),
+            qos=1,
+            retain=True,
         )

isar/storage/utilities.py

@@ -1,15 +1,13 @@
 import json
-import time
+from datetime import datetime, timezone
 from pathlib import Path
 from typing import Tuple
 
-from robot_interface.models.mission.mission import Mission
+from isar.config.settings import settings
 from robot_interface.models.inspection.inspection import Inspection
+from robot_interface.models.mission.mission import Mission
 from robot_interface.utilities.json_service import EnhancedJSONEncoder
 
-from isar.config.settings import settings
-from datetime import date, datetime
-
 
 def construct_paths(inspection: Inspection, mission: Mission) -> Tuple[Path, Path]:
     folder: Path = Path(get_foldername(mission=mission))
@@ -35,29 +33,34 @@ def construct_metadata_file(
         "plant_code": settings.PLANT_CODE,
         "media_orientation_reference_system": settings.MEDIA_ORIENTATION_REFERENCE_SYSTEM,  # noqa: E501
         "additional_meta": {
+            "inspection_id": inspection.id,
             "mission_id": mission.id,
             "mission_name": mission.name,
+            "mission_date": datetime.now(timezone.utc).date(),
             "plant_name": settings.PLANT_NAME,
-            "mission_date": datetime.utcnow().date(),
             "isar_id": settings.ISAR_ID,
             "robot_name": settings.ROBOT_NAME,
+            "inspection_description": inspection.metadata.inspection_description,
+            "tag": inspection.metadata.tag_id,
+            "robot_pose": {
+                "position": {
+                    "x": inspection.metadata.robot_pose.position.x,
+                    "y": inspection.metadata.robot_pose.position.y,
+                    "z": inspection.metadata.robot_pose.position.z,
+                },
+                "orientation": inspection.metadata.robot_pose.orientation.to_quat_array(),
+            },
+            "target_position": {
+                "x": inspection.metadata.target_position.x,
+                "y": inspection.metadata.target_position.y,
+                "z": inspection.metadata.target_position.z,
+            },
+            "timestamp": inspection.metadata.start_time,
         },
-        "data": [
+        "data_files": [
             {
                 "folder": f"/{get_foldername(mission=mission)}",
-                "files": [
-                    {
-                        "file_name": filename,
-                        "timestamp": inspection.metadata.start_time,
-                        "x": inspection.metadata.pose.position.x,
-                        "y": inspection.metadata.pose.position.y,
-                        "z": inspection.metadata.pose.position.z,
-                        "tag": inspection.metadata.tag_id,
-                        "additional_media_metadata": {
-                            "orientation": inspection.metadata.pose.orientation.to_quat_array()  # noqa: E501
-                        },
-                    }
-                ],
+                "file_name": filename,
             }
         ],
     }
@@ -65,14 +68,19 @@ def construct_metadata_file(
     return json.dumps(data, cls=EnhancedJSONEncoder, indent=4).encode()
 
 
-def get_filename(
-    inspection: Inspection,
-) -> str:
-    inspection_type: str = type(inspection).__name__
+def get_filename(inspection: Inspection) -> str:
+    utc_time: str = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
     tag: str = inspection.metadata.tag_id if inspection.metadata.tag_id else "no-tag"
-    epoch_time: int = int(time.time())
-    return f"{tag}__{inspection_type}__{epoch_time}"
+    inspection_type: str = type(inspection).__name__
+    inspection_description: str = (
+        inspection.metadata.inspection_description.replace(" ", "-")
+        if inspection.metadata.inspection_description
+        else "NA"
+    )
+    return f"{tag}__{inspection_type}__{inspection_description}__{utc_time}"
 
 
 def get_foldername(mission: Mission) -> str:
-    return f"{datetime.utcnow().date()}__{settings.PLANT_SHORT_NAME}__{mission.name}__{mission.id}"
+    utc_date: str = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+    mission_name: str = mission.name.replace(" ", "-")
+    return f"{utc_date}__{settings.PLANT_SHORT_NAME}__{mission_name}__{mission.id}"