isar 1.33.1__tar.gz → 1.33.2__tar.gz

{isar-1.33.1 → isar-1.33.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: isar
-Version: 1.33.1
+Version: 1.33.2
 Summary: Integration and Supervisory control of Autonomous Robots
 Author-email: Equinor ASA <fg_robots_dev@equinor.com>
 License: Eclipse Public License version 2.0

{isar-1.33.1 → isar-1.33.2}/src/isar/config/settings.py

@@ -244,6 +244,11 @@ class Settings(BaseSettings):
         default="intervention_needed", validate_default=True
     )
 
+    # List of MQTT Topics Expiry
+    MQTT_ROBOT_HEARTBEAT_EXPIRY: int = Field(default=5)
+    MQTT_TELEMETRY_EXPIRY: int = Field(default=10)
+    MQTT_MISSION_AND_TASK_EXPIRY: int = Field(default=86400)
+
     # Logging
 
     #   Log handlers

{isar-1.33.1 → isar-1.33.2}/src/isar/services/service_connections/mqtt/mqtt_client.py

@@ -1,15 +1,24 @@
 import logging
 import os
+import time
 from queue import Empty, Queue
 
 import backoff
 from paho.mqtt import client as mqtt
 from paho.mqtt.client import Client
+from paho.mqtt.packettypes import PacketTypes
+from paho.mqtt.properties import Properties
 
 from isar.config.settings import settings
 from robot_interface.telemetry.mqtt_client import MqttClientInterface
 
 
+def props_expiry(seconds: int) -> Properties:
+    p = Properties(PacketTypes.PUBLISH)
+    p.MessageExpiryInterval = seconds
+    return p
+
+
 def _on_success(data: dict) -> None:
     logging.getLogger("mqtt_client").info("Connected to MQTT Broker")
     logging.getLogger("mqtt_client").debug(
@@ -53,7 +62,9 @@ class MqttClient(MqttClientInterface):
 
         self.port: int = settings.MQTT_PORT
 
-        self.client: Client = Client(mqtt.CallbackAPIVersion.VERSION1)
+        self.client: Client = Client(
+            protocol=mqtt.MQTTv5, callback_api_version=mqtt.CallbackAPIVersion.VERSION2
+        )
 
         self.client.enable_logger(logger=self.logger)
 
@@ -74,20 +85,36 @@ class MqttClient(MqttClientInterface):
 
         while True:
             if not self.client.is_connected():
+                time.sleep(0)  # avoid CPU spin
                 continue
             try:
-                topic, payload, qos, retain = self.mqtt_queue.get(timeout=1)
+                item = self.mqtt_queue.get(timeout=1)
+                if len(item) == 4:
+                    topic, payload, qos, retain = item
+                    properties = None
+                else:
+                    topic, payload, qos, retain, properties = item
             except Empty:
                 continue
 
-            self.publish(topic=topic, payload=payload, qos=qos, retain=retain)
+            self.publish(
+                topic=topic,
+                payload=payload,
+                qos=qos,
+                retain=retain,
+                properties=properties,
+            )
 
-    def on_connect(self, client, userdata, flags, rc):
-        self.logger.info("Connection returned result: " + mqtt.connack_string(rc))
+    def on_connect(self, client, userdata, flags, reason_code, properties):
+        self.logger.info(
+            "Connection returned result: " + mqtt.connack_string(reason_code)
+        )
 
-    def on_disconnect(self, client, userdata, rc):
-        if rc != 0:
-            self.logger.warning("Unexpected disconnection from MQTT Broker")
+    def on_disconnect(self, client, userdata, reason_code, properties):
+        if reason_code != 0:
+            self.logger.warning(
+                f"Unexpected disconnection from MQTT Broker, {reason_code}"
+            )
 
     @backoff.on_exception(
         backoff.expo,
@@ -102,6 +129,15 @@ class MqttClient(MqttClientInterface):
         self.logger.info("Host: %s, Port: %s", host, port)
         self.client.connect(host=host, port=port)
 
-    def publish(self, topic: str, payload: str, qos: int = 0, retain: bool = False):
+    def publish(
+        self,
+        topic: str,
+        payload: str,
+        qos: int = 0,
+        retain: bool = False,
+        properties=None,
+    ):
         self.logger.debug("Publishing message to topic: %s", topic)
-        self.client.publish(topic=topic, payload=payload, qos=qos, retain=retain)
+        self.client.publish(
+            topic=topic, payload=payload, qos=qos, retain=retain, properties=properties
+        )

{isar-1.33.1 → isar-1.33.2}/src/isar/services/service_connections/mqtt/robot_heartbeat_publisher.py

@@ -4,6 +4,7 @@ from datetime import datetime, timezone
 from queue import Queue
 
 from isar.config.settings import settings
+from isar.services.service_connections.mqtt.mqtt_client import props_expiry
 from robot_interface.telemetry.mqtt_client import MqttPublisher
 from robot_interface.telemetry.payloads import RobotHeartbeatPayload
 from robot_interface.utilities.json_service import EnhancedJSONEncoder
@@ -24,6 +25,8 @@ class RobotHeartbeatPublisher:
             self.mqtt_publisher.publish(
                 topic=settings.TOPIC_ISAR_ROBOT_HEARTBEAT,
                 payload=json.dumps(payload, cls=EnhancedJSONEncoder),
+                retain=True,
+                properties=props_expiry(settings.MQTT_ROBOT_HEARTBEAT_EXPIRY),
             )
 
             time.sleep(settings.ROBOT_HEARTBEAT_PUBLISH_INTERVAL)

{isar-1.33.1 → isar-1.33.2}/src/isar/state_machine/state_machine.py

@@ -15,6 +15,7 @@ from isar.mission_planner.task_selector_interface import (
     TaskSelectorStop,
 )
 from isar.models.events import Events, SharedState
+from isar.services.service_connections.mqtt.mqtt_client import props_expiry
 from isar.state_machine.states.await_next_mission import AwaitNextMission
 from isar.state_machine.states.blocked_protective_stop import BlockedProtectiveStop
 from isar.state_machine.states.home import Home
@@ -285,10 +286,11 @@ class StateMachine(object):
         )
 
         self.mqtt_publisher.publish(
-            topic=settings.TOPIC_ISAR_MISSION,
+            topic=settings.TOPIC_ISAR_MISSION + f"/{self.current_mission.id}",
             payload=json.dumps(payload, cls=EnhancedJSONEncoder),
             qos=1,
             retain=True,
+            properties=props_expiry(settings.MQTT_MISSION_AND_TASK_EXPIRY),
         )
 
     def publish_task_status(self, task: TASKS) -> None:
@@ -316,10 +318,11 @@ class StateMachine(object):
         )
 
         self.mqtt_publisher.publish(
-            topic=settings.TOPIC_ISAR_TASK,
+            topic=settings.TOPIC_ISAR_TASK + f"/{task.id}",
             payload=json.dumps(payload, cls=EnhancedJSONEncoder),
             qos=1,
             retain=True,
+            properties=props_expiry(settings.MQTT_MISSION_AND_TASK_EXPIRY),
         )
 
     def publish_intervention_needed(self, error_message: str) -> None:

isar-1.33.2/src/isar/storage/blob_storage.py

@@ -0,0 +1,101 @@
+import logging
+from pathlib import Path
+
+from azure.core.exceptions import ResourceExistsError
+from azure.storage.blob import BlobServiceClient, ContainerClient
+
+from isar.config.keyvault.keyvault_service import Keyvault
+from isar.config.settings import settings
+from isar.storage.storage_interface import (
+    BlobStoragePath,
+    StorageException,
+    StorageInterface,
+    StoragePaths,
+)
+from isar.storage.utilities import construct_metadata_file, construct_paths
+from robot_interface.models.inspection.inspection import InspectionBlob
+from robot_interface.models.mission.mission import Mission
+
+
+class BlobStorage(StorageInterface):
+    def __init__(self, keyvault: Keyvault) -> None:
+        self.logger = logging.getLogger("uploader")
+
+        self.container_client_data = self._get_container_client(
+            keyvault, "AZURE-STORAGE-CONNECTION-STRING-DATA"
+        )
+        self.container_client_metadata = self._get_container_client(
+            keyvault, "AZURE-STORAGE-CONNECTION-STRING-METADATA"
+        )
+
+    def _get_container_client(self, keyvault: Keyvault, secret_name: str):
+        storage_connection_string = keyvault.get_secret(secret_name).value
+
+        if storage_connection_string is None:
+            raise RuntimeError(f"{secret_name} from keyvault is None")
+
+        try:
+            blob_service_client = BlobServiceClient.from_connection_string(
+                storage_connection_string
+            )
+        except Exception as e:
+            self.logger.error("Unable to retrieve blob service client. Error: %s", e)
+            raise e
+
+        container_client = blob_service_client.get_container_client(
+            settings.BLOB_CONTAINER
+        )
+
+        if not container_client.exists():
+            raise RuntimeError(
+                "The configured blob container %s does not exist",
+                settings.BLOB_CONTAINER,
+            )
+        return container_client
+
+    def store(
+        self, inspection: InspectionBlob, mission: Mission
+    ) -> StoragePaths[BlobStoragePath]:
+        if inspection.data is None:
+            raise StorageException("Nothing to store. The inspection data is empty")
+
+        data_filename, metadata_filename = construct_paths(
+            inspection=inspection, mission=mission
+        )
+
+        metadata_bytes: bytes = construct_metadata_file(
+            inspection=inspection, mission=mission, filename=data_filename.name
+        )
+
+        data_path = self._upload_file(
+            filename=data_filename,
+            data=inspection.data,
+            container_client=self.container_client_data,
+        )
+        metadata_path = self._upload_file(
+            filename=metadata_filename,
+            data=metadata_bytes,
+            container_client=self.container_client_metadata,
+        )
+        return StoragePaths(data_path=data_path, metadata_path=metadata_path)
+
+    def _upload_file(
+        self, filename: Path, data: bytes, container_client: ContainerClient
+    ) -> BlobStoragePath:
+        blob_client = container_client.get_blob_client(filename.as_posix())
+        try:
+            blob_client.upload_blob(data=data)
+        except ResourceExistsError as e:
+            self.logger.error(
+                "Blob %s already exists in container. Error: %s", filename.as_posix(), e
+            )
+            raise StorageException from e
+        except Exception as e:
+            self.logger.error("An unexpected error occurred while uploading blob")
+            raise StorageException from e
+
+        return BlobStoragePath(
+            storage_account=settings.BLOB_STORAGE_ACCOUNT,
+            blob_container=settings.BLOB_CONTAINER,
+            blob_name=blob_client.blob_name,
+        )

{isar-1.33.1 → isar-1.33.2}/src/isar/storage/local_storage.py

@@ -2,7 +2,12 @@ import logging
 from pathlib import Path
 
 from isar.config.settings import settings
-from isar.storage.storage_interface import StorageException, StorageInterface
+from isar.storage.storage_interface import (
+    LocalStoragePath,
+    StorageException,
+    StorageInterface,
+    StoragePaths,
+)
 from isar.storage.utilities import construct_metadata_file, construct_paths
 from robot_interface.models.inspection.inspection import InspectionBlob
 from robot_interface.models.mission.mission import Mission
@@ -13,33 +18,35 @@ class LocalStorage(StorageInterface):
         self.root_folder: Path = Path(settings.LOCAL_STORAGE_PATH)
         self.logger = logging.getLogger("uploader")
 
-    def store(self, inspection: InspectionBlob, mission: Mission) -> str:
+    def store(
+        self, inspection: InspectionBlob, mission: Mission
+    ) -> StoragePaths[LocalStoragePath]:
         if inspection.data is None:
             raise StorageException("Nothing to store. The inspection data is empty")
 
-        local_path, local_metadata_path = construct_paths(
+        local_filename, local_metadata_filename = construct_paths(
             inspection=inspection, mission=mission
         )
 
-        absolute_path: Path = self.root_folder.joinpath(local_path)
-        absolute_metadata_path: Path = self.root_folder.joinpath(local_metadata_path)
+        data_path: Path = self.root_folder.joinpath(local_filename)
+        metadata_path: Path = self.root_folder.joinpath(local_metadata_filename)
 
-        absolute_path.parent.mkdir(parents=True, exist_ok=True)
+        data_path.parent.mkdir(parents=True, exist_ok=True)
 
         metadata_bytes: bytes = construct_metadata_file(
-            inspection=inspection, mission=mission, filename=local_path.name
+            inspection=inspection, mission=mission, filename=local_filename.name
         )
         try:
             with (
-                open(absolute_path, "wb") as file,
-                open(absolute_metadata_path, "wb") as metadata_file,
+                open(data_path, "wb") as file,
+                open(metadata_path, "wb") as metadata_file,
             ):
                 file.write(inspection.data)
                 metadata_file.write(metadata_bytes)
         except IOError as e:
             self.logger.warning(
                 f"Failed open/write for one of the following files: \n"
-                f"{absolute_path}\n{absolute_metadata_path}"
+                f"{data_path}\n{metadata_path}"
             )
             raise StorageException from e
         except Exception as e:
@@ -47,4 +54,7 @@ class LocalStorage(StorageInterface):
                 "An unexpected error occurred while writing to local storage"
             )
             raise StorageException from e
-        return str(absolute_path)
+        return StoragePaths(
+            data_path=LocalStoragePath(file_path=data_path),
+            metadata_path=LocalStoragePath(file_path=metadata_path),
+        )

{isar-1.33.1 → isar-1.33.2}/src/isar/storage/storage_interface.py

@@ -1,25 +1,46 @@
 from abc import ABCMeta, abstractmethod
-from typing import Union
+from pathlib import Path
+from typing import Generic, TypeVar
+
+from pydantic import BaseModel
 
 from robot_interface.models.inspection.inspection import InspectionBlob
 from robot_interface.models.mission.mission import Mission
 
 
+class BlobStoragePath(BaseModel):
+    storage_account: str
+    blob_container: str
+    blob_name: str
+
+
+class LocalStoragePath(BaseModel):
+    file_path: Path
+
+
+TPath = TypeVar("TPath", BlobStoragePath, LocalStoragePath)
+
+
+class StoragePaths(BaseModel, Generic[TPath]):
+    data_path: TPath
+    metadata_path: TPath
+
+
 class StorageInterface(metaclass=ABCMeta):
     @abstractmethod
-    def store(self, inspection: InspectionBlob, mission: Mission) -> Union[str, dict]:
+    def store(self, inspection: InspectionBlob, mission: Mission) -> StoragePaths:
         """
         Parameters
         ----------
-        mission : Mission
-            Mission the inspection is a part of.
         inspection : InspectionBlob
             The inspection object to be stored.
+        mission : Mission
+            Mission the inspection is a part of.
 
         Returns
         ----------
-        String
-            Path of the saved inspection
+        StoragePaths
+            Paths to the data and metadata
 
         Raises
         ----------

{isar-1.33.1 → isar-1.33.2}/src/isar/storage/uploader.py

@@ -4,11 +4,17 @@ from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
 from queue import Empty, Queue
 from threading import Event
-from typing import List, Union
+from typing import List
 
 from isar.config.settings import settings
 from isar.models.events import Events
-from isar.storage.storage_interface import StorageException, StorageInterface
+from isar.storage.storage_interface import (
+    BlobStoragePath,
+    LocalStoragePath,
+    StorageException,
+    StorageInterface,
+    StoragePaths,
+)
 from robot_interface.models.inspection.inspection import (
     Inspection,
     InspectionBlob,
@@ -133,10 +139,10 @@ class Uploader:
             except Empty:
                 continue
 
-    def _upload(self, item: BlobItem) -> Union[str, dict]:
-        inspection_path: Union[str, dict] = ""
+    def _upload(self, item: BlobItem) -> StoragePaths:
+        inspection_paths: StoragePaths
         try:
-            inspection_path = item.storage_handler.store(
+            inspection_paths = item.storage_handler.store(
                 inspection=item.inspection, mission=item.mission
             )
             self.logger.info(
@@ -144,7 +150,7 @@ class Uploader:
                 f"uploaded inspection {str(item.inspection.id)[:8]}"
             )
             self._internal_upload_queue.remove(item)
-        except StorageException:
+        except StorageException as e:
             if item.get_retry_count() < self.max_retry_attempts:
                 item.increment_retry(self.max_wait_time)
                 self.logger.warning(
@@ -160,7 +166,8 @@ class Uploader:
                     f"{str(item.inspection.id)[:8]}. Aborting upload."
                 )
                 self._internal_upload_queue.remove(item)
-        return inspection_path
+            raise e
+        return inspection_paths
 
     def _process_upload_queue(self) -> None:
         def should_upload(_item):
@@ -181,10 +188,17 @@ class Uploader:
                 )
                 self._internal_upload_queue.remove(item)
             elif isinstance(item, BlobItem):
-                inspection_path = self._upload(item)
-                self._publish_inspection_result(
-                    inspection=item.inspection, inspection_path=inspection_path
-                )
+                try:
+                    inspection_paths = self._upload(item)
+                    if isinstance(inspection_paths.data_path, LocalStoragePath):
+                        self.logger.info("Skipping publishing when using local storage")
+                    else:
+                        self._publish_inspection_result(
+                            inspection=item.inspection,
+                            inspection_paths=inspection_paths,
+                        )
+                except StorageException:
+                    pass
             else:
                 self.logger.warning(
                     f"Unable to process upload item as its type {type(item).__name__} is not supported"
@@ -223,7 +237,9 @@ class Uploader:
         )
 
     def _publish_inspection_result(
-        self, inspection: InspectionBlob, inspection_path: Union[str, dict]
+        self,
+        inspection: InspectionBlob,
+        inspection_paths: StoragePaths[BlobStoragePath],
     ) -> None:
         """Publishes the reference of the inspection result to the MQTT Broker
         along with the analysis type
@@ -235,7 +251,8 @@ class Uploader:
             isar_id=settings.ISAR_ID,
             robot_name=settings.ROBOT_NAME,
             inspection_id=inspection.id,
-            inspection_path=inspection_path,
+            blob_storage_data_path=inspection_paths.data_path,
+            blob_storage_metadata_path=inspection_paths.metadata_path,
             installation_code=settings.PLANT_SHORT_NAME,
             tag_id=inspection.metadata.tag_id,
             inspection_type=type(inspection).__name__,

{isar-1.33.1 → isar-1.33.2}/src/isar.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: isar
-Version: 1.33.1
+Version: 1.33.2
 Summary: Integration and Supervisory control of Autonomous Robots
 Author-email: Equinor ASA <fg_robots_dev@equinor.com>
 License: Eclipse Public License version 2.0

{isar-1.33.1 → isar-1.33.2}/src/robot_interface/telemetry/mqtt_client.py

@@ -5,6 +5,10 @@ from datetime import datetime, timezone
 from queue import Queue
 from typing import Callable, Tuple
 
+from paho.mqtt.packettypes import PacketTypes
+from paho.mqtt.properties import Properties
+
+from isar.config.settings import settings
 from robot_interface.models.exceptions.robot_exceptions import (
     RobotTelemetryException,
     RobotTelemetryNoUpdateException,
@@ -14,10 +18,21 @@ from robot_interface.telemetry.payloads import CloudHealthPayload
 from robot_interface.utilities.json_service import EnhancedJSONEncoder
 
 
+def props_expiry(seconds: int) -> Properties:
+    p = Properties(PacketTypes.PUBLISH)
+    p.MessageExpiryInterval = seconds
+    return p
+
+
 class MqttClientInterface(metaclass=ABCMeta):
     @abstractmethod
     def publish(
-        self, topic: str, payload: str, qos: int = 0, retain: bool = False
+        self,
+        topic: str,
+        payload: str,
+        qos: int = 0,
+        retain: bool = False,
+        properties: Properties = None,
     ) -> None:
         """
         Parameters
@@ -42,9 +57,20 @@ class MqttPublisher(MqttClientInterface):
         self.mqtt_queue: Queue = mqtt_queue
 
     def publish(
-        self, topic: str, payload: str, qos: int = 0, retain: bool = False
+        self,
+        topic: str,
+        payload: str,
+        qos: int = 0,
+        retain: bool = False,
+        properties: Properties = None,
     ) -> None:
-        queue_message: Tuple[str, str, int, bool] = (topic, payload, qos, retain)
+        queue_message: Tuple[str, str, int, bool, Properties] = (
+            topic,
+            payload,
+            qos,
+            retain,
+            properties,
+        )
         self.mqtt_queue.put(queue_message)
 
 
@@ -57,6 +83,7 @@ class MqttTelemetryPublisher(MqttClientInterface):
         interval: float,
         qos: int = 0,
         retain: bool = False,
+        properties: Properties = None,
     ) -> None:
         self.mqtt_queue: Queue = mqtt_queue
         self.telemetry_method: Callable = telemetry_method
@@ -64,9 +91,13 @@ class MqttTelemetryPublisher(MqttClientInterface):
         self.interval: float = interval
         self.qos: int = qos
         self.retain: bool = retain
+        self.properties: Properties = properties
 
     def run(self, isar_id: str, robot_name: str) -> None:
         self.cloud_health_topic: str = f"isar/{isar_id}/cloud_health"
+        self.battery_topic: str = f"isar/{isar_id}/battery"
+        self.pose_topic: str = f"isar/{isar_id}/pose"
+        self.pressure_topic: str = f"isar/{isar_id}/pressure"
         topic: str
         payload: str
 
@@ -84,12 +115,37 @@ class MqttTelemetryPublisher(MqttClientInterface):
                 )
                 topic = self.cloud_health_topic
 
-            self.publish(topic=topic, payload=payload, qos=self.qos, retain=self.retain)
+            publish_properties = self.properties
+
+            if topic in (
+                self.battery_topic,
+                self.pose_topic,
+                self.pressure_topic,
+            ):
+                publish_properties = props_expiry(settings.MQTT_TELEMETRY_EXPIRY)
 
+            self.publish(
+                topic=topic,
+                payload=payload,
+                qos=self.qos,
+                retain=self.retain,
+                properties=publish_properties,
+            )
             time.sleep(self.interval)
 
     def publish(
-        self, topic: str, payload: str, qos: int = 0, retain: bool = False
+        self,
+        topic: str,
+        payload: str,
+        qos: int = 0,
+        retain: bool = False,
+        properties: Properties = None,
     ) -> None:
-        queue_message: Tuple[str, str, int, bool] = (topic, payload, qos, retain)
+        queue_message: Tuple[str, str, int, bool, Properties] = (
+            topic,
+            payload,
+            qos,
+            retain,
+            properties,
+        )
         self.mqtt_queue.put(queue_message)

{isar-1.33.1 → isar-1.33.2}/src/robot_interface/telemetry/payloads.py

@@ -1,9 +1,10 @@
 from dataclasses import dataclass
 from datetime import datetime
-from typing import List, Optional, Union
+from typing import List, Optional
 
 from alitra import Pose
 
+from isar.storage.storage_interface import BlobStoragePath
 from robot_interface.models.exceptions.robot_exceptions import ErrorReason
 from robot_interface.models.mission.status import MissionStatus, RobotStatus, TaskStatus
 from robot_interface.models.mission.task import TaskTypes
@@ -119,7 +120,8 @@ class InspectionResultPayload:
     isar_id: str
     robot_name: str
     inspection_id: str
-    inspection_path: Union[str, dict]
+    blob_storage_data_path: BlobStoragePath
+    blob_storage_metadata_path: BlobStoragePath
     installation_code: str
     tag_id: Optional[str]
     inspection_type: Optional[str]

{isar-1.33.1 → isar-1.33.2}/src/robot_interface/utilities/json_service.py

@@ -7,6 +7,7 @@ from uuid import UUID
 
 import numpy as np
 from alitra import Orientation
+from pydantic import BaseModel
 
 
 class EnhancedJSONEncoder(json.JSONEncoder):
@@ -15,6 +16,11 @@ class EnhancedJSONEncoder(json.JSONEncoder):
     """
 
     def default(self, o):
+        if isinstance(o, BaseModel):
+            dump = getattr(o, "model_dump", None)
+            if callable(dump):
+                return dump()
+            return o.__dict__
         if is_dataclass(o):
             return asdict(o)  # type: ignore
         if isinstance(o, UUID):