isar 1.10.14__tar.gz → 1.12.0__tar.gz

{isar-1.10.14 → isar-1.12.0}/.github/workflows/pythonpackage.yml

@@ -17,10 +17,10 @@ jobs:
         python-version: ["3.8", "3.9", "3.10"]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
 

{isar-1.10.14 → isar-1.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: isar
-Version: 1.10.14
+Version: 1.12.0
 Summary: Integration and Supervisory control of Autonomous Robots
 Home-page: https://github.com/equinor/isar
 Author: Equinor ASA

{isar-1.10.14 → isar-1.12.0}/docs/rst_processing.py

@@ -3,7 +3,6 @@ from pathlib import Path
 src_dir = Path("source/api")
 
 for file in src_dir.iterdir():
-
     print(f"Processing .rst-file: {file}")
 
     with open(file, "r") as f:

{isar-1.10.14 → isar-1.12.0}/main.py

@@ -7,12 +7,19 @@ from dotenv import load_dotenv
 from injector import Injector
 
 from isar.apis.api import API
-from isar.config.log import setup_logger
+from isar.config.keyvault.keyvault_service import Keyvault
+from isar.config.log import setup_loggers
 from isar.config.settings import settings
 from isar.models.communication.queues.queues import Queues
 from isar.modules import get_injector
 from isar.services.service_connections.mqtt.mqtt_client import MqttClient
-from isar.state_machine.state_machine import main
+from isar.services.service_connections.mqtt.robot_info_publisher import (
+    RobotInfoPublisher,
+)
+from isar.services.service_connections.mqtt.robot_status_publisher import (
+    RobotStatusPublisher,
+)
+from isar.state_machine.state_machine import StateMachine, main
 from isar.storage.storage_interface import StorageInterface
 from isar.storage.uploader import Uploader
 from robot_interface.robot_interface import RobotInterface
@@ -20,18 +27,20 @@ from robot_interface.robot_interface import RobotInterface
 if __name__ == "__main__":
     load_dotenv()
 
-    setup_logger()
-    logger: Logger = logging.getLogger("main")
-
     injector: Injector = get_injector()
 
+    keyvault_client = injector.get(Keyvault)
+    setup_loggers(keyvault=keyvault_client)
+    logger: Logger = logging.getLogger("main")
+
+    state_machine: StateMachine = injector.get(StateMachine)
     robot: RobotInterface = injector.get(RobotInterface)
     queues: Queues = injector.get(Queues)
 
     threads: List[Thread] = []
 
     state_machine_thread: Thread = Thread(
-        target=main, name="ISAR State Machine", args=[injector], daemon=True
+        target=main, name="ISAR State Machine", args=[state_machine], daemon=True
     )
     threads.append(state_machine_thread)
 
@@ -51,13 +60,31 @@ if __name__ == "__main__":
         mqtt_thread: Thread = Thread(
             target=mqtt_client.run, name="ISAR MQTT Client", daemon=True
         )
-
         threads.append(mqtt_thread)
 
+        robot_status_publisher: RobotStatusPublisher = RobotStatusPublisher(
+            mqtt_queue=queues.mqtt_queue, robot=robot, state_machine=state_machine
+        )
+        robot_status_thread: Thread = Thread(
+            target=robot_status_publisher.run,
+            name="ISAR Robot Status Publisher",
+            daemon=True,
+        )
+        threads.append(robot_status_thread)
+
+        robot_info_publisher: RobotInfoPublisher = RobotInfoPublisher(
+            mqtt_queue=queues.mqtt_queue
+        )
+        robot_info_thread: Thread = Thread(
+            target=robot_info_publisher.run,
+            name="ISAR Robot Info Publisher",
+            daemon=True,
+        )
+        threads.append(robot_info_thread)
+
         publishers: List[Thread] = robot.get_telemetry_publishers(
             queue=queues.mqtt_queue, robot_id=settings.ROBOT_ID
         )
-
         if publishers:
             threads.extend(publishers)
 

{isar-1.10.14 → isar-1.12.0}/setup.py

@@ -36,6 +36,9 @@ setup(
         "fastapi-azure-auth",
         "fastapi",
         "injector",
+        "opencensus-ext-logging",
+        "opencensus-ext-requests",
+        "opencensus-ext-azure",
         "numpy",
         "paho-mqtt",
         "pydantic",

{isar-1.10.14 → isar-1.12.0}/src/isar/apis/api.py

@@ -1,21 +1,34 @@
 import logging
 from http import HTTPStatus
 from logging import Logger
+import os
 from typing import List, Union
 
 import click
 import uvicorn
-from fastapi import FastAPI, Security
+from fastapi import FastAPI, Security, Request
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.routing import APIRouter
 from injector import inject
 from pydantic import AnyHttpUrl
 
+from opencensus.ext.azure.trace_exporter import AzureExporter
+from opencensus.trace.samplers import ProbabilitySampler
+from opencensus.trace.tracer import Tracer
+from opencensus.trace.span import SpanKind
+from opencensus.trace.attributes_helper import COMMON_ATTRIBUTES
+
 from isar.apis.models.models import ControlMissionResponse, StartMissionResponse
 from isar.apis.schedule.scheduling_controller import SchedulingController
 from isar.apis.security.authentication import Authenticator
+from isar.config.configuration_error import ConfigurationError
+from isar.config.keyvault.keyvault_error import KeyvaultError
+from isar.config.keyvault.keyvault_service import Keyvault
 from isar.config.settings import settings
 
+HTTP_URL = COMMON_ATTRIBUTES["HTTP_URL"]
+HTTP_STATUS_CODE = COMMON_ATTRIBUTES["HTTP_STATUS_CODE"]
+
 
 class API:
     @inject
@@ -23,13 +36,16 @@ class API:
         self,
         authenticator: Authenticator,
         scheduling_controller: SchedulingController,
+        keyvault_client: Keyvault,
         port: int = settings.API_PORT,
+        azure_ai_logging_enabled: bool = settings.LOG_HANDLER_APPLICATION_INSIGHTS_ENABLED,
     ) -> None:
-
         self.authenticator: Authenticator = authenticator
         self.scheduling_controller: SchedulingController = scheduling_controller
+        self.keyvault_client: Keyvault = keyvault_client
         self.host: str = "0.0.0.0"  # Locking uvicorn to use 0.0.0.0
         self.port: int = port
+        self.azure_ai_logging_enabled: bool = azure_ai_logging_enabled
 
         self.logger: Logger = logging.getLogger("api")
 
@@ -77,6 +93,9 @@ class API:
                 allow_headers=["*"],
             )
 
+        if self.azure_ai_logging_enabled:
+            self._add_request_logging_middleware(app)
+
         app.include_router(router=self._create_scheduler_router())
 
         app.include_router(router=self._create_info_router())
@@ -84,7 +103,6 @@ class API:
         return app
 
     def _create_scheduler_router(self) -> APIRouter:
-
         router: APIRouter = APIRouter(tags=["Scheduler"])
 
         authentication_dependency: Security = Security(self.authenticator.get_scheme())
@@ -213,7 +231,6 @@ class API:
         return router
 
     def _create_info_router(self) -> APIRouter:
-
         router: APIRouter = APIRouter(tags=["Info"])
 
         authentication_dependency: Security = Security(self.authenticator.get_scheme())
@@ -244,3 +261,36 @@ class API:
             self.port,
             extra={"color_message": color_message},
         )
+
+    def _add_request_logging_middleware(self, app: FastAPI) -> None:
+        connection_string: str
+        try:
+            connection_string = self.keyvault_client.get_secret(
+                "application-insights-connection-string"
+            ).value
+        except KeyvaultError:
+            message: str = (
+                "Missing connection string for Application Insights in key vault. "
+            )
+            self.logger.critical(message)
+            raise ConfigurationError(message)
+
+        @app.middleware("http")
+        async def middlewareOpencensus(request: Request, call_next):
+            tracer = Tracer(
+                exporter=AzureExporter(connection_string=connection_string),
+                sampler=ProbabilitySampler(1.0),
+            )
+            with tracer.span("main") as span:
+                span.span_kind = SpanKind.SERVER
+
+                response = await call_next(request)
+
+                tracer.add_attribute_to_current_span(
+                    attribute_key=HTTP_STATUS_CODE, attribute_value=response.status_code
+                )
+                tracer.add_attribute_to_current_span(
+                    attribute_key=HTTP_URL, attribute_value=str(request.url)
+                )
+
+            return response

{isar-1.10.14 → isar-1.12.0}/src/isar/apis/models/start_mission_definition.py

@@ -1,37 +1,41 @@
-from typing import List, Optional, Type
+from enum import Enum
+from typing import List, Optional
 
 from alitra import Position
 from pydantic import BaseModel, Field
+
+from isar.apis.models.models import InputPose, InputPosition
+from isar.mission_planner.mission_planner_interface import MissionPlannerError
+from isar.models.mission.mission import Mission, Task
 from robot_interface.models.mission.step import (
     STEPS,
     DriveToPose,
-    InspectionStep,
     TakeImage,
     TakeThermalImage,
     TakeThermalVideo,
     TakeVideo,
 )
 
-from isar.apis.models.models import InputPose, InputPosition
-from isar.mission_planner.mission_planner_interface import MissionPlannerError
-from isar.models.mission.mission import Mission, Task
 
-inspection_step_types: List[Type[InspectionStep]] = [
-    TakeImage,
-    TakeThermalImage,
-    TakeVideo,
-    TakeThermalVideo,
-]
+class InspectionTypes(str, Enum):
+    image = "Image"
+    thermal_image = "ThermalImage"
+    video = "Video"
+    thermal_video = "ThermalVideo"
+
+
+class StartMissionInspectionDefinition(BaseModel):
+    type: InspectionTypes = Field(default=InspectionTypes.image)
+    inspection_target: InputPosition
+    analysis_types: Optional[List]
+    duration: Optional[float]
+    metadata: Optional[dict]
 
 
 class StartMissionTaskDefinition(BaseModel):
     pose: InputPose
     tag: Optional[str]
-    inspection_target: InputPosition
-    inspection_types: List[str] = Field(
-        default=[step.get_inspection_type().__name__ for step in inspection_step_types]
-    )
-    video_duration: Optional[float] = Field(default=10)
+    inspections: List[StartMissionInspectionDefinition]
 
 
 class StartMissionDefinition(BaseModel):
@@ -47,14 +51,16 @@ def to_isar_mission(mission_definition: StartMissionDefinition) -> Mission:
             drive_step: DriveToPose = DriveToPose(pose=task.pose.to_alitra_pose())
             inspection_steps: List[STEPS] = [
                 create_inspection_step(
-                    inspection_type=inspection_type,
-                    duration=task.video_duration,
-                    target=task.inspection_target.to_alitra_position(),
+                    inspection_type=inspection.type,
+                    duration=inspection.duration,
+                    target=inspection.inspection_target.to_alitra_position(),
                     tag_id=tag_id,
+                    analysis=inspection.analysis_types,
+                    metadata=inspection.metadata,
                 )
-                for inspection_type in task.inspection_types
+                for inspection in task.inspections
             ]
-        except (ValueError) as e:
+        except ValueError as e:
             raise MissionPlannerError(f"Failed to create task: {str(e)}")
         isar_task: Task = Task(steps=[drive_step, *inspection_steps], tag_id=tag_id)
         isar_tasks.append(isar_task)
@@ -68,27 +74,30 @@ def to_isar_mission(mission_definition: StartMissionDefinition) -> Mission:
 
 
 def create_inspection_step(
-    inspection_type: str, duration: float, target: Position, tag_id: Optional[str]
+    inspection_type: InspectionTypes,
+    duration: float,
+    target: Position,
+    analysis: Optional[List],
+    tag_id: Optional[str],
+    metadata: Optional[dict],
 ) -> STEPS:
     inspection_step: STEPS
-
-    if inspection_type == TakeImage.get_inspection_type().__name__:
+    if inspection_type == InspectionTypes.image.value:
         inspection_step = TakeImage(target=target)
-        if tag_id:
-            inspection_step.tag_id = tag_id
-    elif inspection_type == TakeVideo.get_inspection_type().__name__:
+    elif inspection_type == InspectionTypes.video.value:
         inspection_step = TakeVideo(target=target, duration=duration)
-        if tag_id:
-            inspection_step.tag_id = tag_id
-    elif inspection_type == TakeThermalImage.get_inspection_type().__name__:
+    elif inspection_type == InspectionTypes.thermal_image.value:
         inspection_step = TakeThermalImage(target=target)
-        if tag_id:
-            inspection_step.tag_id = tag_id
-    elif inspection_type == TakeThermalVideo.get_inspection_type().__name__:
+    elif inspection_type == InspectionTypes.thermal_video.value:
         inspection_step = TakeThermalVideo(target=target, duration=duration)
-        if tag_id:
-            inspection_step.tag_id = tag_id
     else:
         raise ValueError(f"Inspection type '{inspection_type}' not supported")
 
+    if tag_id:
+        inspection_step.tag_id = tag_id
+    if analysis:
+        inspection_step.analysis = analysis
+    if metadata:
+        inspection_step.metadata = metadata
+
     return inspection_step

{isar-1.10.14 → isar-1.12.0}/src/isar/apis/security/authentication.py

@@ -1,11 +1,10 @@
 import logging
 
-
 from fastapi import Depends
-from fastapi_azure_auth.exceptions import InvalidAuth
-from fastapi_azure_auth.user import User
 from fastapi.security.base import SecurityBase
 from fastapi_azure_auth import SingleTenantAzureAuthorizationCodeBearer
+from fastapi_azure_auth.exceptions import InvalidAuth
+from fastapi_azure_auth.user import User
 from pydantic import BaseModel
 
 from isar.config.settings import settings
@@ -32,8 +31,8 @@ azure_scheme = SingleTenantAzureAuthorizationCodeBearer(
 
 async def validate_has_role(user: User = Depends(azure_scheme)) -> None:
     """
-    Validate that a user has the `role` role in order to access the API.
-    Raises a 401 authentication error if not.
+    Validate if the user has the required role in order to access the API.
+    Raises a 401 authorization error if not.
     """
     if settings.REQUIRED_ROLE not in user.roles:
         raise InvalidAuth(

isar-1.12.0/src/isar/config/keyvault/keyvault_service.py

@@ -0,0 +1,88 @@
+import logging
+import traceback
+from typing import Union
+
+from azure.core.exceptions import (
+    ClientAuthenticationError,
+    HttpResponseError,
+    ResourceNotFoundError,
+)
+from azure.identity import ClientSecretCredential, DefaultAzureCredential
+from azure.keyvault.secrets import KeyVaultSecret, SecretClient
+
+from isar.config.keyvault.keyvault_error import KeyvaultError
+
+
+class Keyvault:
+    def __init__(
+        self,
+        keyvault_name: str,
+        client_id: str = None,
+        client_secret: str = None,
+        tenant_id: str = None,
+    ):
+        self.name = keyvault_name
+        self.url = "https://" + keyvault_name + ".vault.azure.net"
+        self.client_id = client_id
+        self.client_secret = client_secret
+        self.tenant_id = tenant_id
+        self.logger = logging.getLogger("API")
+        self.client: SecretClient = None
+
+    def get_secret(self, secret_name: str) -> KeyVaultSecret:
+        secret_client: SecretClient = self.get_secret_client()
+        try:
+            secret: KeyVaultSecret = secret_client.get_secret(name=secret_name)
+        except ResourceNotFoundError:
+            self.logger.error(
+                "Secret '%s' was not found in keyvault '%s'.",
+                secret_name,
+                self.name,
+                exc_info=True,
+            )
+            raise KeyvaultError  # type: ignore
+        except HttpResponseError:
+            self.logger.error(
+                "An error occurred while retrieving the secret '%s' from keyvault '%s'.",
+                secret_name,
+                self.name,
+                exc_info=True,
+            )
+            raise KeyvaultError  # type: ignore
+
+        return secret
+
+    def set_secret(self, secret_name: str, secret_value) -> None:
+        secret_client: SecretClient = self.get_secret_client()
+        try:
+            secret_client.set_secret(name=secret_name, value=secret_value)
+        except HttpResponseError:
+            self.logger.error(
+                "An error occurred while setting secret '%s' in keyvault '%s'.",
+                secret_name,
+                self.name,
+                exc_info=True,
+            )
+            raise KeyvaultError  # type: ignore
+
+    def get_secret_client(self) -> SecretClient:
+        if self.client == None:
+            try:
+                credential: Union[ClientSecretCredential, DefaultAzureCredential]
+                if self.client_id and self.client_secret and self.tenant_id:
+                    credential = ClientSecretCredential(
+                        tenant_id=self.tenant_id,
+                        client_id=self.client_id,
+                        client_secret=self.client_secret,
+                    )
+                else:
+                    credential = DefaultAzureCredential()
+            except ClientAuthenticationError:
+                self.logger.error(
+                    "Failed to authenticate to Azure while connecting to KeyVault",
+                    exc_info=True,
+                )
+                raise KeyvaultError
+
+            self.client = SecretClient(vault_url=self.url, credential=credential)
+        return self.client

isar-1.12.0/src/isar/config/log.py

@@ -0,0 +1,61 @@
+import importlib.resources as pkg_resources
+import logging
+import logging.config
+import yaml
+from uvicorn.logging import ColourizedFormatter
+from opencensus.ext.azure.log_exporter import AzureLogHandler
+from isar.config.configuration_error import ConfigurationError
+from isar.config.keyvault.keyvault_error import KeyvaultError
+from isar.config.keyvault.keyvault_service import Keyvault
+from isar.config.settings import settings
+
+
+def setup_loggers(keyvault: Keyvault) -> None:
+    log_levels: dict = settings.LOG_LEVELS
+    with pkg_resources.path("isar.config", "logging.conf") as path:
+        log_config = yaml.safe_load(open(path))
+
+    logging.config.dictConfig(log_config)
+
+    handlers = []
+    if settings.LOG_HANDLER_LOCAL_ENABLED:
+        handlers.append(configure_console_handler(log_config=log_config))
+    if settings.LOG_HANDLER_APPLICATION_INSIGHTS_ENABLED:
+        handlers.append(
+            configure_azure_handler(log_config=log_config, keyvault=keyvault)
+        )
+
+    for log_handler in handlers:
+        for loggers in log_config["loggers"].keys():
+            logging.getLogger(loggers).addHandler(log_handler)
+            logging.getLogger(loggers).setLevel(log_levels[loggers])
+        logging.getLogger().addHandler(log_handler)
+
+
+def configure_console_handler(log_config: dict) -> logging.Handler:
+    handler = logging.StreamHandler()
+    handler.setLevel(log_config["root"]["level"])
+    handler.setFormatter(
+        ColourizedFormatter(
+            log_config["formatters"]["colourized"]["format"],
+            style="{",
+            use_colors=True,
+        )
+    )
+    return handler
+
+
+def configure_azure_handler(log_config: dict, keyvault: Keyvault) -> logging.Handler:
+    connection_string: str
+    try:
+        connection_string = keyvault.get_secret(
+            "application-insights-connection-string"
+        ).value
+    except KeyvaultError:
+        message: str = f"CRITICAL ERROR: Missing connection string for Application Insights in key vault '{keyvault.name}'."
+        print(f"\n{message} \n")
+        raise ConfigurationError(message)
+
+    handler = AzureLogHandler(connection_string=connection_string)
+    handler.setLevel(log_config["root"]["level"])
+    return handler