saviialib 0.9.1__py3-none-any.whl → 1.6.0__py3-none-any.whl

saviialib/services/netcamera/use_cases/get_media_files.py

@@ -0,0 +1,76 @@
+from .types.get_media_files_types import (
+    GetMediaFilesUseCaseInput,
+    GetMediaFilesUseCaseOutput,
+)
+from saviialib.libs.ffmpeg_client import (
+    FfmpegClient,
+    FfmpegClientInitArgs,
+    RecordVideoArgs,
+    RecordPhotoArgs,
+)
+from saviialib.libs.zero_dependency.utils.strings_utils import are_equal
+from typing import Tuple, Dict
+from saviialib.libs.directory_client.directory_client import (
+    DirectoryClient,
+    DirectoryClientArgs,
+)
+from saviialib.general_types.error_types.api.saviia_netcamera_error_types import (
+    NetcameraConnectionError,
+)
+
+
+class GetMediaFilesUseCase:
+    def __init__(self, input: GetMediaFilesUseCaseInput) -> None:
+        self.ffmpeg_client = FfmpegClient(
+            FfmpegClientInitArgs(
+                client_name="ffmpeg_asyncio",
+            )
+        )
+        self.dir_client = DirectoryClient(DirectoryClientArgs("os_client"))
+        self.user = input.username
+        self.pwd = input.password
+        self.cameras: Dict[str, Tuple[str, int]] = input.cameras
+        self.protocol = input.protocol
+        self.logger = input.logger
+        self.dest_path = input.destination_path
+
+    async def _retieve_with_rtsp(self):
+        for name, conn in self.cameras.items():
+            ip, port = conn
+            dest_path = self.dir_client.join_paths(self.dest_path, name)
+            try:
+                # Extraction of photo files into dest_path dir.
+                await self.ffmpeg_client.record_photo(
+                    RecordPhotoArgs(
+                        ip_address=ip,
+                        port=str(port),
+                        destination_path=dest_path,
+                        rtsp_user=self.user,
+                        rtsp_password=self.pwd,
+                        extension="jpg",
+                        frames=1,
+                    )
+                )
+                # Extraction of video files into dest_path dir.
+                await self.ffmpeg_client.record_video(
+                    RecordVideoArgs(
+                        destination_path=dest_path,
+                        ip_address=ip,
+                        port=str(port),
+                        rtsp_user=self.user,
+                        rtsp_password=self.pwd,
+                        extension="mp3",
+                        duration=10,
+                    )
+                )
+            except ConnectionError as error:
+                raise NetcameraConnectionError(reason=error)
+
+    async def execute(self) -> GetMediaFilesUseCaseOutput:
+        if are_equal(self.protocol, "rtsp"):
+            await self._retieve_with_rtsp()
+        else:
+            raise NotImplementedError(
+                f"The media files extraction with {self.protocol} is not implemented yet."
+            )
+        return GetMediaFilesUseCaseOutput()

saviialib/services/netcamera/use_cases/types/get_media_files_types.py

@@ -0,0 +1,18 @@
+from dataclasses import dataclass
+from logging import Logger
+from typing import Dict, Tuple
+
+
+@dataclass
+class GetMediaFilesUseCaseInput:
+    cameras: Dict[str, Tuple[str, int]]
+    username: str
+    password: str
+    protocol: str
+    logger: Logger
+    destination_path: str
+
+
+@dataclass
+class GetMediaFilesUseCaseOutput:
+    pass

saviialib/services/shakes/api.py

@@ -0,0 +1,31 @@
+from .controllers.get_miniseed_files import (
+    GetMiniseedFilesController,
+    GetMiniseedFilesControllerInput,
+)
+from saviialib.general_types.api.saviia_shakes_api_types import SaviiaShakesConfig
+
+from typing import Dict
+
+
+class ShakesAPI:
+    """This class provides methods for interacting with Raspberry Shakes"""
+
+    def __init__(self, config: SaviiaShakesConfig) -> None:
+        self.config = config
+
+    async def get_miniseed_files(self, raspberry_shakes: Dict[str, str]):
+        """Download the MiniSEED files from the SFTP Server provided by each Raspberry Shake.
+        Args:
+            raspberry_shakes (dict): Dictionary where the key is the name of the Raspberry Shake,
+                and the value is the IP Address.
+        Returns:
+            response (dict): A dictionary containg the response from the download operation.
+                This response will tipically include the message, the response status, and metadata.
+        """
+        controller = GetMiniseedFilesController(
+            GetMiniseedFilesControllerInput(
+                config=self.config, raspberry_shakes=raspberry_shakes
+            )
+        )
+        response = await controller.execute()
+        return response.__dict__

saviialib/services/shakes/controllers/get_miniseed_files.py

@@ -0,0 +1,48 @@
+from .types.get_miniseed_files_types import (
+    GetMiniseedFilesControllerInput,
+    GetMiniseedFilesControllerOutput,
+)
+from saviialib.services.shakes.use_cases.get_miniseed_files import (
+    GetMiniseedFilesUseCase,
+    GetMiniseedFilesUseCaseInput,
+)
+from http import HTTPStatus
+from saviialib.general_types.error_types.api.saviia_api_error_types import (
+    ShakesNoContentError,
+)
+from saviialib.general_types.error_types.common.common_types import SftpClientError
+
+
+class GetMiniseedFilesController:
+    def __init__(self, input: GetMiniseedFilesControllerInput) -> None:
+        self.use_case = GetMiniseedFilesUseCase(
+            GetMiniseedFilesUseCaseInput(
+                raspberry_shakes=input.raspberry_shakes,
+                username=input.config.sftp_user,
+                password=input.config.sftp_password,
+                ssh_key_path=input.config.ssh_key_path,
+                port=input.config.sftp_port,
+                logger=input.config.logger,
+            )
+        )
+
+    async def execute(self) -> GetMiniseedFilesControllerOutput:
+        try:
+            res = await self.use_case.execute()
+            return GetMiniseedFilesControllerOutput(
+                message="The MiniSEED files have been downloaded succesfully!",
+                status=HTTPStatus.OK.value,
+                metadata=res.download_status,
+            )
+        except ShakesNoContentError as error:
+            return GetMiniseedFilesControllerOutput(
+                message="No files to upload.",
+                status=HTTPStatus.NO_CONTENT.value,
+                metadata={"error": error.__str__()},
+            )
+        except SftpClientError as error:
+            return GetMiniseedFilesControllerOutput(
+                message="An unexpected error ocurred during SFTP Client connection.",
+                status=HTTPStatus.REQUEST_TIMEOUT.value,
+                metadata={"error": error.__str__()},
+            )

saviialib/services/shakes/controllers/types/get_miniseed_files_types.py

@@ -0,0 +1,16 @@
+from dataclasses import dataclass
+from typing import Dict
+from saviialib.general_types.api.saviia_shakes_api_types import SaviiaShakesConfig
+
+
+@dataclass
+class GetMiniseedFilesControllerInput:
+    raspberry_shakes: Dict[str, str]
+    config: SaviiaShakesConfig
+
+
+@dataclass
+class GetMiniseedFilesControllerOutput:
+    status: int
+    metadata: Dict
+    message: str

saviialib/services/shakes/use_cases/get_miniseed_files.py

@@ -0,0 +1,79 @@
+import asyncio
+from .types.get_miniseed_files_types import (
+    GetMiniseedFilesUseCaseInput,
+    GetMiniseedFilesUseCaseOutput,
+)
+from typing import Dict, Any
+from saviialib.libs.sftp_client import (
+    SFTPClient,
+    SFTPClientInitArgs,
+    ListfilesArgs,
+    DownloadfilesArgs,
+)
+from saviialib.libs.directory_client import DirectoryClient, DirectoryClientArgs
+from saviialib.general_types.error_types.api.saviia_api_error_types import (
+    ShakesNoContentError,
+)
+from saviialib.general_types.error_types.common.common_types import SftpClientError
+from .utils.get_miniseed_files_utils import parse_downloaded_metadata
+
+
+class GetMiniseedFilesUseCase:
+    def __init__(self, input: GetMiniseedFilesUseCaseInput) -> None:
+        self.password = input.password
+        self.username = input.username
+        self.ssh_key_path = input.ssh_key_path
+        self.port = input.port
+        self.raspberry_shakes: Dict[str, str] = input.raspberry_shakes
+        self.dir_client = DirectoryClient(DirectoryClientArgs("os_client"))
+
+    def _initialize_sftp_client(self, ip_address: str):
+        return SFTPClient(
+            SFTPClientInitArgs(
+                "asyncssh_sftp",
+                password=self.password,
+                username=self.username,
+                ssh_key_path=self.ssh_key_path,
+                host=ip_address,
+                port=self.port,
+            )
+        )
+
+    async def _download_mseed_file(self, rs_name: str, rs_ip: str) -> Dict[str, Any]:
+        DEST_BASE_DIR = "./rshakes-mseed-files"
+        SOURCE_PATH = self.dir_client.join_paths("opt", "data", "archive")
+        local_path = self.dir_client.join_paths(DEST_BASE_DIR, rs_name)
+        if not await self.dir_client.isdir(local_path):
+            await self.dir_client.makedirs(local_path)
+        sftp_client = self._initialize_sftp_client(rs_ip)
+
+        local_files = await self.dir_client.listdir(local_path)
+        try:
+            sftp_files = await sftp_client.list_files(ListfilesArgs(path=SOURCE_PATH))
+            pending_files = set(sftp_files) - set(local_files)
+            if not pending_files:
+                raise ShakesNoContentError
+
+            await sftp_client.download_files(
+                DownloadfilesArgs(
+                    source_path=SOURCE_PATH,
+                    destination_path=local_path,
+                    files_to_download=list(pending_files),
+                )
+            )
+        except ConnectionError as error:
+            raise SftpClientError(reason=error)
+        return {
+            "rs_name": rs_name,
+            "destination_path": local_path,
+            "total_files": len(pending_files),
+        }
+
+    async def execute(self):
+        requests = []
+        for rs_name, rs_ip in self.raspberry_shakes:
+            requests.append(self._download_mseed_file(rs_name, rs_ip))
+        responses = await asyncio.gather(*requests, return_exceptions=True)
+        return GetMiniseedFilesUseCaseOutput(
+            download_status=parse_downloaded_metadata(responses),
+        )

saviialib/services/shakes/use_cases/types/get_miniseed_files_types.py

@@ -0,0 +1,18 @@
+from dataclasses import dataclass, field
+from typing import Dict
+from logging import Logger
+
+
+@dataclass
+class GetMiniseedFilesUseCaseInput:
+    raspberry_shakes: Dict[str, str]
+    username: str
+    password: str
+    ssh_key_path: str
+    port: int
+    logger: Logger
+
+
+@dataclass
+class GetMiniseedFilesUseCaseOutput:
+    download_status: Dict[str, str] = field(default_factory=dict)

saviialib/services/shakes/use_cases/utils/get_miniseed_files_utils.py

@@ -0,0 +1,11 @@
+from typing import Dict, List, Any
+
+
+def parse_downloaded_metadata(responses: List[Any]) -> Dict[str, Any]:
+    return {
+        x["rs_name"]: {
+            "destination_path": x["destination_path"],
+            "total_files": x["total_files"],
+        }
+        for x in responses
+    }

saviialib/services/thies/api.py

@@ -0,0 +1,42 @@
+from typing import Any, Dict, List
+
+from .controllers.types.update_thies_data_types import UpdateThiesDataControllerInput
+from .controllers.update_thies_data import UpdateThiesDataController
+from saviialib.general_types.api.saviia_thies_api_types import (
+    SaviiaThiesConfig,
+)
+
+
+class SaviiaThiesAPI:
+    def __init__(self, config: SaviiaThiesConfig) -> None:
+        self.config = config
+
+    async def update_thies_data(
+        self,
+        sharepoint_folders_path: List[str],
+        ftp_server_folders_path: List[str],
+        local_backup_source_path: str,
+    ) -> Dict[str, Any]:
+        """Updates data from a THIES Data Logger by connecting to an FTP server
+        and transferring data to specified Sharepoint folders.
+
+        :param list sharepoint_folders_path: List of Sharepoint folder paths for AVG and EXT data.
+            The AVG path must be the first element.
+        :param list ftp_server_folders_path: List of FTP server folder paths for AVG and EXT data.
+            The AVG path must be the first element.
+        :param str local_backup_source_path: Path of the main directory where the files extracted from
+            the Thies FTP Server are going to be stored
+
+        :return: A dictionary representation of the API response.
+        :rtype: dict
+        """
+        controller = UpdateThiesDataController(
+            UpdateThiesDataControllerInput(
+                self.config,
+                sharepoint_folders_path,
+                ftp_server_folders_path,
+                local_backup_source_path,
+            )
+        )
+        response = await controller.execute()
+        return response.__dict__

saviialib/services/thies/constants/update_thies_data_constants.py

@@ -0,0 +1,67 @@
+AVG_COLUMNS = {
+    "Date": "date",
+    "Time": "time",
+    "AirTemperature": "air_temperature",
+    "Radiation": "radiation",
+    "CO2": "carbon_dioxide",
+    "Precipitation": "precipitation",
+    "WS": "wind_velocity",
+    "WD": "wind_direction",
+    "Humidity": "humidity",
+    "UBat": "battery",
+}
+
+EXT_COLUMNS = {
+    "Date": "date",
+    "Time": "time",
+    "AirTemperature MIN": "air_temperature",
+    "AirTemperature MAX": "air_temperature",
+    "Radiation MIN": "radiation",
+    "Radiation MAX": "radiation",
+    "CO2 MIN": "carbon_dioxide",
+    "CO2 MAX": "carbon_dioxide",
+    "WS MIN": "wind_velocity",
+    "WS MAX gust": "wind_velocity",
+    "WD MIN": "wind_direction",
+    "WD MAX gust": "wind_direction",
+    "Humidity MIN": "humidity",
+    "Humidity MAX": "humidity",
+    "UBat MIN": "battery",
+    "UBat MAX": "battery",
+}
+
+AGG_DICT = {
+    "AirTemperature": "mean",
+    "AirTemperature MIN": "mean",
+    "AirTemperature MAX": "mean",
+    "Precipitation": "sum",
+    "Humidity": "mean",
+    "Humidity MIN": "mean",
+    "Humidity MAX": "mean",
+    "Radiation": "sum",
+    "Radiation MIN": "sum",
+    "Radiation MAX": "sum",
+    "CO2": "mean",
+    "CO2 MIN": "mean",
+    "CO2 MAX": "mean",
+    "WS": "mean",
+    "WS MIN": "mean",
+    "WS MAX gust": "mean",
+    "WD": "mean",
+    "WD MIN": "mean",
+    "WD MAX gust": "mean",
+    "UBat": "mean",
+    "UBat MIN": "mean",
+    "UBat MAX": "mean",
+}
+
+UNITS = {
+    "AirTemperature": "°C",
+    "Precipitation": "mm",
+    "Humidity": "%",
+    "Radiation": "W/m²",
+    "CO2": "ppm",
+    "WS": "m/s",
+    "WD": "°",
+    "UBat": "V",
+}

saviialib/services/{epii → thies}/controllers/types/update_thies_data_types.py

@@ -1,13 +1,14 @@
 from dataclasses import dataclass, field
 from typing import Dict
-from saviialib.general_types.api.epii_api_types import (
-    EpiiUpdateThiesConfig,
-)
+from saviialib.general_types.api.saviia_thies_api_types import SaviiaThiesConfig
 
 
 @dataclass
 class UpdateThiesDataControllerInput:
-    config: EpiiUpdateThiesConfig
+    config: SaviiaThiesConfig
+    sharepoint_folders_path: list
+    ftp_server_folders_path: list
+    local_backup_source_path: str
 
 
 @dataclass

saviialib/services/{epii → thies}/controllers/update_thies_data.py

@@ -1,26 +1,27 @@
 from http import HTTPStatus
 
-from saviialib.general_types.error_types.api.epii_api_error_types import (
+from saviialib.general_types.error_types.api.saviia_api_error_types import (
     SharePointFetchingError,
     ThiesConnectionError,
     ThiesFetchingError,
     SharePointUploadError,
+    SharePointDirectoryError,
 )
 from saviialib.general_types.error_types.common.common_types import (
     EmptyDataError,
     FtpClientError,
     SharepointClientError,
 )
-from saviialib.services.epii.controllers.types.update_thies_data_types import (
+from saviialib.services.thies.controllers.types.update_thies_data_types import (
     UpdateThiesDataControllerInput,
     UpdateThiesDataControllerOutput,
 )
-from saviialib.services.epii.use_cases.types import (
+from saviialib.services.backup.use_cases.types import (
     UpdateThiesDataUseCaseInput,
     SharepointConfig,
     FtpClientConfig,
 )
-from saviialib.services.epii.use_cases.update_thies_data import (
+from saviialib.services.thies.use_cases.update_thies_data import (
     UpdateThiesDataUseCase,
 )
 
@@ -42,6 +43,10 @@ class UpdateThiesDataController:
                     sharepoint_tenant_name=input.config.sharepoint_tenant_name,
                     sharepoint_tenant_id=input.config.sharepoint_tenant_id,
                 ),
+                sharepoint_folders_path=input.sharepoint_folders_path,
+                ftp_server_folders_path=input.ftp_server_folders_path,
+                local_backup_source_path=input.local_backup_source_path,
+                logger=input.config.logger,
             )
         )
 
@@ -51,7 +56,7 @@ class UpdateThiesDataController:
             return UpdateThiesDataControllerOutput(
                 message="THIES was synced successfully",
                 status=HTTPStatus.OK.value,
-                metadata={"data": data},
+                metadata={"data": data},  # type: ignore
             )
         except EmptyDataError:
             return UpdateThiesDataControllerOutput(
@@ -87,7 +92,14 @@ class UpdateThiesDataController:
 
         except SharePointUploadError as error:
             return UpdateThiesDataControllerOutput(
-                message="An error oucrred while uploading files to RCER Cloud",
+                message="An error ocurred while uploading files to RCER Cloud",
+                status=HTTPStatus.BAD_REQUEST.value,
+                metadata={"error": error.__str__()},
+            )
+
+        except SharePointDirectoryError as error:
+            return UpdateThiesDataControllerOutput(
+                message="An error ocurred while extracting folders from Microsoft Sharepoint",
                 status=HTTPStatus.BAD_REQUEST.value,
                 metadata={"error": error.__str__()},
             )

saviialib/services/thies/use_cases/components/create_thies_statistics_file.py

@@ -0,0 +1,115 @@
+from .thies_bp import THIESDayData
+import pandas as pd
+from logging import Logger
+from asyncio import to_thread
+from saviialib.libs.directory_client import DirectoryClient
+from saviialib.libs.zero_dependency.utils.datetime_utils import datetime_to_str, today
+from saviialib.libs.files_client import FilesClient, FilesClientInitArgs, WriteArgs
+import saviialib.services.thies.constants.update_thies_data_constants as c
+
+
+async def create_thies_daily_statistics_file(
+    local_backup_path: str, os_client: DirectoryClient, logger: Logger
+) -> None:
+    csv_client = FilesClient(FilesClientInitArgs(client_name="csv_client"))
+    filename = datetime_to_str(today(), date_format="%Y%m%d") + ".BIN"
+    logger.debug(
+        f"[thies_synchronization_lib] Creating Daily Statistics for {filename}"
+    )
+    path_bin_av = os_client.join_paths(local_backup_path, "thies", "AVG", filename)
+    path_ini_av = os_client.join_paths(
+        local_backup_path, "thies", "AVG", "DESCFILE.INI"
+    )
+    path_bin_ex = os_client.join_paths(local_backup_path, "thies", "EXT", filename)
+    path_ini_ex = os_client.join_paths(
+        local_backup_path, "thies", "EXT", "DESCFILE.INI"
+    )
+
+    ext_df = THIESDayData("ex")
+    await to_thread(ext_df.read_binfile, path_bin_ex, path_ini_ex)
+
+    avg_df = THIESDayData("av")
+    await to_thread(avg_df.read_binfile, path_bin_av, path_ini_av)
+
+    ext_df = ext_df.dataDF[c.EXT_COLUMNS.keys()]
+    avg_df = avg_df.dataDF[c.AVG_COLUMNS.keys()]
+
+    # Merge both dataframes
+    df = avg_df.merge(ext_df, on=["Date", "Time"], how="outer")
+    # Set the date as dd.mm.yyyy format.
+    df["Date"] = df["Date"].str.replace(
+        r"(\d{4})/(\d{2})/(\d{2})", r"\3.\2.\1", regex=True
+    )
+    df["Hour"] = df["Time"].str[:2]
+
+    # Group by hour.
+    hourly_agg = df.groupby(["Date", "Hour"]).agg(c.AGG_DICT).reset_index()
+
+    rows = []
+    # For each attribute in avg_columns (except Date, Time)
+    for col, col_id in c.AVG_COLUMNS.items():
+        if col in ["Date", "Time"]:
+            continue
+        # Determine the corresponding min/max columns if they exist
+        min_col = f"{col} MIN"
+        max_col = f"{col} MAX"
+        mean_col = col
+        if col in ["WS", "WD"]:
+            max_col += " gust"
+
+        unit = c.UNITS.get(col, "")
+
+        for idx, row in hourly_agg.iterrows():
+            statistic_id = f"sensor.saviia_epii_{col_id}"
+            start = f"{row['Date']} {row['Hour']}:00"
+            mean = row[mean_col] if mean_col in row else 0
+            min_val = row[min_col] if min_col in row else mean
+            max_val = row[max_col] if max_col in row else mean
+
+            # If no min/max for this attribute, set as Na or 0 as requested
+            if not (pd.isna(mean) or pd.isna(min_val) or pd.isna(max_val)):
+                pass
+            elif pd.isna(mean) and not (pd.isna(min_val) or pd.isna(max_val)):
+                mean = (min_val + max_val) / 2
+            else:
+                val_notna = [x for x in {mean, min_val, max_val} if not pd.isna(x)]
+                if len(val_notna) >= 1:
+                    mean_val = sum(val_notna) / len(val_notna)
+                    mean = max_val = min_val = mean_val
+                else:
+                    continue  # Do not consider a row with null data
+
+            # Normalize if the mean is upper than maxval or lower than minval
+            if (mean < min_val or mean > max_val) and col not in ["WD"]:
+                mean = (min_val + max_val) / 2
+
+            if col in ["WD"]:  # Avoid error
+                rows.append(
+                    {
+                        "statistic_id": statistic_id,
+                        "unit": unit,
+                        "start": start,
+                        "min": mean,
+                        "max": mean,
+                        "mean": mean,
+                    }
+                )
+            else:
+                rows.append(
+                    {
+                        "statistic_id": statistic_id,
+                        "unit": unit,
+                        "start": start,
+                        "min": min_val,
+                        "max": max_val,
+                        "mean": mean,
+                    }
+                )
+
+    logger.debug("[thies_synchronization_lib] Saving file in the main directory")
+    await csv_client.write(
+        WriteArgs(file_name="thies_daily_statistics.tsv", file_content=rows, mode="w")
+    )
+    logger.debug(
+        "[thies_synchronization_lib] thies_daily_statistics.tsv created successfully!"
+    )