saviialib 1.6.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. saviialib/__init__.py +79 -0
  2. saviialib/general_types/__init__.py +0 -0
  3. saviialib/general_types/api/__init__.py +0 -0
  4. saviialib/general_types/api/saviia_api_types.py +48 -0
  5. saviialib/general_types/api/saviia_backup_api_types.py +24 -0
  6. saviialib/general_types/api/saviia_netcamera_api_types.py +11 -0
  7. saviialib/general_types/api/saviia_shakes_api_types.py +21 -0
  8. saviialib/general_types/api/saviia_thies_api_types.py +31 -0
  9. saviialib/general_types/error_types/__init__.py +0 -0
  10. saviialib/general_types/error_types/api/__init__.py +0 -0
  11. saviialib/general_types/error_types/api/saviia_api_error_types.py +113 -0
  12. saviialib/general_types/error_types/api/saviia_netcamera_error_types.py +7 -0
  13. saviialib/general_types/error_types/common/__init__.py +7 -0
  14. saviialib/general_types/error_types/common/common_types.py +26 -0
  15. saviialib/libs/directory_client/__init__.py +4 -0
  16. saviialib/libs/directory_client/client/os_client.py +55 -0
  17. saviialib/libs/directory_client/directory_client.py +44 -0
  18. saviialib/libs/directory_client/directory_client_contract.py +40 -0
  19. saviialib/libs/directory_client/types/directory_client_types.py +6 -0
  20. saviialib/libs/files_client/__init__.py +4 -0
  21. saviialib/libs/files_client/clients/aiofiles_client.py +44 -0
  22. saviialib/libs/files_client/clients/csv_client.py +42 -0
  23. saviialib/libs/files_client/files_client.py +26 -0
  24. saviialib/libs/files_client/files_client_contract.py +13 -0
  25. saviialib/libs/files_client/types/files_client_types.py +32 -0
  26. saviialib/libs/ftp_client/__init__.py +4 -0
  27. saviialib/libs/ftp_client/clients/__init__.py +0 -0
  28. saviialib/libs/ftp_client/clients/aioftp_client.py +52 -0
  29. saviialib/libs/ftp_client/clients/ftplib_client.py +58 -0
  30. saviialib/libs/ftp_client/ftp_client.py +25 -0
  31. saviialib/libs/ftp_client/ftp_client_contract.py +13 -0
  32. saviialib/libs/ftp_client/types/__init__.py +3 -0
  33. saviialib/libs/ftp_client/types/ftp_client_types.py +18 -0
  34. saviialib/libs/log_client/__init__.py +19 -0
  35. saviialib/libs/log_client/log_client.py +46 -0
  36. saviialib/libs/log_client/log_client_contract.py +28 -0
  37. saviialib/libs/log_client/logging_client/logging_client.py +58 -0
  38. saviialib/libs/log_client/types/__init__.py +0 -0
  39. saviialib/libs/log_client/types/log_client_types.py +47 -0
  40. saviialib/libs/log_client/utils/log_client_utils.py +6 -0
  41. saviialib/libs/sftp_client/__init__.py +8 -0
  42. saviialib/libs/sftp_client/clients/asyncssh_sftp_client.py +83 -0
  43. saviialib/libs/sftp_client/sftp_client.py +26 -0
  44. saviialib/libs/sftp_client/sftp_client_contract.py +13 -0
  45. saviialib/libs/sftp_client/types/sftp_client_types.py +24 -0
  46. saviialib/libs/sharepoint_client/__init__.py +17 -0
  47. saviialib/libs/sharepoint_client/clients/sharepoint_rest_api.py +160 -0
  48. saviialib/libs/sharepoint_client/sharepoint_client.py +58 -0
  49. saviialib/libs/sharepoint_client/sharepoint_client_contract.py +26 -0
  50. saviialib/libs/sharepoint_client/types/sharepoint_client_types.py +30 -0
  51. saviialib/libs/zero_dependency/utils/booleans_utils.py +2 -0
  52. saviialib/libs/zero_dependency/utils/datetime_utils.py +25 -0
  53. saviialib/libs/zero_dependency/utils/strings_utils.py +5 -0
  54. saviialib/services/backup/__init__.py +0 -0
  55. saviialib/services/backup/api.py +36 -0
  56. saviialib/services/backup/controllers/__init__.py +0 -0
  57. saviialib/services/backup/controllers/types/__init__.py +6 -0
  58. saviialib/services/backup/controllers/types/upload_backup_to_sharepoint_types.py +18 -0
  59. saviialib/services/backup/controllers/upload_backup_to_sharepoint.py +87 -0
  60. saviialib/services/backup/use_cases/constants/upload_backup_to_sharepoint_constants.py +5 -0
  61. saviialib/services/backup/use_cases/types/__init__.py +7 -0
  62. saviialib/services/backup/use_cases/types/upload_backup_to_sharepoint_types.py +11 -0
  63. saviialib/services/backup/use_cases/upload_backup_to_sharepoint.py +474 -0
  64. saviialib/services/backup/utils/__init__.py +3 -0
  65. saviialib/services/backup/utils/upload_backup_to_sharepoint_utils.py +100 -0
  66. saviialib/services/netcamera/api.py +30 -0
  67. saviialib/services/netcamera/controllers/get_media_files.py +40 -0
  68. saviialib/services/netcamera/controllers/types/get_media_files_types.py +16 -0
  69. saviialib/services/netcamera/use_cases/get_media_files.py +76 -0
  70. saviialib/services/netcamera/use_cases/types/get_media_files_types.py +18 -0
  71. saviialib/services/shakes/__init__.py +0 -0
  72. saviialib/services/shakes/api.py +31 -0
  73. saviialib/services/shakes/controllers/get_miniseed_files.py +48 -0
  74. saviialib/services/shakes/controllers/types/get_miniseed_files_types.py +16 -0
  75. saviialib/services/shakes/use_cases/get_miniseed_files.py +79 -0
  76. saviialib/services/shakes/use_cases/types/get_miniseed_files_types.py +18 -0
  77. saviialib/services/shakes/use_cases/utils/get_miniseed_files_utils.py +11 -0
  78. saviialib/services/thies/__init__.py +0 -0
  79. saviialib/services/thies/api.py +42 -0
  80. saviialib/services/thies/constants/update_thies_data_constants.py +67 -0
  81. saviialib/services/thies/controllers/types/update_thies_data_types.py +18 -0
  82. saviialib/services/thies/controllers/update_thies_data.py +119 -0
  83. saviialib/services/thies/use_cases/components/create_thies_statistics_file.py +115 -0
  84. saviialib/services/thies/use_cases/components/thies_bp.py +442 -0
  85. saviialib/services/thies/use_cases/types/update_thies_data_types.py +24 -0
  86. saviialib/services/thies/use_cases/update_thies_data.py +391 -0
  87. saviialib/services/thies/utils/update_thies_data_utils.py +21 -0
  88. saviialib-1.6.1.dist-info/METADATA +126 -0
  89. saviialib-1.6.1.dist-info/RECORD +91 -0
  90. saviialib-1.6.1.dist-info/WHEEL +4 -0
  91. saviialib-1.6.1.dist-info/licenses/LICENSE +22 -0
@@ -0,0 +1,76 @@
1
+ from .types.get_media_files_types import (
2
+ GetMediaFilesUseCaseInput,
3
+ GetMediaFilesUseCaseOutput,
4
+ )
5
+ from saviialib.libs.ffmpeg_client import (
6
+ FfmpegClient,
7
+ FfmpegClientInitArgs,
8
+ RecordVideoArgs,
9
+ RecordPhotoArgs,
10
+ )
11
+ from saviialib.libs.zero_dependency.utils.strings_utils import are_equal
12
+ from typing import Tuple, Dict
13
+ from saviialib.libs.directory_client.directory_client import (
14
+ DirectoryClient,
15
+ DirectoryClientArgs,
16
+ )
17
+ from saviialib.general_types.error_types.api.saviia_netcamera_error_types import (
18
+ NetcameraConnectionError,
19
+ )
20
+
21
+
22
+ class GetMediaFilesUseCase:
23
+ def __init__(self, input: GetMediaFilesUseCaseInput) -> None:
24
+ self.ffmpeg_client = FfmpegClient(
25
+ FfmpegClientInitArgs(
26
+ client_name="ffmpeg_asyncio",
27
+ )
28
+ )
29
+ self.dir_client = DirectoryClient(DirectoryClientArgs("os_client"))
30
+ self.user = input.username
31
+ self.pwd = input.password
32
+ self.cameras: Dict[str, Tuple[str, int]] = input.cameras
33
+ self.protocol = input.protocol
34
+ self.logger = input.logger
35
+ self.dest_path = input.destination_path
36
+
37
+ async def _retieve_with_rtsp(self):
38
+ for name, conn in self.cameras.items():
39
+ ip, port = conn
40
+ dest_path = self.dir_client.join_paths(self.dest_path, name)
41
+ try:
42
+ # Extraction of photo files into dest_path dir.
43
+ await self.ffmpeg_client.record_photo(
44
+ RecordPhotoArgs(
45
+ ip_address=ip,
46
+ port=str(port),
47
+ destination_path=dest_path,
48
+ rtsp_user=self.user,
49
+ rtsp_password=self.pwd,
50
+ extension="jpg",
51
+ frames=1,
52
+ )
53
+ )
54
+ # Extraction of video files into dest_path dir.
55
+ await self.ffmpeg_client.record_video(
56
+ RecordVideoArgs(
57
+ destination_path=dest_path,
58
+ ip_address=ip,
59
+ port=str(port),
60
+ rtsp_user=self.user,
61
+ rtsp_password=self.pwd,
62
+ extension="mp3",
63
+ duration=10,
64
+ )
65
+ )
66
+ except ConnectionError as error:
67
+ raise NetcameraConnectionError(reason=error)
68
+
69
+ async def execute(self) -> GetMediaFilesUseCaseOutput:
70
+ if are_equal(self.protocol, "rtsp"):
71
+ await self._retieve_with_rtsp()
72
+ else:
73
+ raise NotImplementedError(
74
+ f"The media files extraction with {self.protocol} is not implemented yet."
75
+ )
76
+ return GetMediaFilesUseCaseOutput()
@@ -0,0 +1,18 @@
1
+ from dataclasses import dataclass
2
+ from logging import Logger
3
+ from typing import Dict, Tuple
4
+
5
+
6
+ @dataclass
7
+ class GetMediaFilesUseCaseInput:
8
+ cameras: Dict[str, Tuple[str, int]]
9
+ username: str
10
+ password: str
11
+ protocol: str
12
+ logger: Logger
13
+ destination_path: str
14
+
15
+
16
+ @dataclass
17
+ class GetMediaFilesUseCaseOutput:
18
+ pass
File without changes
@@ -0,0 +1,31 @@
1
+ from .controllers.get_miniseed_files import (
2
+ GetMiniseedFilesController,
3
+ GetMiniseedFilesControllerInput,
4
+ )
5
+ from saviialib.general_types.api.saviia_shakes_api_types import SaviiaShakesConfig
6
+
7
+ from typing import Dict
8
+
9
+
10
+ class ShakesAPI:
11
+ """This class provides methods for interacting with Raspberry Shakes"""
12
+
13
+ def __init__(self, config: SaviiaShakesConfig) -> None:
14
+ self.config = config
15
+
16
+ async def get_miniseed_files(self, raspberry_shakes: Dict[str, str]):
17
+ """Download the MiniSEED files from the SFTP Server provided by each Raspberry Shake.
18
+ Args:
19
+ raspberry_shakes (dict): Dictionary where the key is the name of the Raspberry Shake,
20
+ and the value is the IP Address.
21
+ Returns:
22
+ response (dict): A dictionary containg the response from the download operation.
23
+ This response will tipically include the message, the response status, and metadata.
24
+ """
25
+ controller = GetMiniseedFilesController(
26
+ GetMiniseedFilesControllerInput(
27
+ config=self.config, raspberry_shakes=raspberry_shakes
28
+ )
29
+ )
30
+ response = await controller.execute()
31
+ return response.__dict__
@@ -0,0 +1,48 @@
1
+ from .types.get_miniseed_files_types import (
2
+ GetMiniseedFilesControllerInput,
3
+ GetMiniseedFilesControllerOutput,
4
+ )
5
+ from saviialib.services.shakes.use_cases.get_miniseed_files import (
6
+ GetMiniseedFilesUseCase,
7
+ GetMiniseedFilesUseCaseInput,
8
+ )
9
+ from http import HTTPStatus
10
+ from saviialib.general_types.error_types.api.saviia_api_error_types import (
11
+ ShakesNoContentError,
12
+ )
13
+ from saviialib.general_types.error_types.common.common_types import SftpClientError
14
+
15
+
16
+ class GetMiniseedFilesController:
17
+ def __init__(self, input: GetMiniseedFilesControllerInput) -> None:
18
+ self.use_case = GetMiniseedFilesUseCase(
19
+ GetMiniseedFilesUseCaseInput(
20
+ raspberry_shakes=input.raspberry_shakes,
21
+ username=input.config.sftp_user,
22
+ password=input.config.sftp_password,
23
+ ssh_key_path=input.config.ssh_key_path,
24
+ port=input.config.sftp_port,
25
+ logger=input.config.logger,
26
+ )
27
+ )
28
+
29
+ async def execute(self) -> GetMiniseedFilesControllerOutput:
30
+ try:
31
+ res = await self.use_case.execute()
32
+ return GetMiniseedFilesControllerOutput(
33
+ message="The MiniSEED files have been downloaded succesfully!",
34
+ status=HTTPStatus.OK.value,
35
+ metadata=res.download_status,
36
+ )
37
+ except ShakesNoContentError as error:
38
+ return GetMiniseedFilesControllerOutput(
39
+ message="No files to upload.",
40
+ status=HTTPStatus.NO_CONTENT.value,
41
+ metadata={"error": error.__str__()},
42
+ )
43
+ except SftpClientError as error:
44
+ return GetMiniseedFilesControllerOutput(
45
+ message="An unexpected error ocurred during SFTP Client connection.",
46
+ status=HTTPStatus.REQUEST_TIMEOUT.value,
47
+ metadata={"error": error.__str__()},
48
+ )
@@ -0,0 +1,16 @@
1
+ from dataclasses import dataclass
2
+ from typing import Dict
3
+ from saviialib.general_types.api.saviia_shakes_api_types import SaviiaShakesConfig
4
+
5
+
6
+ @dataclass
7
+ class GetMiniseedFilesControllerInput:
8
+ raspberry_shakes: Dict[str, str]
9
+ config: SaviiaShakesConfig
10
+
11
+
12
+ @dataclass
13
+ class GetMiniseedFilesControllerOutput:
14
+ status: int
15
+ metadata: Dict
16
+ message: str
@@ -0,0 +1,79 @@
1
+ import asyncio
2
+ from .types.get_miniseed_files_types import (
3
+ GetMiniseedFilesUseCaseInput,
4
+ GetMiniseedFilesUseCaseOutput,
5
+ )
6
+ from typing import Dict, Any
7
+ from saviialib.libs.sftp_client import (
8
+ SFTPClient,
9
+ SFTPClientInitArgs,
10
+ ListfilesArgs,
11
+ DownloadfilesArgs,
12
+ )
13
+ from saviialib.libs.directory_client import DirectoryClient, DirectoryClientArgs
14
+ from saviialib.general_types.error_types.api.saviia_api_error_types import (
15
+ ShakesNoContentError,
16
+ )
17
+ from saviialib.general_types.error_types.common.common_types import SftpClientError
18
+ from .utils.get_miniseed_files_utils import parse_downloaded_metadata
19
+
20
+
21
+ class GetMiniseedFilesUseCase:
22
+ def __init__(self, input: GetMiniseedFilesUseCaseInput) -> None:
23
+ self.password = input.password
24
+ self.username = input.username
25
+ self.ssh_key_path = input.ssh_key_path
26
+ self.port = input.port
27
+ self.raspberry_shakes: Dict[str, str] = input.raspberry_shakes
28
+ self.dir_client = DirectoryClient(DirectoryClientArgs("os_client"))
29
+
30
+ def _initialize_sftp_client(self, ip_address: str):
31
+ return SFTPClient(
32
+ SFTPClientInitArgs(
33
+ "asyncssh_sftp",
34
+ password=self.password,
35
+ username=self.username,
36
+ ssh_key_path=self.ssh_key_path,
37
+ host=ip_address,
38
+ port=self.port,
39
+ )
40
+ )
41
+
42
+ async def _download_mseed_file(self, rs_name: str, rs_ip: str) -> Dict[str, Any]:
43
+ DEST_BASE_DIR = "./rshakes-mseed-files"
44
+ SOURCE_PATH = self.dir_client.join_paths("opt", "data", "archive")
45
+ local_path = self.dir_client.join_paths(DEST_BASE_DIR, rs_name)
46
+ if not await self.dir_client.isdir(local_path):
47
+ await self.dir_client.makedirs(local_path)
48
+ sftp_client = self._initialize_sftp_client(rs_ip)
49
+
50
+ local_files = await self.dir_client.listdir(local_path)
51
+ try:
52
+ sftp_files = await sftp_client.list_files(ListfilesArgs(path=SOURCE_PATH))
53
+ pending_files = set(sftp_files) - set(local_files)
54
+ if not pending_files:
55
+ raise ShakesNoContentError
56
+
57
+ await sftp_client.download_files(
58
+ DownloadfilesArgs(
59
+ source_path=SOURCE_PATH,
60
+ destination_path=local_path,
61
+ files_to_download=list(pending_files),
62
+ )
63
+ )
64
+ except ConnectionError as error:
65
+ raise SftpClientError(reason=error)
66
+ return {
67
+ "rs_name": rs_name,
68
+ "destination_path": local_path,
69
+ "total_files": len(pending_files),
70
+ }
71
+
72
+ async def execute(self):
73
+ requests = []
74
+ for rs_name, rs_ip in self.raspberry_shakes:
75
+ requests.append(self._download_mseed_file(rs_name, rs_ip))
76
+ responses = await asyncio.gather(*requests, return_exceptions=True)
77
+ return GetMiniseedFilesUseCaseOutput(
78
+ download_status=parse_downloaded_metadata(responses),
79
+ )
@@ -0,0 +1,18 @@
1
+ from dataclasses import dataclass, field
2
+ from typing import Dict
3
+ from logging import Logger
4
+
5
+
6
+ @dataclass
7
+ class GetMiniseedFilesUseCaseInput:
8
+ raspberry_shakes: Dict[str, str]
9
+ username: str
10
+ password: str
11
+ ssh_key_path: str
12
+ port: int
13
+ logger: Logger
14
+
15
+
16
+ @dataclass
17
+ class GetMiniseedFilesUseCaseOutput:
18
+ download_status: Dict[str, str] = field(default_factory=dict)
@@ -0,0 +1,11 @@
1
+ from typing import Dict, List, Any
2
+
3
+
4
+ def parse_downloaded_metadata(responses: List[Any]) -> Dict[str, Any]:
5
+ return {
6
+ x["rs_name"]: {
7
+ "destination_path": x["destination_path"],
8
+ "total_files": x["total_files"],
9
+ }
10
+ for x in responses
11
+ }
File without changes
@@ -0,0 +1,42 @@
1
+ from typing import Any, Dict, List
2
+
3
+ from .controllers.types.update_thies_data_types import UpdateThiesDataControllerInput
4
+ from .controllers.update_thies_data import UpdateThiesDataController
5
+ from saviialib.general_types.api.saviia_thies_api_types import (
6
+ SaviiaThiesConfig,
7
+ )
8
+
9
+
10
+ class SaviiaThiesAPI:
11
+ def __init__(self, config: SaviiaThiesConfig) -> None:
12
+ self.config = config
13
+
14
+ async def update_thies_data(
15
+ self,
16
+ sharepoint_folders_path: List[str],
17
+ ftp_server_folders_path: List[str],
18
+ local_backup_source_path: str,
19
+ ) -> Dict[str, Any]:
20
+ """Updates data from a THIES Data Logger by connecting to an FTP server
21
+ and transferring data to specified Sharepoint folders.
22
+
23
+ :param list sharepoint_folders_path: List of Sharepoint folder paths for AVG and EXT data.
24
+ The AVG path must be the first element.
25
+ :param list ftp_server_folders_path: List of FTP server folder paths for AVG and EXT data.
26
+ The AVG path must be the first element.
27
+ :param str local_backup_source_path: Path of the main directory where the files extracted from
28
+ the Thies FTP Server are going to be stored
29
+
30
+ :return: A dictionary representation of the API response.
31
+ :rtype: dict
32
+ """
33
+ controller = UpdateThiesDataController(
34
+ UpdateThiesDataControllerInput(
35
+ self.config,
36
+ sharepoint_folders_path,
37
+ ftp_server_folders_path,
38
+ local_backup_source_path,
39
+ )
40
+ )
41
+ response = await controller.execute()
42
+ return response.__dict__
@@ -0,0 +1,67 @@
1
+ AVG_COLUMNS = {
2
+ "Date": "date",
3
+ "Time": "time",
4
+ "AirTemperature": "air_temperature",
5
+ "Radiation": "radiation",
6
+ "CO2": "carbon_dioxide",
7
+ "Precipitation": "precipitation",
8
+ "WS": "wind_velocity",
9
+ "WD": "wind_direction",
10
+ "Humidity": "humidity",
11
+ "UBat": "battery",
12
+ }
13
+
14
+ EXT_COLUMNS = {
15
+ "Date": "date",
16
+ "Time": "time",
17
+ "AirTemperature MIN": "air_temperature",
18
+ "AirTemperature MAX": "air_temperature",
19
+ "Radiation MIN": "radiation",
20
+ "Radiation MAX": "radiation",
21
+ "CO2 MIN": "carbon_dioxide",
22
+ "CO2 MAX": "carbon_dioxide",
23
+ "WS MIN": "wind_velocity",
24
+ "WS MAX gust": "wind_velocity",
25
+ "WD MIN": "wind_direction",
26
+ "WD MAX gust": "wind_direction",
27
+ "Humidity MIN": "humidity",
28
+ "Humidity MAX": "humidity",
29
+ "UBat MIN": "battery",
30
+ "UBat MAX": "battery",
31
+ }
32
+
33
+ AGG_DICT = {
34
+ "AirTemperature": "mean",
35
+ "AirTemperature MIN": "mean",
36
+ "AirTemperature MAX": "mean",
37
+ "Precipitation": "sum",
38
+ "Humidity": "mean",
39
+ "Humidity MIN": "mean",
40
+ "Humidity MAX": "mean",
41
+ "Radiation": "sum",
42
+ "Radiation MIN": "sum",
43
+ "Radiation MAX": "sum",
44
+ "CO2": "mean",
45
+ "CO2 MIN": "mean",
46
+ "CO2 MAX": "mean",
47
+ "WS": "mean",
48
+ "WS MIN": "mean",
49
+ "WS MAX gust": "mean",
50
+ "WD": "mean",
51
+ "WD MIN": "mean",
52
+ "WD MAX gust": "mean",
53
+ "UBat": "mean",
54
+ "UBat MIN": "mean",
55
+ "UBat MAX": "mean",
56
+ }
57
+
58
+ UNITS = {
59
+ "AirTemperature": "°C",
60
+ "Precipitation": "mm",
61
+ "Humidity": "%",
62
+ "Radiation": "W/m²",
63
+ "CO2": "ppm",
64
+ "WS": "m/s",
65
+ "WD": "°",
66
+ "UBat": "V",
67
+ }
@@ -0,0 +1,18 @@
1
+ from dataclasses import dataclass, field
2
+ from typing import Dict
3
+ from saviialib.general_types.api.saviia_thies_api_types import SaviiaThiesConfig
4
+
5
+
6
+ @dataclass
7
+ class UpdateThiesDataControllerInput:
8
+ config: SaviiaThiesConfig
9
+ sharepoint_folders_path: list
10
+ ftp_server_folders_path: list
11
+ local_backup_source_path: str
12
+
13
+
14
+ @dataclass
15
+ class UpdateThiesDataControllerOutput:
16
+ message: str
17
+ status: int
18
+ metadata: Dict[str, str] = field(default_factory=dict)
@@ -0,0 +1,119 @@
1
+ from http import HTTPStatus
2
+
3
+ from saviialib.general_types.error_types.api.saviia_api_error_types import (
4
+ SharePointFetchingError,
5
+ ThiesConnectionError,
6
+ ThiesFetchingError,
7
+ SharePointUploadError,
8
+ SharePointDirectoryError,
9
+ )
10
+ from saviialib.general_types.error_types.common.common_types import (
11
+ EmptyDataError,
12
+ FtpClientError,
13
+ SharepointClientError,
14
+ )
15
+ from saviialib.services.thies.controllers.types.update_thies_data_types import (
16
+ UpdateThiesDataControllerInput,
17
+ UpdateThiesDataControllerOutput,
18
+ )
19
+ from saviialib.services.backup.use_cases.types import (
20
+ UpdateThiesDataUseCaseInput,
21
+ SharepointConfig,
22
+ FtpClientConfig,
23
+ )
24
+ from saviialib.services.thies.use_cases.update_thies_data import (
25
+ UpdateThiesDataUseCase,
26
+ )
27
+
28
+
29
+ class UpdateThiesDataController:
30
+ def __init__(self, input: UpdateThiesDataControllerInput):
31
+ self.use_case = UpdateThiesDataUseCase(
32
+ UpdateThiesDataUseCaseInput(
33
+ ftp_config=FtpClientConfig(
34
+ ftp_host=input.config.ftp_host,
35
+ ftp_password=input.config.ftp_password,
36
+ ftp_port=input.config.ftp_port,
37
+ ftp_user=input.config.ftp_user,
38
+ ),
39
+ sharepoint_config=SharepointConfig(
40
+ sharepoint_client_id=input.config.sharepoint_client_id,
41
+ sharepoint_client_secret=input.config.sharepoint_client_secret,
42
+ sharepoint_site_name=input.config.sharepoint_site_name,
43
+ sharepoint_tenant_name=input.config.sharepoint_tenant_name,
44
+ sharepoint_tenant_id=input.config.sharepoint_tenant_id,
45
+ ),
46
+ sharepoint_folders_path=input.sharepoint_folders_path,
47
+ ftp_server_folders_path=input.ftp_server_folders_path,
48
+ local_backup_source_path=input.local_backup_source_path,
49
+ logger=input.config.logger,
50
+ )
51
+ )
52
+
53
+ async def execute(self) -> UpdateThiesDataControllerOutput:
54
+ try:
55
+ data = await self.use_case.execute()
56
+ return UpdateThiesDataControllerOutput(
57
+ message="THIES was synced successfully",
58
+ status=HTTPStatus.OK.value,
59
+ metadata={"data": data}, # type: ignore
60
+ )
61
+ except EmptyDataError:
62
+ return UpdateThiesDataControllerOutput(
63
+ message="No files to upload", status=HTTPStatus.NO_CONTENT.value
64
+ )
65
+
66
+ except (AttributeError, NameError, ValueError) as error:
67
+ return UpdateThiesDataControllerOutput(
68
+ message="An unexpected error occurred during use case initialization.",
69
+ status=HTTPStatus.BAD_REQUEST.value,
70
+ metadata={"error": error.__str__()},
71
+ )
72
+ except FtpClientError as error:
73
+ return UpdateThiesDataControllerOutput(
74
+ message="Ftp Client initialization fails.",
75
+ status=HTTPStatus.BAD_REQUEST.value,
76
+ metadata={"error": error.__str__()},
77
+ )
78
+
79
+ except SharepointClientError as error:
80
+ return UpdateThiesDataControllerOutput(
81
+ message="Sharepoint Client initialization fails.",
82
+ status=HTTPStatus.INTERNAL_SERVER_ERROR.value,
83
+ metadata={"error": error.__str__()},
84
+ )
85
+
86
+ except SharePointFetchingError as error:
87
+ return UpdateThiesDataControllerOutput(
88
+ message="An error occurred while retrieving file names from Microsoft SharePoint",
89
+ status=HTTPStatus.BAD_REQUEST.value,
90
+ metadata={"error": error.__str__()},
91
+ )
92
+
93
+ except SharePointUploadError as error:
94
+ return UpdateThiesDataControllerOutput(
95
+ message="An error ocurred while uploading files to RCER Cloud",
96
+ status=HTTPStatus.BAD_REQUEST.value,
97
+ metadata={"error": error.__str__()},
98
+ )
99
+
100
+ except SharePointDirectoryError as error:
101
+ return UpdateThiesDataControllerOutput(
102
+ message="An error ocurred while extracting folders from Microsoft Sharepoint",
103
+ status=HTTPStatus.BAD_REQUEST.value,
104
+ metadata={"error": error.__str__()},
105
+ )
106
+
107
+ except ThiesFetchingError as error:
108
+ return UpdateThiesDataControllerOutput(
109
+ message="An error ocurred while retrieving file names from THIES FTP Server.",
110
+ status=HTTPStatus.NO_CONTENT.value,
111
+ metadata={"error": error.__str__()},
112
+ )
113
+
114
+ except ThiesConnectionError as error:
115
+ return UpdateThiesDataControllerOutput(
116
+ message="Unable to connect to THIES Data Logger FTP Server.",
117
+ status=HTTPStatus.INTERNAL_SERVER_ERROR.value,
118
+ metadata={"error": error.__str__()},
119
+ )
@@ -0,0 +1,115 @@
1
+ from .thies_bp import THIESDayData
2
+ import pandas as pd
3
+ from logging import Logger
4
+ from asyncio import to_thread
5
+ from saviialib.libs.directory_client import DirectoryClient
6
+ from saviialib.libs.zero_dependency.utils.datetime_utils import datetime_to_str, today
7
+ from saviialib.libs.files_client import FilesClient, FilesClientInitArgs, WriteArgs
8
+ import saviialib.services.thies.constants.update_thies_data_constants as c
9
+
10
+
11
+ async def create_thies_daily_statistics_file(
12
+ local_backup_path: str, os_client: DirectoryClient, logger: Logger
13
+ ) -> None:
14
+ csv_client = FilesClient(FilesClientInitArgs(client_name="csv_client"))
15
+ filename = datetime_to_str(today(), date_format="%Y%m%d") + ".BIN"
16
+ logger.debug(
17
+ f"[thies_synchronization_lib] Creating Daily Statistics for {filename}"
18
+ )
19
+ path_bin_av = os_client.join_paths(local_backup_path, "thies", "AVG", filename)
20
+ path_ini_av = os_client.join_paths(
21
+ local_backup_path, "thies", "AVG", "DESCFILE.INI"
22
+ )
23
+ path_bin_ex = os_client.join_paths(local_backup_path, "thies", "EXT", filename)
24
+ path_ini_ex = os_client.join_paths(
25
+ local_backup_path, "thies", "EXT", "DESCFILE.INI"
26
+ )
27
+
28
+ ext_df = THIESDayData("ex")
29
+ await to_thread(ext_df.read_binfile, path_bin_ex, path_ini_ex)
30
+
31
+ avg_df = THIESDayData("av")
32
+ await to_thread(avg_df.read_binfile, path_bin_av, path_ini_av)
33
+
34
+ ext_df = ext_df.dataDF[c.EXT_COLUMNS.keys()]
35
+ avg_df = avg_df.dataDF[c.AVG_COLUMNS.keys()]
36
+
37
+ # Merge both dataframes
38
+ df = avg_df.merge(ext_df, on=["Date", "Time"], how="outer")
39
+ # Set the date as dd.mm.yyyy format.
40
+ df["Date"] = df["Date"].str.replace(
41
+ r"(\d{4})/(\d{2})/(\d{2})", r"\3.\2.\1", regex=True
42
+ )
43
+ df["Hour"] = df["Time"].str[:2]
44
+
45
+ # Group by hour.
46
+ hourly_agg = df.groupby(["Date", "Hour"]).agg(c.AGG_DICT).reset_index()
47
+
48
+ rows = []
49
+ # For each attribute in avg_columns (except Date, Time)
50
+ for col, col_id in c.AVG_COLUMNS.items():
51
+ if col in ["Date", "Time"]:
52
+ continue
53
+ # Determine the corresponding min/max columns if they exist
54
+ min_col = f"{col} MIN"
55
+ max_col = f"{col} MAX"
56
+ mean_col = col
57
+ if col in ["WS", "WD"]:
58
+ max_col += " gust"
59
+
60
+ unit = c.UNITS.get(col, "")
61
+
62
+ for idx, row in hourly_agg.iterrows():
63
+ statistic_id = f"sensor.saviia_epii_{col_id}"
64
+ start = f"{row['Date']} {row['Hour']}:00"
65
+ mean = row[mean_col] if mean_col in row else 0
66
+ min_val = row[min_col] if min_col in row else mean
67
+ max_val = row[max_col] if max_col in row else mean
68
+
69
+ # If no min/max for this attribute, set as Na or 0 as requested
70
+ if not (pd.isna(mean) or pd.isna(min_val) or pd.isna(max_val)):
71
+ pass
72
+ elif pd.isna(mean) and not (pd.isna(min_val) or pd.isna(max_val)):
73
+ mean = (min_val + max_val) / 2
74
+ else:
75
+ val_notna = [x for x in {mean, min_val, max_val} if not pd.isna(x)]
76
+ if len(val_notna) >= 1:
77
+ mean_val = sum(val_notna) / len(val_notna)
78
+ mean = max_val = min_val = mean_val
79
+ else:
80
+ continue # Do not consider a row with null data
81
+
82
+ # Normalize if the mean is upper than maxval or lower than minval
83
+ if (mean < min_val or mean > max_val) and col not in ["WD"]:
84
+ mean = (min_val + max_val) / 2
85
+
86
+ if col in ["WD"]: # Avoid error
87
+ rows.append(
88
+ {
89
+ "statistic_id": statistic_id,
90
+ "unit": unit,
91
+ "start": start,
92
+ "min": mean,
93
+ "max": mean,
94
+ "mean": mean,
95
+ }
96
+ )
97
+ else:
98
+ rows.append(
99
+ {
100
+ "statistic_id": statistic_id,
101
+ "unit": unit,
102
+ "start": start,
103
+ "min": min_val,
104
+ "max": max_val,
105
+ "mean": mean,
106
+ }
107
+ )
108
+
109
+ logger.debug("[thies_synchronization_lib] Saving file in the main directory")
110
+ await csv_client.write(
111
+ WriteArgs(file_name="thies_daily_statistics.tsv", file_content=rows, mode="w")
112
+ )
113
+ logger.debug(
114
+ "[thies_synchronization_lib] thies_daily_statistics.tsv created successfully!"
115
+ )