saviialib 0.9.1__py3-none-any.whl → 1.6.0__py3-none-any.whl

saviialib/services/epii/api.py

@@ -1,80 +0,0 @@
-from typing import Any, Dict
-
-from .controllers.types.update_thies_data_types import UpdateThiesDataControllerInput
-from .controllers.types.upload_backup_to_sharepoint_types import (
-    UploadBackupToSharepointControllerInput,
-)
-from .controllers.update_thies_data import UpdateThiesDataController
-from .controllers.upload_backup_to_sharepoint import UploadBackupToSharepointController
-from saviialib.general_types.api.epii_api_types import (
-    EpiiUpdateThiesConfig,
-    EpiiSharepointBackupConfig,
-    EpiiAPIConfig,
-)
-
-
-class EpiiAPI:
-    """
-    EpiiAPI is a service class that provides methods to interact with Patagonia Center system.
-    """
-
-    def __init__(self, config: EpiiAPIConfig):
-        self.ftp_port = config.ftp_port
-        self.ftp_host = config.ftp_host
-        self.ftp_user = config.ftp_user
-        self.ftp_password = config.ftp_password
-        self.sharepoint_client_id = config.sharepoint_client_id
-        self.sharepoint_client_secret = config.sharepoint_client_secret
-        self.sharepoint_tenant_id = config.sharepoint_tenant_id
-        self.sharepoint_tenant_name = config.sharepoint_tenant_name
-        self.sharepoint_site_name = config.sharepoint_site_name
-
-    async def update_thies_data(self) -> Dict[str, Any]:
-        """
-        This method establishes a connection to an FTP server using the provided
-        credentials and updates data related to THIES Data Logger.
-
-        Returns:
-            response (dict): A dictionary representation of the API response.
-        """
-        config = EpiiUpdateThiesConfig(
-            ftp_port=self.ftp_port,
-            ftp_host=self.ftp_host,
-            ftp_user=self.ftp_user,
-            ftp_password=self.ftp_password,
-            sharepoint_client_id=self.sharepoint_client_id,
-            sharepoint_client_secret=self.sharepoint_client_secret,
-            sharepoint_site_name=self.sharepoint_site_name,
-            sharepoint_tenant_id=self.sharepoint_tenant_id,
-            sharepoint_tenant_name=self.sharepoint_tenant_name,
-        )
-        controller = UpdateThiesDataController(UpdateThiesDataControllerInput(config))
-        response = await controller.execute()
-        return response.__dict__
-
-    async def upload_backup_to_sharepoint(
-        self, local_backup_source_path: str, destination_folders: dict[str, str]
-    ) -> Dict[str, Any]:
-        """Migrate a backup folder from Home assistant to Sharepoint directory.
-        Args:
-            local_backup_source_path (str): Local path to backup.
-        Returns:
-            response (dict): A dictionary containing the response from the upload operation.
-                This dictionary will typically include information about the success or
-                failure of the upload, as well as any relevant metadata.
-        """
-        config = EpiiSharepointBackupConfig(
-            sharepoint_client_id=self.sharepoint_client_id,
-            sharepoint_client_secret=self.sharepoint_client_secret,
-            sharepoint_site_name=self.sharepoint_site_name,
-            sharepoint_tenant_id=self.sharepoint_tenant_id,
-            sharepoint_tenant_name=self.sharepoint_tenant_name,
-            local_backup_source_path=local_backup_source_path,
-            destination_folders=destination_folders
-        )
-
-        controller = UploadBackupToSharepointController(
-            UploadBackupToSharepointControllerInput(config)
-        )
-        response = await controller.execute()
-        return response.__dict__

saviialib/services/epii/use_cases/constants/update_thies_data_constants.py

@@ -1,5 +0,0 @@
-SHAREPOINT_BASE_URL = "/sites/uc365_CentrosyEstacionesRegionalesUC/Shared%20Documents/General/Test_Raspberry/THIES"
-SHAREPOINT_THIES_FOLDERS = ["AVG", "EXT"]
-
-FTP_SERVER_PATH_AVG_FILES = "ftp/thies/BINFILES/ARCH_AV1"
-FTP_SERVER_PATH_EXT_FILES = "ftp/thies/BINFILES/ARCH_EX1"

saviialib/services/epii/use_cases/constants/upload_backup_to_sharepoint_constants.py

@@ -1,5 +0,0 @@
-import logging
-
-SHAREPOINT_BASE_URL = "/sites/uc365_CentrosyEstacionesRegionalesUC/Shared%20Documents/General/Test_Raspberry"
-
-LOGGER = logging.getLogger(__package__)

saviialib/services/epii/use_cases/update_thies_data.py

@@ -1,171 +0,0 @@
-import saviialib.services.epii.use_cases.constants.update_thies_data_constants as c
-from saviialib.general_types.error_types.api.epii_api_error_types import (
-    SharePointFetchingError,
-    SharePointUploadError,
-    ThiesConnectionError,
-    ThiesFetchingError,
-)
-from saviialib.general_types.error_types.common import (
-    EmptyDataError,
-    FtpClientError,
-    SharepointClientError,
-)
-from saviialib.libs.ftp_client import (
-    FTPClient,
-    FtpClientInitArgs,
-    FtpListFilesArgs,
-    FtpReadFileArgs,
-)
-from saviialib.libs.sharepoint_client import (
-    SharepointClient,
-    SharepointClientInitArgs,
-    SpListFilesArgs,
-    SpUploadFileArgs,
-)
-from saviialib.services.epii.use_cases.types import (
-    FtpClientConfig,
-    SharepointConfig,
-    UpdateThiesDataUseCaseInput,
-)
-from saviialib.services.epii.utils import (
-    parse_execute_response,
-)
-
-
-class UpdateThiesDataUseCase:
-    def __init__(self, input: UpdateThiesDataUseCaseInput):
-        self.sharepoint_client = self._initialize_sharepoint_client(
-            input.sharepoint_config
-        )
-        self.thies_ftp_client = self._initialize_thies_ftp_client(input.ftp_config)
-        self.uploading = set()
-
-    def _initialize_sharepoint_client(
-        self, config: SharepointConfig
-    ) -> SharepointClient:
-        """Initialize the HTTP client."""
-        try:
-            return SharepointClient(
-                SharepointClientInitArgs(config, client_name="sharepoint_rest_api")
-            )
-        except ConnectionError as error:
-            raise SharepointClientError(error)
-
-    def _initialize_thies_ftp_client(self, config: FtpClientConfig) -> FTPClient:
-        """Initialize the FTP client."""
-        try:
-            return FTPClient(FtpClientInitArgs(config, client_name="aioftp_client"))
-        except RuntimeError as error:
-            raise FtpClientError(error)
-
-    async def fetch_cloud_file_names(self) -> set[str]:
-        """Fetch file names from the RCER cloud."""
-
-        try:
-            cloud_files = set()
-            async with self.sharepoint_client:
-                for folder in c.SHAREPOINT_THIES_FOLDERS:
-                    args = SpListFilesArgs(
-                        folder_relative_url=f"{c.SHAREPOINT_BASE_URL}/{folder}"
-                    )
-                    response = await self.sharepoint_client.list_files(args)
-                    cloud_files.update(
-                        {f"{folder}_{item['Name']}" for item in response["value"]}
-                    )
-            return cloud_files
-        except ConnectionError as error:
-            raise SharePointFetchingError(reason=error)
-
-    async def fetch_thies_file_names(self) -> set[str]:
-        """Fetch file names from the THIES FTP server."""
-        try:
-            avg_files = await self.thies_ftp_client.list_files(
-                FtpListFilesArgs(path=c.FTP_SERVER_PATH_AVG_FILES)
-            )
-            ext_files = await self.thies_ftp_client.list_files(
-                FtpListFilesArgs(path=c.FTP_SERVER_PATH_EXT_FILES)
-            )
-            return {f"AVG_{name}" for name in avg_files} | {
-                f"EXT_{name}" for name in ext_files
-            }
-        except ConnectionRefusedError as error:
-            raise ThiesConnectionError(reason=error)
-        except ConnectionAbortedError as error:
-            raise ThiesFetchingError(reason=error)
-
-    async def fetch_thies_file_content(self) -> dict[str, bytes]:
-        """Fetch the content of files from the THIES FTP server."""
-        try:
-            content_files = {}
-            for file in self.uploading:
-                origin, filename = file.split("_", 1)
-                file_path = (
-                    f"{c.FTP_SERVER_PATH_AVG_FILES}/{filename}"
-                    if origin == "AVG"
-                    else f"{c.FTP_SERVER_PATH_EXT_FILES}/{filename}"
-                )
-                content = await self.thies_ftp_client.read_file(
-                    FtpReadFileArgs(file_path)
-                )
-                content_files[file] = content  # Save the file with its prefix
-            return content_files
-        except ConnectionRefusedError as error:
-            raise ThiesConnectionError(reason=error)
-        except ConnectionAbortedError as error:
-            raise ThiesFetchingError(reason=error)
-
-    async def upload_thies_files_to_sharepoint(
-        self, files: dict
-    ) -> dict[str, list[str]]:
-        """Upload files to SharePoint and categorize the results."""
-        upload_results = {"failed_files": [], "new_files": []}
-
-        async with self.sharepoint_client:
-            for file, file_content in files.items():
-                try:
-                    folder, file_name = file.split("_", 1)
-                    args = SpUploadFileArgs(
-                        folder_relative_url=f"{c.SHAREPOINT_BASE_URL}/{folder}",
-                        file_content=file_content,
-                        file_name=file_name,
-                    )
-                    await self.sharepoint_client.upload_file(args)
-                    upload_results["new_files"].append(file)
-
-                except ConnectionError as error:
-                    upload_results["failed_files"].append(
-                        f"{file} (Error: {str(error)})"
-                    )
-
-        if upload_results["failed_files"]:
-            raise SharePointUploadError(
-                reason="Files failed to upload: "
-                + ", ".join(upload_results["failed_files"])
-            )
-
-        return upload_results
-
-    async def execute(self) -> dict:
-        """Synchronize data from the THIES Center to the cloud."""
-        try:
-            thies_files = await self.fetch_thies_file_names()
-        except RuntimeError as error:
-            raise FtpClientError(error)
-        try:
-            cloud_files = await self.fetch_cloud_file_names()
-        except RuntimeError as error:
-            raise SharepointClient(error)
-
-        self.uploading = thies_files - cloud_files
-        if not self.uploading:
-            raise EmptyDataError(reason="No files to upload.")
-
-        # Fetch the content of the files to be uploaded from THIES FTP Server
-        thies_fetched_files = await self.fetch_thies_file_content()
-
-        # Upload the fetched files to SharePoint and gather statistics
-        upload_statistics = await self.upload_thies_files_to_sharepoint(
-            thies_fetched_files
-        )
-
-        return parse_execute_response(thies_fetched_files, upload_statistics)

saviialib/services/epii/use_cases/upload_backup_to_sharepoint.py

@@ -1,241 +0,0 @@
-import asyncio
-import os
-from time import time
-import saviialib.services.epii.use_cases.constants.upload_backup_to_sharepoint_constants as c
-from saviialib.general_types.error_types.api.epii_api_error_types import (
-    BackupEmptyError,
-    BackupSourcePathError,
-    BackupUploadError,
-)
-from saviialib.general_types.error_types.common import (
-    SharepointClientError,
-)
-from saviialib.libs.files_client import FilesClient, FilesClientInitArgs, ReadArgs
-from saviialib.libs.sharepoint_client import (
-    SharepointClient,
-    SharepointClientInitArgs,
-    SpUploadFileArgs,
-)
-from saviialib.services.epii.utils.upload_backup_to_sharepoint_utils import (
-    calculate_percentage_uploaded,
-    count_files_in_directory,
-    directory_exists,
-    extract_error_message,
-    parse_execute_response,
-    show_upload_result,
-)
-
-from .types.upload_backup_to_sharepoint_types import (
-    UploadBackupToSharepointUseCaseInput,
-)
-from .constants.upload_backup_to_sharepoint_constants import LOGGER
-
-
-class UploadBackupToSharepointUsecase:
-    def __init__(self, input: UploadBackupToSharepointUseCaseInput):
-        self.sharepoint_config = input.sharepoint_config
-        self.local_backup_source_path = input.local_backup_source_path
-        self.destination_folders = input.destination_folders
-        self.files_client = self._initialize_files_client()
-        self.log_history = []
-        self.grouped_files_by_folder = None
-        self.total_files = None
-
-    def _initialize_files_client(self):
-        return FilesClient(FilesClientInitArgs(client_name="aiofiles_client"))
-
-    async def _extract_filesnames_by_folder(self) -> dict[str, list[str]]:
-        """Groups files by their parent folder."""
-        backup_folder_exists = await asyncio.to_thread(
-            os.path.exists, self.local_backup_source_path
-        )
-        if not backup_folder_exists:
-            return {}
-        folder_names = await asyncio.to_thread(
-            os.listdir, self.local_backup_source_path
-        )
-        return {
-            folder_name: [
-                file_name
-                for file_name in await asyncio.to_thread(
-                    os.listdir, os.path.join(self.local_backup_source_path, folder_name)
-                )
-            ]
-            for folder_name in folder_names
-        }
-
-    def _save_log_history(self) -> None:
-        text_file = "\n".join(self.log_history)
-        log_history_filepath = "BACKUP_LOG_HISTORY.log"
-        with open(log_history_filepath, "w") as file:
-            file.write(text_file)
-
-    async def export_file_to_sharepoint(
-        self, folder_name: str, file_name: str, file_content: bytes
-    ) -> tuple[bool, str]:
-        """Uploads a file to the specified folder in SharePoint."""
-        uploaded = None
-        error_message = ""
-
-        try:
-            sharepoint_client = SharepointClient(
-                SharepointClientInitArgs(
-                    self.sharepoint_config, client_name="sharepoint_rest_api"
-                )
-            )
-        except ConnectionError as error:
-            raise SharepointClientError(error)
-
-        async with sharepoint_client:
-            try:
-                destination_folder = self.destination_folders.get(
-                    folder_name, folder_name
-                )
-                folder_url = f"{c.SHAREPOINT_BASE_URL}/{destination_folder}"
-                args = SpUploadFileArgs(
-                    folder_relative_url=folder_url,
-                    file_content=file_content,
-                    file_name=file_name,
-                )
-                await sharepoint_client.upload_file(args)
-                uploaded = True
-            except ConnectionError as error:
-                error_message = str(error)
-                uploaded = False
-
-        return uploaded, error_message
-
-    async def upload_and_log_progress_task(self, folder_name, file_name) -> dict:
-        """Task for uploads a file and logs progress."""
-        uploading_message = (
-            f"[BACKUP] Uploading file '{file_name}' from '{folder_name}' "
-        )
-        self.log_history.append(uploading_message)
-        LOGGER.debug(uploading_message)
-        file_path = os.path.join(self.local_backup_source_path, folder_name, file_name)
-        file_content = await self.files_client.read(ReadArgs(file_path, mode="rb"))
-        uploaded, error_message = await self.export_file_to_sharepoint(
-            folder_name, file_name, file_content
-        )
-        result_message = show_upload_result(uploaded, file_name)
-        LOGGER.debug(result_message)
-        self.log_history.append(result_message)
-        return {
-            "parent_folder": folder_name,
-            "file_name": file_name,
-            "uploaded": uploaded,
-            "error_message": error_message,
-        }
-
-    async def retry_upload_failed_files(self, results) -> None:
-        failed_files = [item for item in results if not item["uploaded"]]
-        tasks = []
-        retry_message = (
-            f"[BACKUP] Retrying upload for {len(failed_files)} failed files... 🚨"
-        )
-        self.log_history.append(retry_message)
-        LOGGER.debug(retry_message)
-        for file in failed_files:
-            tasks.append(
-                self.upload_and_log_progress_task(
-                    file["parent_folder"], file["file_name"]
-                )
-            )
-        results = await asyncio.gather(*tasks, return_exceptions=True)
-        success = calculate_percentage_uploaded(results, self.total_files)
-        if success < 100.0:
-            raise BackupUploadError(reason=extract_error_message(results, success))
-        else:
-            successful_upload_retry = (
-                "[BACKUP] All files uploaded successfully after retry."
-            )
-            self.log_history.append(successful_upload_retry)
-            LOGGER.debug(successful_upload_retry)
-            self._save_log_history()
-            return parse_execute_response(results)
-
-    async def execute(self):
-        """Exports all files from the local backup folder to SharePoint cloud."""
-        self.grouped_files_by_folder = await self._extract_filesnames_by_folder()
-        self.total_files = sum(
-            len(files) for files in self.grouped_files_by_folder.values()
-        )
-        tasks = []
-        start_time = time()
-
-        # Check if the local path exists in the main directory
-        if not await directory_exists(self.local_backup_source_path):
-            raise BackupSourcePathError(
-                reason=f"'{self.local_backup_source_path}' doesn't exist."
-            )
-
-        # Check if the current folder only have files.
-        items = [
-            item
-            for item in await asyncio.to_thread(
-                os.listdir, self.local_backup_source_path
-            )
-        ]
-        for item in items:
-            folder_included = item in self.destination_folders.keys()
-            is_file = not await asyncio.to_thread(
-                os.path.isdir, os.path.join(self.local_backup_source_path, item)
-            )
-
-            if not folder_included and not is_file:
-                raise BackupSourcePathError(
-                    reason=(
-                        f"'{item}' must be included in the destination folders dictionary",
-                    )
-                )
-            elif folder_included and is_file:
-                print(folder_included, is_file)
-                raise BackupSourcePathError(reason=(f"'{item}' must be a directory.",))
-
-        if self.total_files == 0:
-            no_files_message = (
-                f"[BACKUP] {self.local_backup_source_path} has no files ⚠️"
-            )
-            self.log_history.append(no_files_message)
-            LOGGER.debug(no_files_message)
-            raise BackupEmptyError
-        # Create task for each file stored in the the local backup folder.
-        for folder_name in self.grouped_files_by_folder:
-            if (
-                await count_files_in_directory(
-                    self.local_backup_source_path, folder_name
-                )
-                == 0
-            ):
-                empty_folder_message = f"[BACKUP] The folder '{folder_name}' is empty ⚠️"
-                LOGGER.debug(empty_folder_message)
-                self.log_history.append(empty_folder_message)
-                continue
-            extracting_files_message = (
-                "[BACKUP]" + f" Extracting files from '{folder_name} ".center(15, "*")
-            )
-            self.log_history.append(extracting_files_message)
-            LOGGER.debug(extracting_files_message)
-            for file_name in self.grouped_files_by_folder[folder_name]:
-                tasks.append(self.upload_and_log_progress_task(folder_name, file_name))
-
-        # Execution of multiple asynchronous tasks for files migration.
-        results = await asyncio.gather(*tasks, return_exceptions=True)
-        success = calculate_percentage_uploaded(results, self.total_files)
-        if success < 100.0:
-            await self.retry_upload_failed_files(results)
-        else:
-            end_time = time()
-            backup_time = end_time - start_time
-            successful_backup_message = (
-                f"[BACKUP] Migration time: {backup_time:.2f} seconds ✨"
-            )
-            self.log_history.append(successful_backup_message)
-
-            finished_backup_message = (
-                "[BACKUP] All the files were uploaded successfully 🎉"
-            )
-            self.log_history.append(finished_backup_message)
-
-            self._save_log_history()
-            return parse_execute_response(results)

saviialib/services/epii/utils/__init__.py

@@ -1,3 +0,0 @@
-from .update_thies_data_utils import parse_execute_response
-
-__all__ = ["parse_execute_response"]

saviialib/services/epii/utils/upload_backup_to_sharepoint_utils.py

@@ -1,102 +0,0 @@
-import re
-from typing import List, Dict, Optional
-import asyncio
-import os
-from saviialib.general_types.error_types.api.epii_api_error_types import (
-    BackupSourcePathError,
-)
-from saviialib.services.epii.use_cases.constants.upload_backup_to_sharepoint_constants import (
-    LOGGER,
-)
-
-
-def extract_error_information(error: str) -> Optional[Dict[str, str]]:
-    match = re.search(r"(\d+), message='([^']*)', url=\"([^\"]*)\"", error)
-    if match:
-        return {
-            "status_code": match.group(1),
-            "message": match.group(2),
-            "url": match.group(3),
-        }
-    return None
-
-
-def explain_status_code(status_code: int) -> str:
-    explanations = {
-        404: "Probably an error with file or folder source path.",
-        403: "Permission denied when accessing the source path.",
-        500: "Internal server error occurred during upload.",
-    }
-    return explanations.get(status_code, "Unknown error occurred.")
-
-
-def extract_error_message(results: List[Dict], success: float) -> str:
-    LOGGER.info(
-        "[BACKUP] Not all files uploaded ⚠️\n"
-        f"[BACKUP] Files failed to upload: {(1 - success):.2%}"
-    )
-
-    failed_files = [item for item in results if not item.get("uploaded")]
-
-    error_data = []
-    for item in failed_files:
-        error_info = extract_error_information(item.get("error_message", ""))
-        if error_info:
-            error_data.append(
-                {
-                    "file_name": item["file_name"],
-                    "status_code": error_info["status_code"],
-                    "message": error_info["message"],
-                    "url": error_info["url"],
-                }
-            )
-
-    # Group errors by code.
-    grouped_errors: Dict[str, List[Dict]] = {}
-    for error in error_data:
-        code = error["status_code"]
-        grouped_errors.setdefault(code, []).append(error)
-
-    # Summary
-    for code, items in grouped_errors.items():
-        LOGGER.info(f"[BACKUP] Status code {code} - {explain_status_code(int(code))}")
-        for item in items:
-            LOGGER.info(
-                f"[BACKUP] File {item['file_name']}, url: {item['url']}, message: {item['message']}"
-            )
-
-    failed_file_names = [item["file_name"] for item in failed_files]
-    return f"Failed files: {', '.join(failed_file_names)}."
-
-
-def parse_execute_response(results: List[Dict]) -> Dict[str, List[str]]:
-    try:
-        return {
-            "new_files": len(
-                [item["file_name"] for item in results if item.get("uploaded")]
-            ),
-        }
-    except (IsADirectoryError, AttributeError, ConnectionError) as error:
-        raise BackupSourcePathError(reason=error)
-
-
-def show_upload_result(uploaded: bool, file_name: str) -> str:
-    status = "✅" if uploaded else "❌"
-    message = "was uploaded successfully" if uploaded else "failed to upload"
-    result = f"[BACKUP] File {file_name} {message} {status}"
-    return result
-
-
-def calculate_percentage_uploaded(results: List[Dict], total_files: int) -> float:
-    uploaded_count = sum(
-        1 for result in results if isinstance(result, dict) and result.get("uploaded")
-    )
-    return (uploaded_count / total_files) * 100 if total_files > 0 else 0
-
-
-async def directory_exists(path: str) -> bool:
-    return await asyncio.to_thread(os.path.exists, path)
-
-
-async def count_files_in_directory(path: str, folder_name: str) -> int:
-    return len(await asyncio.to_thread(os.listdir, os.path.join(path, folder_name)))