hid_data_transfer_lib 0.3.0__tar.gz → 0.3.2__tar.gz

{hid_data_transfer_lib-0.3.0 → hid_data_transfer_lib-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: hid_data_transfer_lib
-Version: 0.3.0
+Version: 0.3.2
 Summary: HiDALGO Data Transfer library provides methods to transfer data between different data providers and consumers using NIFI pipelines
 License: APL-2.0
 Author: Jesús Gorroñogoitia
@@ -72,6 +72,9 @@ nifi_secure_connection=True
 keycloak_endpoint=https://idm.hidalgo2.eu
 keycloak_client_id=nifi
 keycloak_client_secret=<keycloak_nifi_client_secret>
+
+[Network]
+check_status_sleep_lapse=5
 ```
 Under the NIFI section, 
 - We define the url of the NIFI service (*nifi_endpoint*), 
@@ -84,6 +87,8 @@ Under the Keycloak section, you can configure the Keycloak integrated with NIFI,
 - The NIFI client in Keycloak (*keycloak_client*)
 - The NIFI secret in Keycloak (*keycloak_client_secret*)
 
+Under the Network section, you can configure the lapse time (in seconds) each processor in the NIFI pipeline is checked for complation. Most of users should leave the default value.
+
 HiDALGO2 developers can contact the Keycloak administrator for the *keycloak_client_secret*
 
 ### User's accounts in environment variables
@@ -166,6 +171,9 @@ dt_client.ckan2hpc(
 )
 ```
 
+## Support for HPC clusters that require a 2FA token
+This library includes methods (suffixed as _2fa) to transfer data to/from HPC clusters that require a 2FA token. These methods offer an optional parameter *callback_2fa* that points to a method that should return (as str) the 2FA token when invoked by the library. If not set by the method caller, these methods call a default implementation that prompts the user (in the standard input) for the token. 
+
 ## Data transfer process with NIFI
 The following UML Sequence Diagram describes the data transfer process for each command, for instance *ckan2hpc*, leveraging the associated NIFI pipeline.
 The Data Transfer (DT) Consumer (a client of this library) invokes a *ckan2hpc* command by following these steps:

{hid_data_transfer_lib-0.3.0 → hid_data_transfer_lib-0.3.2}/README.md

@@ -54,6 +54,9 @@ nifi_secure_connection=True
 keycloak_endpoint=https://idm.hidalgo2.eu
 keycloak_client_id=nifi
 keycloak_client_secret=<keycloak_nifi_client_secret>
+
+[Network]
+check_status_sleep_lapse=5
 ```
 Under the NIFI section, 
 - We define the url of the NIFI service (*nifi_endpoint*), 
@@ -66,6 +69,8 @@ Under the Keycloak section, you can configure the Keycloak integrated with NIFI,
 - The NIFI client in Keycloak (*keycloak_client*)
 - The NIFI secret in Keycloak (*keycloak_client_secret*)
 
+Under the Network section, you can configure the lapse time (in seconds) each processor in the NIFI pipeline is checked for complation. Most of users should leave the default value.
+
 HiDALGO2 developers can contact the Keycloak administrator for the *keycloak_client_secret*
 
 ### User's accounts in environment variables
@@ -148,6 +153,9 @@ dt_client.ckan2hpc(
 )
 ```
 
+## Support for HPC clusters that require a 2FA token
+This library includes methods (suffixed as _2fa) to transfer data to/from HPC clusters that require a 2FA token. These methods offer an optional parameter *callback_2fa* that points to a method that should return (as str) the 2FA token when invoked by the library. If not set by the method caller, these methods call a default implementation that prompts the user (in the standard input) for the token. 
+
 ## Data transfer process with NIFI
 The following UML Sequence Diagram describes the data transfer process for each command, for instance *ckan2hpc*, leveraging the associated NIFI pipeline.
 The Data Transfer (DT) Consumer (a client of this library) invokes a *ckan2hpc* command by following these steps:

{hid_data_transfer_lib-0.3.0 → hid_data_transfer_lib-0.3.2}/hid_data_transfer_lib/conf/hid_dt.cfg

@@ -11,3 +11,6 @@ keycloak_client_secret=<keycloak_client_secret>
 
 [Logging]
 logging_level=INFO
+
+[Network]
+check_status_sleep_lapse=5

{hid_data_transfer_lib-0.3.0 → hid_data_transfer_lib-0.3.2}/hid_data_transfer_lib/conf/hid_dt_configuration.py

@@ -43,6 +43,8 @@ class HidDataTransferConfiguration:
     __keycloak_login = None
     __keycloak_passwd = None
 
+    __check_status_sleep_lapse = 10  # seconds
+
     def __init__(self, logging_level: Optional[int] = None) -> None:
         self.__get_configuration(logging_level)
 
@@ -208,6 +210,10 @@ class HidDataTransferConfiguration:
             if isinstance(handler, logging.FileHandler):
                 return handler
         return None
+    
+    def check_status_sleep_lapse(self):
+        """Returns the NIFI login name"""
+        return self.__check_status_sleep_lapse
 
     def __get_configuration(self, logging_level: Optional[int]):
         """
@@ -251,6 +257,10 @@ class HidDataTransferConfiguration:
                 logging_level_str = config.get("Logging", "logging_level")
                 self.__logging_level = getattr(
                     logging, logging_level_str.upper(), logging.INFO)
+                
+            # Network section
+            self.__check_status_sleep_lapse = config.get(
+                "Network", "check_status_sleep_lapse")
 
         except configparser.NoSectionError as ex:
             raise HidDataTransferException(

{hid_data_transfer_lib-0.3.0 → hid_data_transfer_lib-0.3.2}/hid_data_transfer_lib/nifi/nifi_client.py

@@ -64,9 +64,11 @@ def is_binary_file(filename: str) -> bool:
         ".pdf", ".doc", ".docx", ".ppt", ".pptx", ".xls", ".xlsx"
     ]
 
-    file_extension = filename.lower().rsplit(".", 1)[-1] if "." in filename else ""
+    file_extension = filename.lower().rsplit(".", 1)[-1] \
+        if "." in filename else ""
     return f".{file_extension}" in binary_extensions
 
+
 # Helper enums
 # An enum describing the types of processor exectution, either once or forever
 class ExecutionType(Enum):
@@ -83,9 +85,6 @@ class NIFIClient:
     (taken from configuration)
     """
 
-    # Requests constants
-    __CHECK_STATUS_SLEEP_LAPSE = 10
-
     # Commands parameters
     HPC_HOST = "hpc.host"
     HPC_PORT = "hpc.port"
@@ -110,6 +109,8 @@ class NIFIClient:
         self.keycloak_rest_client = KeycloakRESTClient(
             conf, secure, refresh=True)
         self.__logger = self.__conf.logger("nifi.v2.client")
+        self.check_status_sleep_lapse = int(
+            self.__conf.check_status_sleep_lapse())
 
     def get_access_token(self) -> str:
         """gets the NIFI access token,
@@ -811,7 +812,7 @@ class NIFIClient:
                     f"Processor {processor_name} has failed with exception: "
                     f"{error_bulletins}"
                 )
-            sleep(self.__CHECK_STATUS_SLEEP_LAPSE)
+            sleep(self.check_status_sleep_lapse)
         self.stop_forever_running_processor(
             execution_type[processor_name], processor)
 
@@ -996,6 +997,23 @@ class NIFIClient:
             source=key,
             target=__target,
         )
+        # Check if the key is accompanied by a certificate in the same folder
+        # If so, upload the certificate as well
+        cert_prefix = "-cert.pub"
+        if os.path.exists(os.path.expanduser(key) + cert_prefix):
+            __target_cert = os.path.join(
+                self.__conf.nifi_upload_folder(),
+                str(__key_name) + cert_prefix)
+            sftp_put(
+                server=self.__conf.nifi_server(),
+                private_key=self.__conf.nifi_server_private_key(),
+                username=self.__conf.nifi_server_user_name(),
+                target_permissions="660",
+                target_ownership=ownership,
+                source=key + cert_prefix,
+                target=__target_cert,
+            )
+
         return __target
 
     def upload_files(self, local_folder: str,

{hid_data_transfer_lib-0.3.0 → hid_data_transfer_lib-0.3.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "hid_data_transfer_lib"
-version = "0.3.0"
+version = "0.3.2"
 description = "HiDALGO Data Transfer library provides methods to transfer data between different data providers and consumers using NIFI pipelines"
 authors = [{name="Jesús Gorroñogoitia", email = "jesus.gorronogoitia@eviden.com"}]
 license = "APL-2.0"