hid_data_transfer_lib 0.3.10__tar.gz → 0.4.0__tar.gz

{hid_data_transfer_lib-0.3.10 → hid_data_transfer_lib-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hid_data_transfer_lib
-Version: 0.3.10
+Version: 0.4.0
 Summary: HiDALGO Data Transfer library provides methods to transfer data between different data providers and consumers using NIFI pipelines
 License: APL-2.0
 Author: Jesús Gorroñogoitia
@@ -51,7 +51,7 @@ The Data Transfer Library sends requests to NIFI through its REST API.
 To use the Data Transfer library, it is required the following requirements:
  - **Python3** execution environment
  - **Poetry** python package management tool (optional)
- - **NIFI** instance, with a NIFI server SSH account (for keys transfer)
+ - **NIFI** instance, with a NIFI 2.X server SSH account (for keys transfer). Latest NIFI supported version: 2.8
  - **Keycloak** instance, with a KEYCLOAK user's account
  - **HDFS** instance, with an user's Kerberos principal account
  - **CKAN** instance, with an user APIKey
@@ -123,7 +123,7 @@ keycloak_login=<user_name>
 keycloak_password=<password>
 ```
 
-Under the Nifi section, you must also specify a user account (username, private_key) that grants to upload/download files to the NIFI server (as requested to upload temporary HPC keys or to support local file transfer). This user's account is provided by Hidalgo2 infrastructure provider and it is user's or service's specific. 
+Under the Nifi section, you must also specify a user account (username, private_key) that grants to upload/download files to the NIFI server (as requested to upload temporary HPC keys or to support local file transfer). This user's account is provided by Hidalgo2 infrastructure provider and it is user's or service's specific. Note: this user's account requires to belond to group *users*.
 
 Under the Keycloak section, you must specify your Keycloak account (username and password). This account granted with access to the NIFI service.
 

{hid_data_transfer_lib-0.3.10 → hid_data_transfer_lib-0.4.0}/README.md

@@ -32,7 +32,7 @@ The Data Transfer Library sends requests to NIFI through its REST API.
 To use the Data Transfer library, it is required the following requirements:
  - **Python3** execution environment
  - **Poetry** python package management tool (optional)
- - **NIFI** instance, with a NIFI server SSH account (for keys transfer)
+ - **NIFI** instance, with a NIFI 2.X server SSH account (for keys transfer). Latest NIFI supported version: 2.8
  - **Keycloak** instance, with a KEYCLOAK user's account
  - **HDFS** instance, with an user's Kerberos principal account
  - **CKAN** instance, with an user APIKey
@@ -104,7 +104,7 @@ keycloak_login=<user_name>
 keycloak_password=<password>
 ```
 
-Under the Nifi section, you must also specify a user account (username, private_key) that grants to upload/download files to the NIFI server (as requested to upload temporary HPC keys or to support local file transfer). This user's account is provided by Hidalgo2 infrastructure provider and it is user's or service's specific. 
+Under the Nifi section, you must also specify a user account (username, private_key) that grants to upload/download files to the NIFI server (as requested to upload temporary HPC keys or to support local file transfer). This user's account is provided by Hidalgo2 infrastructure provider and it is user's or service's specific. Note: this user's account requires to belond to group *users*.
 
 Under the Keycloak section, you must specify your Keycloak account (username and password). This account granted with access to the NIFI service.
 

{hid_data_transfer_lib-0.3.10 → hid_data_transfer_lib-0.4.0}/hid_data_transfer_lib/conf/hid_dt_configuration.py

@@ -24,6 +24,7 @@ from hid_data_transfer_lib.exceptions.hid_dt_exceptions import (
     HidDataTransferException
 )
 
+STATUS_SLEEP_LAPSE = 10 # seconds
 
 class HidDataTransferConfiguration:
     """This class manages the configuration of the Data Transfer library"""
@@ -44,21 +45,22 @@ class HidDataTransferConfiguration:
     __keycloak_login = None
     __keycloak_passwd = None
 
-    __check_status_sleep_lapse = 10  # seconds
+    __check_status_sleep_lapse = STATUS_SLEEP_LAPSE  # seconds
 
     def __init__(self, logging_level: Optional[int] = logging.INFO):
         self.__logging_level = logging_level
 
-    def check_keycloak_conf(self):
+    def check_keycloak_conf(self, use_keycloak_account):
         """Check if keycloak configuration is valid"""
-        if self.__keycloak_login is None:
-            raise HidDataTransferException(
-                "Keycloak login must be set in the configuration"
-            )
-        if self.__keycloak_passwd is None:
-            raise HidDataTransferException(
-                "Keycloak passwd must be set in the configuration"
-            )
+        if use_keycloak_account:
+            if self.__keycloak_login is None:
+                raise HidDataTransferException(
+                    "Keycloak login must be set in the configuration"
+                )
+            if self.__keycloak_passwd is None:
+                raise HidDataTransferException(
+                    "Keycloak passwd must be set in the configuration"
+                )
         if self.__keycloak_endpoint is None:
             raise HidDataTransferException(
                 "Keycloak endpoint must be set in the configuration"
@@ -195,6 +197,14 @@ class HidDataTransferConfiguration:
         """Returns the NIFI login password"""
         return self.__keycloak_passwd
 
+    def reset_keycloak_login(self):
+        """Reset the NIFI login name"""
+        self.__keycloak_login = None
+
+    def reset_keycloak_passwd(self):
+        """Reset the NIFI login password"""
+        self.__keycloak_passwd = None
+
     def logger(self, logger_name):
         """Returns the logging level"""
         return self.__set_logger(logger_name)
@@ -256,7 +266,8 @@ class HidDataTransferConfiguration:
         self,
         user_config_file: Optional[str] = None,
         user_config_dict: Optional[dict] = None,
-        logging_level: Optional[int] = None
+        logging_level: Optional[int] = None,
+        use_keycloak_account: Bool = True,
     ) -> HidDataTransferConfiguration:
         """
         Reads the configuration and sets this class' configuration values
@@ -276,38 +287,33 @@ class HidDataTransferConfiguration:
             self.load_nifi_configuration(config)
 
             # Keycloak section
-            self.load_keycloak_configuration(config)
-
-            # Check if either NIFI or Keycloak login and password are set
-            if (self.__nifi_login is None and self.__nifi_passwd is None) and (
-                self.__keycloak_login is None and
-                self.__keycloak_passwd is None
-            ):
-                raise HidDataTransferException(
-                    "Either (KEYCLOAK_LOGIN and KEYCLOAK_PASSWORD) "
-                    "or (NIFI_LOGIN and NIFI_PASSWORD) "
-                    "environment variables must be set"
-                )
+            self.load_keycloak_configuration(
+                config, check_configuration = use_keycloak_account)
+
+            # # Check if either NIFI or Keycloak login and password are set
+            # if not use_keycloak_token and (
+            #     self.__nifi_login is None and self.__nifi_passwd is None
+            #     ) and (
+            #     self.__keycloak_login is None and
+            #     self.__keycloak_passwd is None
+            # ):
+            #     raise HidDataTransferException(
+            #         "Either (KEYCLOAK_LOGIN and KEYCLOAK_PASSWORD) "
+            #         "or (NIFI_LOGIN and NIFI_PASSWORD) "
+            #         "environment variables must be set"
+            #     )
 
             # Logging section
             if logging_level is not None:
                 self.__logging_level = logging_level
             else:
-                logging_level_str = config.get("Logging", "logging_level")
+                logging_level_str = config.get("Logging", "logging_level", fallback="Info")
                 self.__logging_level = getattr(
                     logging, logging_level_str.upper(), logging.INFO)
 
             # Network section
-            self.__check_status_sleep_lapse = config.get(
-                "Network", "check_status_sleep_lapse")
-
-            # Network section
-            self.__check_status_sleep_lapse = config.get(
-                "Network", "check_status_sleep_lapse")
-
-            # Network section
-            self.__check_status_sleep_lapse = config.get(
-                "Network", "check_status_sleep_lapse")
+            self.__check_status_sleep_lapse = int(config.get(
+                "Network", "check_status_sleep_lapse", fallback=STATUS_SLEEP_LAPSE))
 
             return self
 
@@ -316,69 +322,67 @@ class HidDataTransferConfiguration:
                 "Error parsing CLI configuration file"
             ) from ex
 
-    def load_keycloak_configuration(self, config):
+    def load_keycloak_configuration(self, config, check_configuration = True):
         """Load Keycloak configuration from config file"""
         # Read Keycloak login and password from configuration
-        self.__keycloak_login = config.get("Keycloak", "keycloak_login")
-        self.__keycloak_passwd = config.get("Keycloak", "keycloak_password")
-        if (self.__keycloak_login and self.__keycloak_passwd is None) or (
-            self.__keycloak_login is None and self.__keycloak_passwd
-        ):
+        self.__keycloak_login = config.get("Keycloak", "keycloak_login", fallback=None)
+        self.__keycloak_passwd = config.get("Keycloak", "keycloak_password", fallback=None)
+        if check_configuration and (
+            self.__keycloak_login is None or self.__keycloak_passwd is None):
             raise HidDataTransferException(
                 "Both KEYCLOAK_LOGIN and KEYCLOAK_PASSWORD "
                 "environment variables must be set"
             )
 
         # Read other keycloak configuration from config file
-        if self.__keycloak_login or self.__keycloak_passwd:
-            self.__keycloak_endpoint = config.get(
-                "Keycloak", "keycloak_endpoint")
-            if self.__keycloak_endpoint is None:
-                raise HidDataTransferException(
-                    "Could not find Keycloak endpoint in config file."
-                )
+        self.__keycloak_endpoint = config.get(
+            "Keycloak", "keycloak_endpoint", fallback=None)
+        if self.__keycloak_endpoint is None:
+            raise HidDataTransferException(
+                "Could not find Keycloak endpoint in config file."
+            )
 
-            self.__keycloak_client_id = config.get(
-                "Keycloak", "keycloak_client_id")
-            if self.__keycloak_client_id is None:
-                raise HidDataTransferException(
-                    "Could not find Keycloak client id "
-                    "for NIFI in config file."
-                )
+        self.__keycloak_client_id = config.get(
+            "Keycloak", "keycloak_client_id", fallback=None)
+        if self.__keycloak_client_id is None:
+            raise HidDataTransferException(
+                "Could not find Keycloak client id "
+                "for NIFI in config file."
+            )
 
-            keycloak_client_secret = config.get(
-                "Keycloak", "keycloak_client_secret")
-            if keycloak_client_secret is not None:
-                self.__keycloak_client_secret = keycloak_client_secret
-            else:
-                raise HidDataTransferException(
-                    "Could not find Keycloak client secret "
-                    "for NIFI in config file."
-                )
+        keycloak_client_secret = config.get(
+            "Keycloak", "keycloak_client_secret", fallback=None)
+        if keycloak_client_secret is not None:
+            self.__keycloak_client_secret = keycloak_client_secret
+        else:
+            raise HidDataTransferException(
+                "Could not find Keycloak client secret "
+                "for NIFI in config file."
+            )
 
     def load_nifi_configuration(self, config):
         """Load NIFI configuration from config file"""
-        self.__nifi_endpoint = config.get("Nifi", "nifi_endpoint")
+        self.__nifi_endpoint = config.get("Nifi", "nifi_endpoint", fallback=None)
         if self.__nifi_endpoint is None:
             raise HidDataTransferException(
                 "Could not find Nifi endpoint in config file."
             )
 
-        nifi_secure_connection = config.get("Nifi", "nifi_secure_connection")
+        nifi_secure_connection = config.get("Nifi", "nifi_secure_connection", fallback=None)
         if nifi_secure_connection is not None:
             self.__nifi_secure_connection = \
                 nifi_secure_connection.lower() == "true"
         else:
             self.__nifi_secure_connection = False
 
-        self.__nifi_upload_folder = config.get("Nifi", "nifi_upload_folder")
+        self.__nifi_upload_folder = config.get("Nifi", "nifi_upload_folder", fallback=None)
         if self.__nifi_upload_folder is None:
             raise HidDataTransferException(
                 "Could not find Nifi nifi_upload_folder in config file"
             )
 
         self.__nifi_download_folder = config.get(
-            "Nifi", "nifi_download_folder")
+            "Nifi", "nifi_download_folder", fallback=None)
         if self.__nifi_download_folder is None:
             raise HidDataTransferException(
                 "Could not find Nifi nifi_download_folder in config file"
@@ -386,9 +390,9 @@ class HidDataTransferConfiguration:
 
         # Optional NIFI server account in case of local file transfer
         self.__nifi_server_user_name = config.get(
-            "Nifi", "nifi_server_username")
+            "Nifi", "nifi_server_username", fallback=None)
         self.__nifi_server_private_key = config.get(
-            "Nifi", "nifi_server_private_key")
+            "Nifi", "nifi_server_private_key", fallback=None)
 
         if (
             self.__nifi_server_user_name and
@@ -400,7 +404,7 @@ class HidDataTransferConfiguration:
                 "environment variables must be set"
             )
 
-        nifi_concurrent_tasks = config.get("Nifi", "nifi_concurrent_tasks")
+        nifi_concurrent_tasks = config.get("Nifi", "nifi_concurrent_tasks", fallback=None)
         if nifi_concurrent_tasks is not None:
             self.__nifi_concurrent_tasks = int(nifi_concurrent_tasks)
         else:

{hid_data_transfer_lib-0.3.10 → hid_data_transfer_lib-0.4.0}/hid_data_transfer_lib/hid_dt_lib.py

@@ -34,10 +34,11 @@ from hid_data_transfer_lib.nifi.nifi_client import (
     NIFIClient, ExecutionType, AccountingInfo
 )
 
-KERBEROS_PRINCIPAL = "kerberos.principal"
 HDFS_DATA_FOLDER = "hdfs.data_folder"
+HDFS_DATA_FILENAME = "hdfs.data_filename"
 HDFS_FILENAME = "hdfs.filename"
 HPC_TARGET_FOLDER = "hpc.target_folder"
+HPC_TARGET_FILENAME = "hpc.target_filename"
 HPC_DATA_FOLDER = "hpc.data_folder"
 HPC_FILENAME = "hpc.filename"
 HPC_FILENAME_REGEX = "hpc.filename.regex"
@@ -69,13 +70,16 @@ class HIDDataTransfer:
         """
         self.__conf = conf
         self.__secure = secure
-        self.__nifi_client = NIFIClient().configure(conf, secure)
+        self.use_keycloak_account = (keycloak_token is None)
+        self.__nifi_client = NIFIClient().configure(
+            conf, secure, use_keycloak_account=self.use_keycloak_account)
         self.__logger = self.__conf.logger("hid_data_transfer_lib")
         self.__keycloak_token = keycloak_token
 
     def reset(self):
         """ Reset NIFIClient configuration"""
-        self.__nifi_client = NIFIClient().configure(self.__conf, self.__secure)
+        self.__nifi_client = NIFIClient().configure(
+            self.__conf, self.__secure, use_keycloak_account=self.use_keycloak_account)
 
     # Helper methods
     def process_wildcards(self, name):
@@ -158,9 +162,13 @@ class HIDDataTransfer:
             hpc_secret_key=hpc_secret_key_path,
             data_source=data_source,
             data_target=data_target,
+            recursive=recursive,
+        )
+
+        processors_with_kerberos = ["ListHDFS", "FetchHDFS"]
+        kerberos_account = SimpleNamespace(
             kerberos_principal=kerberos_principal,
             kerberos_password=kerberos_password,
-            recursive=recursive,
         )
 
         self.__logger.info(
@@ -170,12 +178,26 @@ class HIDDataTransfer:
         try:
             arguments: Dict[str, Any] = {}
             self.__nifi_client.add_default_hpc_parameters(arguments, args)
-            arguments[HPC_TARGET_FOLDER] = args.data_target
+
             arguments[HDFS_DATA_FOLDER] = \
                 "/".join(args.data_source.split("/")[:-1])
             arguments[HDFS_FILENAME] = \
                 self.process_wildcards(args.data_source.split("/")[-1])
-            arguments[KERBEROS_PRINCIPAL] = args.kerberos_principal
+
+            # If multiple data source files are selected, data_target represents a folder
+            # Else If data_target ends with / represents a folder, otherwise a file
+            arguments[HPC_TARGET_FILENAME] = ""
+            if args.data_source and (
+                "*" in args.data_source or
+                "?" in args.data_source
+            ):
+                arguments[HPC_TARGET_FOLDER] = args.data_target
+            elif args.data_target.endswith('/'):
+                arguments[HPC_TARGET_FOLDER] = args.data_target[:-1]
+            else:
+                arguments[HPC_TARGET_FOLDER] = "/".join(args.data_target.split("/")[:-1])
+                arguments[HPC_TARGET_FILENAME] = "/".join(args.data_target.split("/")[-1:])
+
             arguments[RECURSIVE] = args.recursive
 
             execution_type = {
@@ -197,14 +219,6 @@ class HIDDataTransfer:
                     args
                 )
 
-            processors_with_hdfs_sensitive_parameters = \
-                ["ListHDFS", "FetchHDFS"]
-            sensitive_parameters.update(
-                self.__nifi_client.set_sensitive_hdfs_parameters(
-                    processors_with_hdfs_sensitive_parameters, args
-                )
-            )
-
             if not args.hpc_secret_key and not args.hpc_password:
                 raise HidDataTransferException(HPC_ACCOUNT_MSG)
 
@@ -227,6 +241,8 @@ class HIDDataTransfer:
                 accounting_target="PutSFTP",
                 concurrent_tasks=concurrent_tasks_dict,
                 ignore_bulletins_in_processors=["PutSFTP"],
+                processors_with_kerberos=processors_with_kerberos,
+                kerberos_account=kerberos_account,
             )
 
         except Exception as ex:
@@ -387,6 +403,12 @@ class HIDDataTransfer:
             kerberos_password=kerberos_password,
         )
 
+        processors_with_kerberos = ["ListHDFS", "FetchHDFS"]
+        kerberos_account = SimpleNamespace(
+            kerberos_principal=kerberos_principal,
+            kerberos_password=kerberos_password,
+        )
+
         self.__logger.info(
             "executing hdfs2hpc_2fa command with args: %s",
             self.__nifi_client.format_args_to_string(args),
@@ -399,7 +421,6 @@ class HIDDataTransfer:
                 "/".join(args.data_source.split("/")[:-1])
             arguments[HDFS_FILENAME] = \
                 self.process_wildcards(args.data_source.split("/")[-1])
-            arguments[KERBEROS_PRINCIPAL] = args.kerberos_principal
 
             # set local.data_folder to keep dataset retrieved from HDFS
             __uuid = uuid.uuid4()
@@ -422,16 +443,6 @@ class HIDDataTransfer:
                 ],
             }
 
-            sensitive_parameters = {}
-
-            processors_with_hdfs_sensitive_parameters = \
-                ["ListHDFS", "FetchHDFS"]
-            sensitive_parameters.update(
-                self.__nifi_client.set_sensitive_hdfs_parameters(
-                    processors_with_hdfs_sensitive_parameters, args
-                )
-            )
-
             if not args.hpc_secret_key:
                 raise HidDataTransferException(HPC_ACCOUNT_MSG)
 
@@ -450,10 +461,12 @@ class HIDDataTransfer:
                 execution_type,
                 arguments,
                 keys_arguments=keys_arguments,
-                sensitive_parameters=sensitive_parameters,
+                sensitive_parameters=None,
                 keycloak_token=self.__keycloak_token,
                 accounting_target="PutFile",
                 concurrent_tasks=concurrent_tasks_dict,
+                processors_with_kerberos=processors_with_kerberos,
+                kerberos_account=kerberos_account,
             )
             process_ids.append(process_id)
 
@@ -462,7 +475,7 @@ class HIDDataTransfer:
                 execution_type,
                 arguments,
                 keys_arguments=keys_arguments,
-                sensitive_parameters=sensitive_parameters,
+                sensitive_parameters=None,
                 keycloak_token=self.__keycloak_token,
                 processor_with_2fa="SftpTransferToHPCWithVPNCode",
                 callback_2fa=callback_2fa,
@@ -536,6 +549,12 @@ class HIDDataTransfer:
             recursive=recursive,
         )
 
+        processors_with_kerberos = ["PutHDFS"]
+        kerberos_account = SimpleNamespace(
+            kerberos_principal=kerberos_principal,
+            kerberos_password=kerberos_password,
+        )
+
         self.__logger.info(
             "executing hpc2hdfs command with args: %s",
             self.__nifi_client.format_args_to_string(args),
@@ -548,8 +567,21 @@ class HIDDataTransfer:
             arguments[HPC_DATA_FOLDER] = args.data_source[
                 : args.data_source.rfind("/")
             ]
-            arguments[HDFS_DATA_FOLDER] = args.data_target
-            arguments[KERBEROS_PRINCIPAL] = args.kerberos_principal
+
+            # If multiple data source files are selected, data_target represents a folder
+            # Else If data_target ends with / represents a folder, otherwise a file
+            arguments[HDFS_DATA_FILENAME] = ""
+            if args.data_source and (
+                "*" in args.data_source or
+                "?" in args.data_source
+            ):
+                arguments[HDFS_DATA_FOLDER] = args.data_target
+            elif args.data_target.endswith('/'):
+                arguments[HDFS_DATA_FOLDER] = args.data_target[:-1]
+            else:
+                arguments[HDFS_DATA_FOLDER] = "/".join(args.data_target.split("/")[:-1])
+                arguments[HDFS_DATA_FILENAME] = "/".join(args.data_target.split("/")[-1:])
+
             arguments[RECURSIVE] = args.recursive
 
             execution_type = {
@@ -572,14 +604,6 @@ class HIDDataTransfer:
                     processors_with_hpc_sensitive_parameters, args
                 )
 
-            processors_with_hdfs_sensitive_parameters = \
-                ["PutHDFS"]
-            sensitive_parameters.update(
-                self.__nifi_client.set_sensitive_hdfs_parameters(
-                    processors_with_hdfs_sensitive_parameters, args
-                )
-            )
-
             if not args.hpc_secret_key and not args.hpc_password:
                 raise HidDataTransferException(HPC_ACCOUNT_MSG)
 
@@ -602,6 +626,8 @@ class HIDDataTransfer:
                 accounting_target="PutHDFS",
                 concurrent_tasks=concurrent_tasks_dict,
                 ignore_bulletins_in_processors=["FetchSFTP"],
+                processors_with_kerberos=processors_with_kerberos,
+                kerberos_account=kerberos_account,
             )
 
         except Exception as ex:
@@ -712,6 +738,12 @@ class HIDDataTransfer:
             kerberos_password=kerberos_password,
         )
 
+        processors_with_kerberos = ["PutHDFS"]
+        kerberos_account = SimpleNamespace(
+            kerberos_principal=kerberos_principal,
+            kerberos_password=kerberos_password,
+        )
+
         self.__logger.info(
             "executing hpc2hdfs_2fa command with args: %s",
             self.__nifi_client.format_args_to_string(args),
@@ -728,7 +760,6 @@ class HIDDataTransfer:
                 : args.data_source.rfind("/")
             ]
             arguments[HDFS_DATA_FOLDER] = args.data_target
-            arguments[KERBEROS_PRINCIPAL] = args.kerberos_principal
 
             # set local.data_folder to keep dataset retrieved from HPC
             __uuid = uuid.uuid4()
@@ -758,14 +789,6 @@ class HIDDataTransfer:
                     processors_with_hpc_sensitive_parameters, args
                 )
 
-            processors_with_hdfs_sensitive_parameters = \
-                ["PutHDFS"]
-            sensitive_parameters.update(
-                self.__nifi_client.set_sensitive_hdfs_parameters(
-                    processors_with_hdfs_sensitive_parameters, args
-                )
-            )
-
             if not args.hpc_secret_key and not args.hpc_password:
                 raise HidDataTransferException(HPC_ACCOUNT_MSG)
 
@@ -788,7 +811,6 @@ class HIDDataTransfer:
                 keycloak_token=self.__keycloak_token,
                 processor_with_2fa="SftpTransferFromHPCWithVPNCode",
                 callback_2fa=callback_2fa,
-                accounting_target="PutHDFS",
                 concurrent_tasks=concurrent_tasks_dict,
             )
             process_ids.append(process_id)
@@ -802,6 +824,8 @@ class HIDDataTransfer:
                 keycloak_token=self.__keycloak_token,
                 accounting_target="PutHDFS",
                 concurrent_tasks=concurrent_tasks_dict,
+                processors_with_kerberos=processors_with_kerberos,
+                kerberos_account=kerberos_account,
             )
             process_ids.append(process_id)
 
@@ -913,6 +937,12 @@ class HIDDataTransfer:
             recursive=recursive,
         )
 
+        processors_with_kerberos = ["ListHDFS", "FetchHDFS"]
+        kerberos_account = SimpleNamespace(
+            kerberos_principal=kerberos_principal,
+            kerberos_password=kerberos_password,
+        )
+
         self.__logger.info(
             "executing hdfs2ckan command with args: %s",
             self.__nifi_client.format_args_to_string(args),
@@ -923,7 +953,6 @@ class HIDDataTransfer:
                 "/".join(args.data_source.split("/")[:-1])
             arguments[HDFS_FILENAME] = \
                 self.process_wildcards(args.data_source.split("/")[-1])
-            arguments[KERBEROS_PRINCIPAL] = args.kerberos_principal
             self.__nifi_client.add_default_ckan_parameters(arguments, args)
             arguments[self.__nifi_client.CKAN_RESOURCE] = \
                 self.generate_ckan_resource_name(args)
@@ -943,14 +972,6 @@ class HIDDataTransfer:
                     processors_with_ckan_sensitive_parameters, args
                 )
 
-            processors_with_hdfs_sensitive_parameters = \
-                ["ListHDFS", "FetchHDFS"]
-            sensitive_parameters.update(
-                self.__nifi_client.set_sensitive_hdfs_parameters(
-                    processors_with_hdfs_sensitive_parameters, args
-                )
-            )
-
             # Set the number of concurrent tasks
             # for parallel processors in the pipeline
             if concurrent_tasks is None:
@@ -982,6 +1003,8 @@ class HIDDataTransfer:
                 accounting_target="MergeContent",
                 concurrent_tasks=concurrent_tasks_dict,
                 synchronization_processor_name="MergeContent",
+                processors_with_kerberos=processors_with_kerberos,
+                kerberos_account=kerberos_account,
             )
 
         except Exception as ex:
@@ -1033,6 +1056,12 @@ class HIDDataTransfer:
             kerberos_password=kerberos_password,
         )
 
+        processors_with_kerberos = ["PutHDFS"]
+        kerberos_account = SimpleNamespace(
+            kerberos_principal=kerberos_principal,
+            kerberos_password=kerberos_password,
+        )
+
         self.__logger.info(
             "executing ckan2hdfs command with args: %s",
             self.__nifi_client.format_args_to_string(args),
@@ -1042,8 +1071,20 @@ class HIDDataTransfer:
             self.__nifi_client.add_default_ckan_parameters(arguments, args)
             arguments[self.__nifi_client.CKAN_RESOURCE] = \
                 self.process_wildcards(args.ckan_resource)
-            arguments[HDFS_DATA_FOLDER] = args.data_target
-            arguments[KERBEROS_PRINCIPAL] = args.kerberos_principal
+
+            # If multiple ckan resource files are selected, data_target represents a folder
+            # Else If data_target ends with / represents a folder, otherwise a file
+            arguments[HDFS_DATA_FILENAME] = ""
+            if args.ckan_resource and (
+                "*" in args.ckan_resource or
+                "?" in args.ckan_resource
+            ):
+                arguments[HDFS_DATA_FOLDER] = args.data_target
+            elif args.data_target.endswith('/'):
+                arguments[HDFS_DATA_FOLDER] = args.data_target[:-1]
+            else:
+                arguments[HDFS_DATA_FOLDER] = "/".join(args.data_target.split("/")[:-1])
+                arguments[HDFS_DATA_FILENAME] = "/".join(args.data_target.split("/")[-1:])
 
             execution_type = {
                 "CKANFileDownloader": ExecutionType.ONCE,
@@ -1056,14 +1097,6 @@ class HIDDataTransfer:
                     processors_with_ckan_sensitive_parameters, args
                 )
 
-            processors_with_hdfs_sensitive_parameters = \
-                ["PutHDFS"]
-            sensitive_parameters.update(
-                self.__nifi_client.set_sensitive_hdfs_parameters(
-                    processors_with_hdfs_sensitive_parameters, args
-                )
-            )
-
             # Set the number of concurrent tasks
             # for parallel processors in the pipeline
             if concurrent_tasks is None:
@@ -1095,7 +1128,9 @@ class HIDDataTransfer:
                 sensitive_parameters,
                 keycloak_token=self.__keycloak_token,
                 accounting_target="PutHDFS",
-                concurrent_tasks=concurrent_tasks_dict
+                concurrent_tasks=concurrent_tasks_dict,
+                processors_with_kerberos=processors_with_kerberos,
+                kerberos_account=kerberos_account,
             )
 
         except Exception as ex:
@@ -1169,11 +1204,24 @@ class HIDDataTransfer:
         try:
             arguments: Dict[str, Any] = {}
             self.__nifi_client.add_default_hpc_parameters(arguments, args)
-            arguments[HPC_TARGET_FOLDER] = args.data_target
             self.__nifi_client.add_default_ckan_parameters(arguments, args)
             arguments[self.__nifi_client.CKAN_RESOURCE] = \
                 self.process_wildcards(args.ckan_resource)
 
+            # If multiple ckan resource files are selected, data_target represents a folder
+            # Else If data_target ends with / represents a folder, otherwise a file
+            arguments[HPC_TARGET_FILENAME] = ""
+            if args.ckan_resource and (
+                "*" in args.ckan_resource or
+                "?" in args.ckan_resource
+            ):
+                arguments[HPC_TARGET_FOLDER] = args.data_target
+            elif args.data_target.endswith('/'):
+                arguments[HPC_TARGET_FOLDER] = args.data_target[:-1]
+            else:
+                arguments[HPC_TARGET_FOLDER] = "/".join(args.data_target.split("/")[:-1])
+                arguments[HPC_TARGET_FILENAME] = "/".join(args.data_target.split("/")[-1:])
+
             keys_arguments = {
                 self.__nifi_client.HPC_SECRET_KEY_PATH: arguments[
                     self.__nifi_client.HPC_SECRET_KEY_PATH

{hid_data_transfer_lib-0.3.10 → hid_data_transfer_lib-0.4.0}/hid_data_transfer_lib/keycloak/keycloak_rest.py

@@ -69,12 +69,13 @@ class KeycloakRESTClient:
 
     def configure(
         self, conf: HidDataTransferConfiguration,
-        secure: bool = False, refresh: bool = False
+        secure: bool = False, refresh: bool = False,
+        use_keycloak_account = True,
     ) -> KeycloakRESTClient:
         """constructs a API client,
         with the Keycloak endpoint taken from configuration
         """
-        conf.check_keycloak_conf()
+        conf.check_keycloak_conf(use_keycloak_account)
         self.__conf = conf
         self.__refresh = refresh
         self.__keycloak_login = conf.keycloak_login()
@@ -117,6 +118,7 @@ class KeycloakRESTClient:
 
     def set_token(self, user, token, expires_in, refresh_token):
         """Set the given token and renews it every time it expires"""
+        self.__keycloak_login = user
         self.__token = token
         self.__refresh_token = refresh_token
         self.__set_expires_in(expires_in)

{hid_data_transfer_lib-0.3.10 → hid_data_transfer_lib-0.4.0}/hid_data_transfer_lib/nifi/nifi_client.py

@@ -135,14 +135,15 @@ class NIFIClient:
         self.check_status_sleep_lapse = 1
 
     def configure(self, conf: HidDataTransferConfiguration,
-                  secure: bool = False) -> NIFIClient:
+                  secure: bool = False, use_keycloak_account = True) -> NIFIClient:
         """constructs a NIFI client,
         with the NIFI endpoint taken from configuration
         """
         self.__conf = conf
         self.nifi_rest_client = NIFIRESTClient().configure(conf, secure)
+        self.use_keycloak_account = use_keycloak_account
         self.keycloak_rest_client = KeycloakRESTClient().configure(
-            conf, secure, refresh=True)
+            conf, secure, refresh=True, use_keycloak_account=self.use_keycloak_account)
         self.__logger = self.__conf.logger("nifi.v2.client")
         self.check_status_sleep_lapse = int(
             self.__conf.check_status_sleep_lapse())
@@ -152,14 +153,21 @@ class NIFIClient:
         """gets the NIFI access token,
         using NIFI account taken from configuration
         this access token is required for any further request sent to NIFI
+        If a taken was provided upon configuration, it gets the latest renewed
         """
+
         if self.__conf.nifi_login() and self.__conf.nifi_passwd():
             return self.nifi_rest_client.get_access_token()
-        elif self.__conf.keycloak_login() and self.__conf.keycloak_passwd():
+        # Keycloak token provided, but not an account
+        elif not self.use_keycloak_account:
             return self.keycloak_rest_client.get_token()
-        else:
+        # Keycloak token not provided, check an account is available
+        elif self.use_keycloak_account and self.__conf.keycloak_login() and \
+            self.__conf.keycloak_passwd():
+            return self.keycloak_rest_client.get_token()
+        else: # Not valid token provider configuration found
             raise HidDataTransferException(
-                "NIFI or Keycloak login and password must be provided"
+                "NIFI or Keycloak account (login and password) or Keycloak token must be provided"
             )
 
     def set_access_token(self, keycloak_token):
@@ -192,13 +200,13 @@ class NIFIClient:
     def get_process_group_processors(self, group_id) -> dict:
         """gets the processors of a process group identified by id"""
         return self.nifi_rest_client.process_group_get_request(
-            group_id, self.token(), "processors"
+            group_id, self.token(), entity="processors"
         )
 
     def get_process_group_connections(self, group_id) -> dict:
         """gets the connetions of a process group identified by id"""
         return self.nifi_rest_client.process_group_get_request(
-            group_id, self.token(), "connections"
+            group_id, self.token(), entity="connections"
         )
 
     def get_processors_sequence(self, group_id) -> list:
@@ -324,6 +332,22 @@ class NIFIClient:
         processor["component"]["config"][configuration_name] = \
             configuration_value
 
+    def update_kpus_kerberos_account(
+        self, kpus, kerberos_account,
+    ):
+        """updates the Kerberos account of a kpus"""
+        kpus["component"]["properties"]['Kerberos Principal'] = \
+            kerberos_account.kerberos_principal
+        kpus["component"]["properties"]['Kerberos Password'] = \
+            kerberos_account.kerberos_password
+
+    def update_processor_kus(
+        self, processor, kus_id,
+    ):
+        """updates the KUs of a processor"""
+        processor["component"]['config']["properties"]['Kerberos User Service'] = \
+            kus_id
+
     # DELETE requests
 
     # MULTI-REQUESTS methods
@@ -890,6 +914,8 @@ class NIFIClient:
         accounting_target: Optional[str] = None,
         concurrent_tasks: Optional[dict] = None,
         ignore_bulletins_in_processors=None,
+        processors_with_kerberos=None,
+        kerberos_account=None,
     ) -> tuple[str, AccountingInfo]:
         """MAIN method to run a processor from a template identify by name
         The procedure is as follows:
@@ -946,6 +972,11 @@ class NIFIClient:
             self.__logger.info(
                 "Process group %s created\n", process_group["id"])
 
+            # Set Kerberos configuration
+            if kerberos_account and processors_with_kerberos:
+                self.set_kerberos_configuration(
+                    process_group["id"], kerberos_account, processors_with_kerberos)
+
             # Get ordered list of all processors in the process group
             processors = self.get_ordered_processors_in_group(
                 process_group["id"])
@@ -1102,6 +1133,8 @@ class NIFIClient:
         concurrent_tasks: Optional[dict] = None,
         synchronization_processor_name=None,
         ignore_bulletins_in_processors=None,
+        processors_with_kerberos=None,
+        kerberos_account=None,
     ) -> tuple[str, AccountingInfo]:
         """MAIN method to run a processor from a template identify by name
         The procedure is as follows:
@@ -1127,6 +1160,11 @@ class NIFIClient:
             self.__logger.info(
                 "Process group %s created\n", process_group["id"])
 
+            # Set Kerberos configuration
+            if kerberos_account and processors_with_kerberos:
+                self.set_kerberos_configuration(
+                    process_group["id"], kerberos_account, processors_with_kerberos)
+
             # Accounting initialization
             pipeline_start_time = time.time()
 
@@ -1156,7 +1194,8 @@ class NIFIClient:
                 accounting_target,
                 parameter_context,
                 synchronization_processor_name=synchronization_processor_name,
-                first_processor_expected_outgoing_flowfiles=first_processor_expected_outgoing_flowfiles,
+                first_processor_expected_outgoing_flowfiles=
+                    first_processor_expected_outgoing_flowfiles,
                 ignore_bulletins_in_processors=ignore_bulletins_in_processors,
             )
 
@@ -1338,21 +1377,21 @@ class NIFIClient:
 
         ok, response = self.start_process_group(process_group)
 
-        # Wait for first processor to find source files
-        if not self.has_processor_executed_once_and_produce_outgoing_flowfiles(
-                processors[0]):
-            raise HidDataTransferException("Source files not found")
-
-        # manage the synchronization point if required
-        if synchronization_processor_name:
-            self.synchronize_processor_execution(
-                process_group['id'],
-                synchronization_processor_name,
-                execution_type,
-                first_processor_expected_outgoing_flowfiles,
-                ignore_bulletins_in_processors)
-
         if ok:
+            # Wait for first processor to find source files
+            if not self.has_processor_executed_once_and_produce_outgoing_flowfiles(
+                    processors[0]):
+                raise HidDataTransferException("Source files not found")
+
+            # manage the synchronization point if required
+            if synchronization_processor_name:
+                self.synchronize_processor_execution(
+                    process_group['id'],
+                    synchronization_processor_name,
+                    execution_type,
+                    first_processor_expected_outgoing_flowfiles,
+                    ignore_bulletins_in_processors)
+
             self.wait_for_process_group_completion(
                 process_group['id'], processors,
                 ignore_bulletins_in_processors)
@@ -1401,6 +1440,30 @@ class NIFIClient:
             processor, self.token())
         return processor
 
+    def set_kpus_kerberos_account(
+            self, kpus, kerberos_account):
+        """Set the Kerberos account of a kpus instance"""
+
+        self.update_kpus_kerberos_account(
+            kpus,
+            kerberos_account,
+        )
+        kpus = self.nifi_rest_client.update_kpus(
+            kpus, self.token())
+        return kpus
+
+    def set_processor_kus(
+            self, processor, kus_id):
+        """Set the Kerberos User Service of a processor instance"""
+
+        self.update_processor_kus(
+            processor,
+            kus_id,
+        )
+        processor = self.nifi_rest_client.update_processor(
+            processor, self.token())
+        return processor
+
     def set_2fa_token(self, processor, parameter_context,
                       processor_with_2fa, callback_2fa):
         '''Set the 2FA token in context parameter
@@ -1707,19 +1770,25 @@ class NIFIClient:
                 )
         return sensitive_parameters
 
-    def set_sensitive_hdfs_parameters(
-        self, processors_with_hdfs_sensitive_parameters, args
-    ):
-        """Set sensitive parameters for HDFS connection"""
-        sensitive_parameters = {}
-        for processor in processors_with_hdfs_sensitive_parameters:
-            inner_sensitive_parameters = {}
-            sensitive_parameters[processor] = inner_sensitive_parameters
-            if args.kerberos_password:
-                inner_sensitive_parameters[self.KERBEROS_PASSWORD] = (
-                    args.kerberos_password
-                )
-        return sensitive_parameters
+    def set_kerberos_configuration(self, process_group_id,
+        kerberos_account, processors_with_kerberos):
+        """Set Kerberos configuration for processors"""
+        # Implementation to configure Kerberos for specified processors
+        processors = self.get_ordered_processors_in_group(process_group_id)
+
+        # Create a Kerberos Password User Service (KPUS) associated to a process group
+        kpus = self.nifi_rest_client.instantiate_kpus(process_group_id, self.token())
+        # Set principal and password in KPUS
+        kpus = self.set_kpus_kerberos_account(kpus, kerberos_account)
+        # Enable KUPS
+        kpus = self.nifi_rest_client.enable_kpus(kpus, self.token())
+
+        for processor_name in processors_with_kerberos:
+            # Get processor by name
+            processor = self.find_processor_by_name(processors, processor_name)
+
+            # Save changes in processor
+            self.set_processor_kus(processor, kpus['id'])
 
     def clean_process_group_up(self, process_group_id, retry=True):
         """Clean up the process group and associated parameter context"""
@@ -1729,13 +1798,17 @@ class NIFIClient:
         try:
             # Remove transferred keys
             self.__logger.debug("Removing uploaded keys\n")
-            self.remove_uploaded_keys(process_group["id"])
+            self.remove_uploaded_keys(process_group_id)
 
             # Stop process_group and wait for all processosr to be stopped
-            self.stop_process_group(process_group["id"])
+            self.stop_process_group(process_group_id)
+
+            # Disable controller services
+            self.nifi_rest_client.disable_controller_services(
+                process_group_id, self.token())
 
             # Empty process group queues
-            self.empty_process_group_queues(process_group["id"])
+            self.empty_process_group_queues(process_group_id)
 
             # Get parameter context
             if "parameterContext" in process_group["component"]:
@@ -1759,10 +1832,6 @@ class NIFIClient:
         except HidDataTransferException:
             # If case of exception try clean up again
             if retry:
-                # self.stop_process_group(
-                #     process_group["id"])
-                # self.empty_process_group_queues(
-                #     process_group["id"])
                 self.clean_process_group_up(
                     process_group["id"], retry=False)
 

{hid_data_transfer_lib-0.3.10 → hid_data_transfer_lib-0.4.0}/hid_data_transfer_lib/nifi/nifi_rest.py

@@ -126,14 +126,16 @@ class NIFIRESTClient:
         url = self.build_url("nifi-api", "flow", "process-groups", "root")
         return self._get_entity(url, token)
 
-    def process_group_get_request(self, group_id, token, entity=None) -> dict:
+    def process_group_get_request(
+        self, group_id, token, process_group_flow=False, entity=None) -> dict:
         """helper method to process a get request for a process group
         identified by id"""
-        url = (
-            self.build_url("nifi-api", "process-groups", group_id, entity)
-            if entity
-            else self.build_url("nifi-api", "process-groups", group_id)
-        )
+        if entity:
+            url = self.build_url("nifi-api", "process-groups", group_id, entity)
+        elif process_group_flow:
+            url = self.build_url("nifi-api", "flow", "process-groups", group_id)
+        else:
+            url = self.build_url("nifi-api", "process-groups", group_id)
         return self._get_entity(url, token)
 
     def get_templates(self, token) -> dict:
@@ -318,8 +320,8 @@ class NIFIRESTClient:
             "Content-Type": self.__JSON_CONTENT_TYPE,
             "Authorization": f"Bearer {token}",
         }
-        x = random.uniform(0, 3)
-        y = random.uniform(0, 3)
+        x = random.uniform(0, 1000)
+        y = random.uniform(0, 1000)
         payload = {
             "revision": {"version": 0},
             "disconnectedNodeAcknowledged": False,
@@ -353,10 +355,11 @@ class NIFIRESTClient:
         if response.ok:
             response_json = response.content.decode()
             process_group = json.loads(response_json)
+
             # Rename process_group as flow_definition_name + process_group_id
             process_group_name = process_group["component"]["name"]
             process_group = self.rename_process_group(
-                process_group, process_group_name, token
+                process_group["id"], process_group_name, token
             )
             return process_group
         else:
@@ -365,18 +368,34 @@ class NIFIRESTClient:
                 f"{response.content.decode()}"
             )
 
-    def rename_process_group(self, process_group, name, token) -> str:
-        """creates a process group from a given template"""
+    def rename_process_group(self, process_group_id, name, token) -> str:
+        # TODO Send only what is required to modify the name
+        """rename a process group name"""
+
+        # Reload process_group
+        process_group = self.process_group_get_request(
+            process_group_id, token, process_group_flow=True)
+
         headers = {
             "Content-Type": self.__JSON_CONTENT_TYPE,
             "Authorization": f"Bearer {token}",
         }
-        process_group_id = process_group["id"]
-        process_group["component"]["name"] = name + "_" + process_group_id
-        payload = json.dumps(process_group)
+        process_group_id = process_group["processGroupFlow"]["id"]
+
+        payload = {
+            "component": {
+                "id": process_group_id,
+                "name": name + "_" + process_group_id
+            },
+            "revision": {"version": process_group["revision"]["version"]},
+        }
+
+        # payload = json.dumps(process_group)
+        payload_json = json.dumps(payload)
+
         response = requests.put(
             url=self.build_url("nifi-api", "process-groups", process_group_id),
-            data=payload,
+            data=payload_json,
             verify=self.__conf.nifi_secure_connection(),
             headers=headers,
             timeout=self.__REQUEST_TIMEOUT,
@@ -472,6 +491,51 @@ class NIFIRESTClient:
                 f"{response.content.decode()}"
             )
 
+    def instantiate_kpus(
+        self, process_group_id, token
+    ) -> dict:
+        """creates a Kerberos Password User Service"""
+        headers = {
+            "Content-Type": self.__JSON_CONTENT_TYPE,
+            "Authorization": f"Bearer {token}",
+        }
+
+        payload = {
+            "revision": {"version": 0},
+            "disconnectedNodeAcknowledged": False,
+            "component": {
+                "bundle": { 
+                    "group": "org.apache.nifi", 
+                    "artifact": "nifi-kerberos-user-service-nar", 
+                    "version": "2.8.0" 
+                },
+                "type": "org.apache.nifi.kerberos.KerberosPasswordUserService" 
+            }
+        } # /nifi-api/process-groups/e2dde1d0-019c-1000-b433-43f8611cef38/controller-services
+        payload_json = json.dumps(payload)
+        response = requests.post(
+            url=self.build_url(
+                "nifi-api",
+                "process-groups",
+                process_group_id,
+                "controller-services",
+            ),
+            data=payload_json,
+            verify=self.__conf.nifi_secure_connection(),
+            headers=headers,
+            timeout=self.__REQUEST_TIMEOUT,
+        )
+        if response.ok:
+            response_json = response.content.decode()
+            kpus = json.loads(response_json)
+            return kpus
+        else:
+            raise HidDataTransferException(
+                "Instantiate kpus exception: "
+                f"{response.content.decode()}"
+            )
+
+
     # PUT requests
 
     def associate_parameter_context_to_group(
@@ -486,7 +550,6 @@ class NIFIRESTClient:
         payload = {
             "revision": {
                 "version": process_group["revision"]["version"],
-                # "clientId": parameter_context['revision']['clientId']
             },
             "disconnectedNodeAcknowledged": False,
             "component": {
@@ -541,6 +604,84 @@ class NIFIRESTClient:
                 f"Processor exception: {response.content.decode()}"
             )
 
+    def update_kpus(self, kpus, token) -> dict:
+        """updates a kpus identified by id"""
+        headers = {
+            "Content-Type": self.__JSON_CONTENT_TYPE,
+            "Authorization": f"Bearer {token}",
+        }
+        response = requests.put(
+            url=self.build_url("nifi-api", "controller-services", kpus["id"]),
+            verify=self.__conf.nifi_secure_connection(),
+            data=json.dumps(kpus),
+            headers=headers,
+            timeout=self.__REQUEST_TIMEOUT,
+        )
+        if response.ok:
+            response_json = response.content.decode()
+            return json.loads(response_json)
+        else:
+            raise HidDataTransferException(
+                f"Controller service exception: {response.content.decode()}"
+            )
+
+    def enable_kpus(self, kpus, token) -> dict:
+        """updates a kpus identified by id"""
+        headers = {
+            "Content-Type": self.__JSON_CONTENT_TYPE,
+            "Authorization": f"Bearer {token}",
+        }
+
+        payload = {
+            "revision":{"version":kpus["revision"]["version"]},
+            "state":"ENABLED"
+        }
+        payload_json = json.dumps(payload)
+
+        response = requests.put(
+            url=self.build_url("nifi-api", "controller-services", kpus['id'], "run-status"),
+            verify=self.__conf.nifi_secure_connection(),
+            data=payload_json,
+            headers=headers,
+            timeout=self.__REQUEST_TIMEOUT,
+        )
+        if response.ok:
+            response_json = response.content.decode()
+            return json.loads(response_json)
+        else:
+            raise HidDataTransferException(
+                f"Controller service exception: {response.content.decode()}"
+            )
+
+    def disable_controller_services(self, process_group_id, token):
+        ''' disable the controller services'''
+        headers = {
+            "Content-Type": self.__JSON_CONTENT_TYPE,
+            "Authorization": f"Bearer {token}",
+        }
+
+        payload = {
+            "id": process_group_id,
+            "state":"DISABLED"
+        }
+        payload_json = json.dumps(payload)
+
+        response = requests.put(
+            url=self.build_url(
+                "nifi-api", "flow", "process-groups", process_group_id, "controller-services"),
+            verify=self.__conf.nifi_secure_connection(),
+            data=payload_json,
+            headers=headers,
+            timeout=self.__REQUEST_TIMEOUT,
+        )
+        if response.ok:
+            response_json = response.content.decode()
+            return json.loads(response_json)
+        else:
+            raise HidDataTransferException(
+                f"Process group exception: {response.content.decode()}"
+            )
+
     # DELETE requests
     def delete_update_request(self, entity_name,
                               entity_id, request_id, token) -> bool:

{hid_data_transfer_lib-0.3.10 → hid_data_transfer_lib-0.4.0}/hid_data_transfer_lib/util/util.py

@@ -127,10 +127,12 @@ def sftp_put(
                 sftp.put(source, target)
                 if target_permissions:
                     command = f"chmod {target_permissions} {target}"
-                    ssh.exec_command(command)
+                    _, stdout, _ = ssh.exec_command(command)
+                    stdout.channel.recv_exit_status()  # Block until done, drains buffers
                 if target_ownership:
                     command = f"chown {target_ownership} {target}"
-                    ssh.exec_command(command)
+                    _, stdout, _ = ssh.exec_command(command)
+                    stdout.channel.recv_exit_status()  # Block until done, drains buffers
             except Exception as ex:
                 msg = "Exception transferring keys with SFTP"
                 raise HidDataTransferException(msg) from ex
@@ -189,8 +191,9 @@ def __ssh_execute_command(server, username, command, private_key=None):
             ssh.connect(server, username=username)
         else:
             ssh.connect(server, username=username, key_filename=private_key)
-        _, _, stderr = ssh.exec_command(command)
+        _, stdout, stderr = ssh.exec_command(command)
         error = stderr.read().decode("utf-8")
+        stdout.channel.recv_exit_status()  # Block until done, drains buffers
         if error:
             raise HidDataTransferException(
                 f"Error '{error}' executing command '{command}'"
@@ -217,6 +220,7 @@ def ssh_list(path, server, username, private_key=None, prefix=None) -> list:
         else:
             command = f'find {path} -maxdepth 1 -type f -printf "%p\n"'
         _, stdout, _ = ssh.exec_command(command)
+        stdout.channel.recv_exit_status()  # Block until done, drains buffers
 
         # read lines from stdout in a list
         files = []

{hid_data_transfer_lib-0.3.10 → hid_data_transfer_lib-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "hid_data_transfer_lib"
-version = "0.3.10"
+version = "0.4.0"
 description = "HiDALGO Data Transfer library provides methods to transfer data between different data providers and consumers using NIFI pipelines"
 authors = [
     { name = "Jesús Gorroñogoitia", email = "jesus.gorronogoitia@eviden.com" },
@@ -28,6 +28,7 @@ include = ["py.typed"]
 [tool.poetry.group.dev.dependencies]
 flake8 = "^7.2.0"
 black = "^25.1.0"
+ipykernel = "^7.2.0"
 
 
 [tool.poetry.group.types.dependencies]