hid_data_transfer_lib 0.3.10__tar.gz → 0.3.11__tar.gz

{hid_data_transfer_lib-0.3.10 → hid_data_transfer_lib-0.3.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hid_data_transfer_lib
-Version: 0.3.10
+Version: 0.3.11
 Summary: HiDALGO Data Transfer library provides methods to transfer data between different data providers and consumers using NIFI pipelines
 License: APL-2.0
 Author: Jesús Gorroñogoitia
@@ -51,7 +51,7 @@ The Data Transfer Library sends requests to NIFI through its REST API.
 To use the Data Transfer library, it is required the following requirements:
  - **Python3** execution environment
  - **Poetry** python package management tool (optional)
- - **NIFI** instance, with a NIFI server SSH account (for keys transfer)
+ - **NIFI** instance, with a NIFI 2.X server SSH account (for keys transfer). Latest NIFI supported version: 2.8
  - **Keycloak** instance, with a KEYCLOAK user's account
  - **HDFS** instance, with an user's Kerberos principal account
  - **CKAN** instance, with an user APIKey

{hid_data_transfer_lib-0.3.10 → hid_data_transfer_lib-0.3.11}/README.md

@@ -32,7 +32,7 @@ The Data Transfer Library sends requests to NIFI through its REST API.
 To use the Data Transfer library, it is required the following requirements:
  - **Python3** execution environment
  - **Poetry** python package management tool (optional)
- - **NIFI** instance, with a NIFI server SSH account (for keys transfer)
+ - **NIFI** instance, with a NIFI 2.X server SSH account (for keys transfer). Latest NIFI supported version: 2.8
  - **Keycloak** instance, with a KEYCLOAK user's account
  - **HDFS** instance, with an user's Kerberos principal account
  - **CKAN** instance, with an user APIKey

{hid_data_transfer_lib-0.3.10 → hid_data_transfer_lib-0.3.11}/hid_data_transfer_lib/hid_dt_lib.py

@@ -34,7 +34,6 @@ from hid_data_transfer_lib.nifi.nifi_client import (
     NIFIClient, ExecutionType, AccountingInfo
 )
 
-KERBEROS_PRINCIPAL = "kerberos.principal"
 HDFS_DATA_FOLDER = "hdfs.data_folder"
 HDFS_FILENAME = "hdfs.filename"
 HPC_TARGET_FOLDER = "hpc.target_folder"
@@ -158,9 +157,13 @@ class HIDDataTransfer:
             hpc_secret_key=hpc_secret_key_path,
             data_source=data_source,
             data_target=data_target,
+            recursive=recursive,
+        )
+
+        processors_with_kerberos = ["ListHDFS", "FetchHDFS"]
+        kerberos_account = SimpleNamespace(
             kerberos_principal=kerberos_principal,
             kerberos_password=kerberos_password,
-            recursive=recursive,
         )
 
         self.__logger.info(
@@ -175,7 +178,7 @@ class HIDDataTransfer:
                 "/".join(args.data_source.split("/")[:-1])
             arguments[HDFS_FILENAME] = \
                 self.process_wildcards(args.data_source.split("/")[-1])
-            arguments[KERBEROS_PRINCIPAL] = args.kerberos_principal
+
             arguments[RECURSIVE] = args.recursive
 
             execution_type = {
@@ -197,14 +200,6 @@ class HIDDataTransfer:
                     args
                 )
 
-            processors_with_hdfs_sensitive_parameters = \
-                ["ListHDFS", "FetchHDFS"]
-            sensitive_parameters.update(
-                self.__nifi_client.set_sensitive_hdfs_parameters(
-                    processors_with_hdfs_sensitive_parameters, args
-                )
-            )
-
             if not args.hpc_secret_key and not args.hpc_password:
                 raise HidDataTransferException(HPC_ACCOUNT_MSG)
 
@@ -227,6 +222,8 @@ class HIDDataTransfer:
                 accounting_target="PutSFTP",
                 concurrent_tasks=concurrent_tasks_dict,
                 ignore_bulletins_in_processors=["PutSFTP"],
+                processors_with_kerberos=processors_with_kerberos,
+                kerberos_account=kerberos_account,
             )
 
         except Exception as ex:
@@ -387,6 +384,12 @@ class HIDDataTransfer:
             kerberos_password=kerberos_password,
         )
 
+        processors_with_kerberos = ["ListHDFS", "FetchHDFS"]
+        kerberos_account = SimpleNamespace(
+            kerberos_principal=kerberos_principal,
+            kerberos_password=kerberos_password,
+        )
+
         self.__logger.info(
             "executing hdfs2hpc_2fa command with args: %s",
             self.__nifi_client.format_args_to_string(args),
@@ -399,7 +402,6 @@ class HIDDataTransfer:
                 "/".join(args.data_source.split("/")[:-1])
             arguments[HDFS_FILENAME] = \
                 self.process_wildcards(args.data_source.split("/")[-1])
-            arguments[KERBEROS_PRINCIPAL] = args.kerberos_principal
 
             # set local.data_folder to keep dataset retrieved from HDFS
             __uuid = uuid.uuid4()
@@ -422,16 +424,6 @@ class HIDDataTransfer:
                 ],
             }
 
-            sensitive_parameters = {}
-
-            processors_with_hdfs_sensitive_parameters = \
-                ["ListHDFS", "FetchHDFS"]
-            sensitive_parameters.update(
-                self.__nifi_client.set_sensitive_hdfs_parameters(
-                    processors_with_hdfs_sensitive_parameters, args
-                )
-            )
-
             if not args.hpc_secret_key:
                 raise HidDataTransferException(HPC_ACCOUNT_MSG)
 
@@ -450,10 +442,12 @@ class HIDDataTransfer:
                 execution_type,
                 arguments,
                 keys_arguments=keys_arguments,
-                sensitive_parameters=sensitive_parameters,
+                sensitive_parameters=None,
                 keycloak_token=self.__keycloak_token,
                 accounting_target="PutFile",
                 concurrent_tasks=concurrent_tasks_dict,
+                processors_with_kerberos=processors_with_kerberos,
+                kerberos_account=kerberos_account,
             )
             process_ids.append(process_id)
 
@@ -462,7 +456,7 @@ class HIDDataTransfer:
                 execution_type,
                 arguments,
                 keys_arguments=keys_arguments,
-                sensitive_parameters=sensitive_parameters,
+                sensitive_parameters=None,
                 keycloak_token=self.__keycloak_token,
                 processor_with_2fa="SftpTransferToHPCWithVPNCode",
                 callback_2fa=callback_2fa,
@@ -536,6 +530,12 @@ class HIDDataTransfer:
             recursive=recursive,
         )
 
+        processors_with_kerberos = ["PutHDFS"]
+        kerberos_account = SimpleNamespace(
+            kerberos_principal=kerberos_principal,
+            kerberos_password=kerberos_password,
+        )
+
         self.__logger.info(
             "executing hpc2hdfs command with args: %s",
             self.__nifi_client.format_args_to_string(args),
@@ -549,7 +549,6 @@ class HIDDataTransfer:
                 : args.data_source.rfind("/")
             ]
             arguments[HDFS_DATA_FOLDER] = args.data_target
-            arguments[KERBEROS_PRINCIPAL] = args.kerberos_principal
             arguments[RECURSIVE] = args.recursive
 
             execution_type = {
@@ -572,14 +571,6 @@ class HIDDataTransfer:
                     processors_with_hpc_sensitive_parameters, args
                 )
 
-            processors_with_hdfs_sensitive_parameters = \
-                ["PutHDFS"]
-            sensitive_parameters.update(
-                self.__nifi_client.set_sensitive_hdfs_parameters(
-                    processors_with_hdfs_sensitive_parameters, args
-                )
-            )
-
             if not args.hpc_secret_key and not args.hpc_password:
                 raise HidDataTransferException(HPC_ACCOUNT_MSG)
 
@@ -602,6 +593,8 @@ class HIDDataTransfer:
                 accounting_target="PutHDFS",
                 concurrent_tasks=concurrent_tasks_dict,
                 ignore_bulletins_in_processors=["FetchSFTP"],
+                processors_with_kerberos=processors_with_kerberos,
+                kerberos_account=kerberos_account,
             )
 
         except Exception as ex:
@@ -712,6 +705,12 @@ class HIDDataTransfer:
             kerberos_password=kerberos_password,
         )
 
+        processors_with_kerberos = ["PutHDFS"]
+        kerberos_account = SimpleNamespace(
+            kerberos_principal=kerberos_principal,
+            kerberos_password=kerberos_password,
+        )
+
         self.__logger.info(
             "executing hpc2hdfs_2fa command with args: %s",
             self.__nifi_client.format_args_to_string(args),
@@ -728,7 +727,6 @@ class HIDDataTransfer:
                 : args.data_source.rfind("/")
             ]
             arguments[HDFS_DATA_FOLDER] = args.data_target
-            arguments[KERBEROS_PRINCIPAL] = args.kerberos_principal
 
             # set local.data_folder to keep dataset retrieved from HPC
             __uuid = uuid.uuid4()
@@ -758,14 +756,6 @@ class HIDDataTransfer:
                     processors_with_hpc_sensitive_parameters, args
                 )
 
-            processors_with_hdfs_sensitive_parameters = \
-                ["PutHDFS"]
-            sensitive_parameters.update(
-                self.__nifi_client.set_sensitive_hdfs_parameters(
-                    processors_with_hdfs_sensitive_parameters, args
-                )
-            )
-
             if not args.hpc_secret_key and not args.hpc_password:
                 raise HidDataTransferException(HPC_ACCOUNT_MSG)
 
@@ -788,7 +778,6 @@ class HIDDataTransfer:
                 keycloak_token=self.__keycloak_token,
                 processor_with_2fa="SftpTransferFromHPCWithVPNCode",
                 callback_2fa=callback_2fa,
-                accounting_target="PutHDFS",
                 concurrent_tasks=concurrent_tasks_dict,
             )
             process_ids.append(process_id)
@@ -802,6 +791,8 @@ class HIDDataTransfer:
                 keycloak_token=self.__keycloak_token,
                 accounting_target="PutHDFS",
                 concurrent_tasks=concurrent_tasks_dict,
+                processors_with_kerberos=processors_with_kerberos,
+                kerberos_account=kerberos_account,
             )
             process_ids.append(process_id)
 
@@ -913,6 +904,12 @@ class HIDDataTransfer:
             recursive=recursive,
         )
 
+        processors_with_kerberos = ["ListHDFS", "FetchHDFS"]
+        kerberos_account = SimpleNamespace(
+            kerberos_principal=kerberos_principal,
+            kerberos_password=kerberos_password,
+        )
+
         self.__logger.info(
             "executing hdfs2ckan command with args: %s",
             self.__nifi_client.format_args_to_string(args),
@@ -923,7 +920,6 @@ class HIDDataTransfer:
                 "/".join(args.data_source.split("/")[:-1])
             arguments[HDFS_FILENAME] = \
                 self.process_wildcards(args.data_source.split("/")[-1])
-            arguments[KERBEROS_PRINCIPAL] = args.kerberos_principal
             self.__nifi_client.add_default_ckan_parameters(arguments, args)
             arguments[self.__nifi_client.CKAN_RESOURCE] = \
                 self.generate_ckan_resource_name(args)
@@ -943,14 +939,6 @@ class HIDDataTransfer:
                     processors_with_ckan_sensitive_parameters, args
                 )
 
-            processors_with_hdfs_sensitive_parameters = \
-                ["ListHDFS", "FetchHDFS"]
-            sensitive_parameters.update(
-                self.__nifi_client.set_sensitive_hdfs_parameters(
-                    processors_with_hdfs_sensitive_parameters, args
-                )
-            )
-
             # Set the number of concurrent tasks
             # for parallel processors in the pipeline
             if concurrent_tasks is None:
@@ -982,6 +970,8 @@ class HIDDataTransfer:
                 accounting_target="MergeContent",
                 concurrent_tasks=concurrent_tasks_dict,
                 synchronization_processor_name="MergeContent",
+                processors_with_kerberos=processors_with_kerberos,
+                kerberos_account=kerberos_account,
             )
 
         except Exception as ex:
@@ -1033,6 +1023,12 @@ class HIDDataTransfer:
             kerberos_password=kerberos_password,
         )
 
+        processors_with_kerberos = ["PutHDFS"]
+        kerberos_account = SimpleNamespace(
+            kerberos_principal=kerberos_principal,
+            kerberos_password=kerberos_password,
+        )
+
         self.__logger.info(
             "executing ckan2hdfs command with args: %s",
             self.__nifi_client.format_args_to_string(args),
@@ -1043,7 +1039,6 @@ class HIDDataTransfer:
             arguments[self.__nifi_client.CKAN_RESOURCE] = \
                 self.process_wildcards(args.ckan_resource)
             arguments[HDFS_DATA_FOLDER] = args.data_target
-            arguments[KERBEROS_PRINCIPAL] = args.kerberos_principal
 
             execution_type = {
                 "CKANFileDownloader": ExecutionType.ONCE,
@@ -1056,14 +1051,6 @@ class HIDDataTransfer:
                     processors_with_ckan_sensitive_parameters, args
                 )
 
-            processors_with_hdfs_sensitive_parameters = \
-                ["PutHDFS"]
-            sensitive_parameters.update(
-                self.__nifi_client.set_sensitive_hdfs_parameters(
-                    processors_with_hdfs_sensitive_parameters, args
-                )
-            )
-
             # Set the number of concurrent tasks
             # for parallel processors in the pipeline
             if concurrent_tasks is None:
@@ -1095,7 +1082,9 @@ class HIDDataTransfer:
                 sensitive_parameters,
                 keycloak_token=self.__keycloak_token,
                 accounting_target="PutHDFS",
-                concurrent_tasks=concurrent_tasks_dict
+                concurrent_tasks=concurrent_tasks_dict,
+                processors_with_kerberos=processors_with_kerberos,
+                kerberos_account=kerberos_account,
             )
 
         except Exception as ex:

{hid_data_transfer_lib-0.3.10 → hid_data_transfer_lib-0.3.11}/hid_data_transfer_lib/nifi/nifi_client.py

@@ -192,13 +192,13 @@ class NIFIClient:
     def get_process_group_processors(self, group_id) -> dict:
         """gets the processors of a process group identified by id"""
         return self.nifi_rest_client.process_group_get_request(
-            group_id, self.token(), "processors"
+            group_id, self.token(), entity="processors"
         )
 
     def get_process_group_connections(self, group_id) -> dict:
         """gets the connetions of a process group identified by id"""
         return self.nifi_rest_client.process_group_get_request(
-            group_id, self.token(), "connections"
+            group_id, self.token(), entity="connections"
         )
 
     def get_processors_sequence(self, group_id) -> list:
@@ -324,6 +324,22 @@ class NIFIClient:
         processor["component"]["config"][configuration_name] = \
             configuration_value
 
+    def update_kpus_kerberos_account(
+        self, kpus, kerberos_account,
+    ):
+        """updates the Kerberos account of a kpus"""
+        kpus["component"]["properties"]['Kerberos Principal'] = \
+            kerberos_account.kerberos_principal
+        kpus["component"]["properties"]['Kerberos Password'] = \
+            kerberos_account.kerberos_password
+
+    def update_processor_kus(
+        self, processor, kus_id,
+    ):
+        """updates the KUs of a processor"""
+        processor["component"]['config']["properties"]['Kerberos User Service'] = \
+            kus_id
+
     # DELETE requests
 
     # MULTI-REQUESTS methods
@@ -890,6 +906,8 @@ class NIFIClient:
         accounting_target: Optional[str] = None,
         concurrent_tasks: Optional[dict] = None,
         ignore_bulletins_in_processors=None,
+        processors_with_kerberos=None,
+        kerberos_account=None,
     ) -> tuple[str, AccountingInfo]:
         """MAIN method to run a processor from a template identify by name
         The procedure is as follows:
@@ -946,6 +964,11 @@ class NIFIClient:
             self.__logger.info(
                 "Process group %s created\n", process_group["id"])
 
+            # Set Kerberos configuration
+            if kerberos_account and processors_with_kerberos:
+                self.set_kerberos_configuration(
+                    process_group["id"], kerberos_account, processors_with_kerberos)
+
             # Get ordered list of all processors in the process group
             processors = self.get_ordered_processors_in_group(
                 process_group["id"])
@@ -1102,6 +1125,8 @@ class NIFIClient:
         concurrent_tasks: Optional[dict] = None,
         synchronization_processor_name=None,
         ignore_bulletins_in_processors=None,
+        processors_with_kerberos=None,
+        kerberos_account=None,
     ) -> tuple[str, AccountingInfo]:
         """MAIN method to run a processor from a template identify by name
         The procedure is as follows:
@@ -1127,6 +1152,11 @@ class NIFIClient:
             self.__logger.info(
                 "Process group %s created\n", process_group["id"])
 
+            # Set Kerberos configuration
+            if kerberos_account and processors_with_kerberos:
+                self.set_kerberos_configuration(
+                    process_group["id"], kerberos_account, processors_with_kerberos)
+
             # Accounting initialization
             pipeline_start_time = time.time()
 
@@ -1156,7 +1186,8 @@ class NIFIClient:
                 accounting_target,
                 parameter_context,
                 synchronization_processor_name=synchronization_processor_name,
-                first_processor_expected_outgoing_flowfiles=first_processor_expected_outgoing_flowfiles,
+                first_processor_expected_outgoing_flowfiles=
+                    first_processor_expected_outgoing_flowfiles,
                 ignore_bulletins_in_processors=ignore_bulletins_in_processors,
             )
 
@@ -1338,21 +1369,21 @@ class NIFIClient:
 
         ok, response = self.start_process_group(process_group)
 
-        # Wait for first processor to find source files
-        if not self.has_processor_executed_once_and_produce_outgoing_flowfiles(
-                processors[0]):
-            raise HidDataTransferException("Source files not found")
-
-        # manage the synchronization point if required
-        if synchronization_processor_name:
-            self.synchronize_processor_execution(
-                process_group['id'],
-                synchronization_processor_name,
-                execution_type,
-                first_processor_expected_outgoing_flowfiles,
-                ignore_bulletins_in_processors)
-
         if ok:
+            # Wait for first processor to find source files
+            if not self.has_processor_executed_once_and_produce_outgoing_flowfiles(
+                    processors[0]):
+                raise HidDataTransferException("Source files not found")
+
+            # manage the synchronization point if required
+            if synchronization_processor_name:
+                self.synchronize_processor_execution(
+                    process_group['id'],
+                    synchronization_processor_name,
+                    execution_type,
+                    first_processor_expected_outgoing_flowfiles,
+                    ignore_bulletins_in_processors)
+
             self.wait_for_process_group_completion(
                 process_group['id'], processors,
                 ignore_bulletins_in_processors)
@@ -1401,6 +1432,30 @@ class NIFIClient:
             processor, self.token())
         return processor
 
+    def set_kpus_kerberos_account(
+            self, kpus, kerberos_account):
+        """Set the Kerberos account of a kpus instance"""
+
+        self.update_kpus_kerberos_account(
+            kpus,
+            kerberos_account,
+        )
+        kpus = self.nifi_rest_client.update_kpus(
+            kpus, self.token())
+        return kpus
+
+    def set_processor_kus(
+            self, processor, kus_id):
+        """Set the Kerberos User Service of a processor instance"""
+
+        self.update_processor_kus(
+            processor,
+            kus_id,
+        )
+        processor = self.nifi_rest_client.update_processor(
+            processor, self.token())
+        return processor
+
     def set_2fa_token(self, processor, parameter_context,
                       processor_with_2fa, callback_2fa):
         '''Set the 2FA token in context parameter
@@ -1707,19 +1762,25 @@ class NIFIClient:
                 )
         return sensitive_parameters
 
-    def set_sensitive_hdfs_parameters(
-        self, processors_with_hdfs_sensitive_parameters, args
-    ):
-        """Set sensitive parameters for HDFS connection"""
-        sensitive_parameters = {}
-        for processor in processors_with_hdfs_sensitive_parameters:
-            inner_sensitive_parameters = {}
-            sensitive_parameters[processor] = inner_sensitive_parameters
-            if args.kerberos_password:
-                inner_sensitive_parameters[self.KERBEROS_PASSWORD] = (
-                    args.kerberos_password
-                )
-        return sensitive_parameters
+    def set_kerberos_configuration(self, process_group_id,
+        kerberos_account, processors_with_kerberos):
+        """Set Kerberos configuration for processors"""
+        # Implementation to configure Kerberos for specified processors
+        processors = self.get_ordered_processors_in_group(process_group_id)
+
+        # Create a Kerberos Password User Service (KPUS) associated to a process group
+        kpus = self.nifi_rest_client.instantiate_kpus(process_group_id, self.token())
+        # Set principal and password in KPUS
+        kpus = self.set_kpus_kerberos_account(kpus, kerberos_account)
+        # Enable KUPS
+        kpus = self.nifi_rest_client.enable_kpus(kpus, self.token())
+
+        for processor_name in processors_with_kerberos:
+            # Get processor by name
+            processor = self.find_processor_by_name(processors, processor_name)
+
+            # Save changes in processor
+            self.set_processor_kus(processor, kpus['id'])
 
     def clean_process_group_up(self, process_group_id, retry=True):
         """Clean up the process group and associated parameter context"""
@@ -1729,13 +1790,17 @@ class NIFIClient:
         try:
             # Remove transferred keys
             self.__logger.debug("Removing uploaded keys\n")
-            self.remove_uploaded_keys(process_group["id"])
+            self.remove_uploaded_keys(process_group_id)
 
             # Stop process_group and wait for all processosr to be stopped
-            self.stop_process_group(process_group["id"])
+            self.stop_process_group(process_group_id)
+
+            # Disable controller services
+            self.nifi_rest_client.disable_controller_services(
+                process_group_id, self.token())
 
             # Empty process group queues
-            self.empty_process_group_queues(process_group["id"])
+            self.empty_process_group_queues(process_group_id)
 
             # Get parameter context
             if "parameterContext" in process_group["component"]:
@@ -1759,10 +1824,6 @@ class NIFIClient:
         except HidDataTransferException:
             # If case of exception try clean up again
             if retry:
-                # self.stop_process_group(
-                #     process_group["id"])
-                # self.empty_process_group_queues(
-                #     process_group["id"])
                 self.clean_process_group_up(
                     process_group["id"], retry=False)
 

{hid_data_transfer_lib-0.3.10 → hid_data_transfer_lib-0.3.11}/hid_data_transfer_lib/nifi/nifi_rest.py

@@ -126,14 +126,16 @@ class NIFIRESTClient:
         url = self.build_url("nifi-api", "flow", "process-groups", "root")
         return self._get_entity(url, token)
 
-    def process_group_get_request(self, group_id, token, entity=None) -> dict:
+    def process_group_get_request(
+        self, group_id, token, process_group_flow=False, entity=None) -> dict:
         """helper method to process a get request for a process group
         identified by id"""
-        url = (
-            self.build_url("nifi-api", "process-groups", group_id, entity)
-            if entity
-            else self.build_url("nifi-api", "process-groups", group_id)
-        )
+        if entity:
+            url = self.build_url("nifi-api", "process-groups", group_id, entity)
+        elif process_group_flow:
+            url = self.build_url("nifi-api", "flow", "process-groups", group_id)
+        else:
+            url = self.build_url("nifi-api", "process-groups", group_id)
         return self._get_entity(url, token)
 
     def get_templates(self, token) -> dict:
@@ -318,8 +320,8 @@ class NIFIRESTClient:
             "Content-Type": self.__JSON_CONTENT_TYPE,
             "Authorization": f"Bearer {token}",
         }
-        x = random.uniform(0, 3)
-        y = random.uniform(0, 3)
+        x = random.uniform(0, 1000)
+        y = random.uniform(0, 1000)
         payload = {
             "revision": {"version": 0},
             "disconnectedNodeAcknowledged": False,
@@ -353,10 +355,11 @@ class NIFIRESTClient:
         if response.ok:
             response_json = response.content.decode()
             process_group = json.loads(response_json)
+
             # Rename process_group as flow_definition_name + process_group_id
             process_group_name = process_group["component"]["name"]
             process_group = self.rename_process_group(
-                process_group, process_group_name, token
+                process_group["id"], process_group_name, token
             )
             return process_group
         else:
@@ -365,18 +368,34 @@ class NIFIRESTClient:
                 f"{response.content.decode()}"
             )
 
-    def rename_process_group(self, process_group, name, token) -> str:
-        """creates a process group from a given template"""
+    def rename_process_group(self, process_group_id, name, token) -> str:
+        # TODO Send only what is required to modify the name
+        """rename a process group name"""
+
+        # Reload process_group
+        process_group = self.process_group_get_request(
+            process_group_id, token, process_group_flow=True)
+
         headers = {
             "Content-Type": self.__JSON_CONTENT_TYPE,
             "Authorization": f"Bearer {token}",
         }
-        process_group_id = process_group["id"]
-        process_group["component"]["name"] = name + "_" + process_group_id
-        payload = json.dumps(process_group)
+        process_group_id = process_group["processGroupFlow"]["id"]
+
+        payload = {
+            "component": {
+                "id": process_group_id,
+                "name": name + "_" + process_group_id
+            },
+            "revision": {"version": process_group["revision"]["version"]},
+        }
+
+        # payload = json.dumps(process_group)
+        payload_json = json.dumps(payload)
+
         response = requests.put(
             url=self.build_url("nifi-api", "process-groups", process_group_id),
-            data=payload,
+            data=payload_json,
             verify=self.__conf.nifi_secure_connection(),
             headers=headers,
             timeout=self.__REQUEST_TIMEOUT,
@@ -472,6 +491,51 @@ class NIFIRESTClient:
                 f"{response.content.decode()}"
             )
 
+    def instantiate_kpus(
+        self, process_group_id, token
+    ) -> dict:
+        """creates a Kerberos Password User Service"""
+        headers = {
+            "Content-Type": self.__JSON_CONTENT_TYPE,
+            "Authorization": f"Bearer {token}",
+        }
+
+        payload = {
+            "revision": {"version": 0},
+            "disconnectedNodeAcknowledged": False,
+            "component": {
+                "bundle": { 
+                    "group": "org.apache.nifi", 
+                    "artifact": "nifi-kerberos-user-service-nar", 
+                    "version": "2.8.0" 
+                },
+                "type": "org.apache.nifi.kerberos.KerberosPasswordUserService" 
+            }
+        } # /nifi-api/process-groups/e2dde1d0-019c-1000-b433-43f8611cef38/controller-services
+        payload_json = json.dumps(payload)
+        response = requests.post(
+            url=self.build_url(
+                "nifi-api",
+                "process-groups",
+                process_group_id,
+                "controller-services",
+            ),
+            data=payload_json,
+            verify=self.__conf.nifi_secure_connection(),
+            headers=headers,
+            timeout=self.__REQUEST_TIMEOUT,
+        )
+        if response.ok:
+            response_json = response.content.decode()
+            kpus = json.loads(response_json)
+            return kpus
+        else:
+            raise HidDataTransferException(
+                "Instantiate kpus exception: "
+                f"{response.content.decode()}"
+            )
+
+
     # PUT requests
 
     def associate_parameter_context_to_group(
@@ -486,7 +550,6 @@ class NIFIRESTClient:
         payload = {
             "revision": {
                 "version": process_group["revision"]["version"],
-                # "clientId": parameter_context['revision']['clientId']
             },
             "disconnectedNodeAcknowledged": False,
             "component": {
@@ -541,6 +604,84 @@ class NIFIRESTClient:
                 f"Processor exception: {response.content.decode()}"
             )
 
+    def update_kpus(self, kpus, token) -> dict:
+        """updates a kpus identified by id"""
+        headers = {
+            "Content-Type": self.__JSON_CONTENT_TYPE,
+            "Authorization": f"Bearer {token}",
+        }
+        response = requests.put(
+            url=self.build_url("nifi-api", "controller-services", kpus["id"]),
+            verify=self.__conf.nifi_secure_connection(),
+            data=json.dumps(kpus),
+            headers=headers,
+            timeout=self.__REQUEST_TIMEOUT,
+        )
+        if response.ok:
+            response_json = response.content.decode()
+            return json.loads(response_json)
+        else:
+            raise HidDataTransferException(
+                f"Controller service exception: {response.content.decode()}"
+            )
+
+    def enable_kpus(self, kpus, token) -> dict:
+        """updates a kpus identified by id"""
+        headers = {
+            "Content-Type": self.__JSON_CONTENT_TYPE,
+            "Authorization": f"Bearer {token}",
+        }
+
+        payload = {
+            "revision":{"version":kpus["revision"]["version"]},
+            "state":"ENABLED"
+        }
+        payload_json = json.dumps(payload)
+
+        response = requests.put(
+            url=self.build_url("nifi-api", "controller-services", kpus['id'], "run-status"),
+            verify=self.__conf.nifi_secure_connection(),
+            data=payload_json,
+            headers=headers,
+            timeout=self.__REQUEST_TIMEOUT,
+        )
+        if response.ok:
+            response_json = response.content.decode()
+            return json.loads(response_json)
+        else:
+            raise HidDataTransferException(
+                f"Controller service exception: {response.content.decode()}"
+            )
+
+    def disable_controller_services(self, process_group_id, token):
+        ''' disable the controller services'''
+        headers = {
+            "Content-Type": self.__JSON_CONTENT_TYPE,
+            "Authorization": f"Bearer {token}",
+        }
+
+        payload = {
+            "id": process_group_id,
+            "state":"DISABLED"
+        }
+        payload_json = json.dumps(payload)
+
+        response = requests.put(
+            url=self.build_url(
+                "nifi-api", "flow", "process-groups", process_group_id, "controller-services"),
+            verify=self.__conf.nifi_secure_connection(),
+            data=payload_json,
+            headers=headers,
+            timeout=self.__REQUEST_TIMEOUT,
+        )
+        if response.ok:
+            response_json = response.content.decode()
+            return json.loads(response_json)
+        else:
+            raise HidDataTransferException(
+                f"Process group exception: {response.content.decode()}"
+            )
+
     # DELETE requests
     def delete_update_request(self, entity_name,
                               entity_id, request_id, token) -> bool:

{hid_data_transfer_lib-0.3.10 → hid_data_transfer_lib-0.3.11}/hid_data_transfer_lib/util/util.py

@@ -127,10 +127,12 @@ def sftp_put(
                 sftp.put(source, target)
                 if target_permissions:
                     command = f"chmod {target_permissions} {target}"
-                    ssh.exec_command(command)
+                    _, stdout, _ = ssh.exec_command(command)
+                    stdout.channel.recv_exit_status()  # Block until done, drains buffers
                 if target_ownership:
                     command = f"chown {target_ownership} {target}"
-                    ssh.exec_command(command)
+                    _, stdout, _ = ssh.exec_command(command)
+                    stdout.channel.recv_exit_status()  # Block until done, drains buffers
             except Exception as ex:
                 msg = "Exception transferring keys with SFTP"
                 raise HidDataTransferException(msg) from ex
@@ -189,8 +191,9 @@ def __ssh_execute_command(server, username, command, private_key=None):
             ssh.connect(server, username=username)
         else:
             ssh.connect(server, username=username, key_filename=private_key)
-        _, _, stderr = ssh.exec_command(command)
+        _, stdout, stderr = ssh.exec_command(command)
         error = stderr.read().decode("utf-8")
+        stdout.channel.recv_exit_status()  # Block until done, drains buffers
         if error:
             raise HidDataTransferException(
                 f"Error '{error}' executing command '{command}'"
@@ -217,6 +220,7 @@ def ssh_list(path, server, username, private_key=None, prefix=None) -> list:
         else:
             command = f'find {path} -maxdepth 1 -type f -printf "%p\n"'
         _, stdout, _ = ssh.exec_command(command)
+        stdout.channel.recv_exit_status()  # Block until done, drains buffers
 
         # read lines from stdout in a list
         files = []

{hid_data_transfer_lib-0.3.10 → hid_data_transfer_lib-0.3.11}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "hid_data_transfer_lib"
-version = "0.3.10"
+version = "0.3.11"
 description = "HiDALGO Data Transfer library provides methods to transfer data between different data providers and consumers using NIFI pipelines"
 authors = [
     { name = "Jesús Gorroñogoitia", email = "jesus.gorronogoitia@eviden.com" },