hid_data_transfer_lib 0.2.0__tar.gz → 0.2.2__tar.gz

{hid_data_transfer_lib-0.2.0 → hid_data_transfer_lib-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hid_data_transfer_lib
-Version: 0.2.0
+Version: 0.2.2
 Summary: HiDALGO Data Transfer library provides methods to transfer data between different data providers and consumers using NIFI pipelines
 License: APL-2.0
 Author: Jesús Gorroñogoitia

{hid_data_transfer_lib-0.2.0 → hid_data_transfer_lib-0.2.2}/hid_data_transfer_lib/conf/hid_dt_configuration.py

@@ -191,6 +191,23 @@ class HidDataTransferConfiguration:
         """Returns the logging level"""
         return self.__set_logger(self.__logging_level, logger_name)
 
+    def is_logger_valid(self, logger):
+        """Check if the logger is valid:
+        checks if the FileHandler file is still open
+        """
+        file_handler = self.__get_file_handler(logger)
+        if file_handler is not None:
+            return not file_handler.stream.closed
+        else:
+            return False
+
+    def __get_file_handler(self, logger):
+        '''Returns the FileHandler of the logger'''
+        for handler in logger.handlers:
+            if isinstance(handler, logging.FileHandler):
+                return handler
+        return None
+
     def __get_configuration(self):
         """
         Reads the configuration file and sets this class' configuration values

{hid_data_transfer_lib-0.2.0 → hid_data_transfer_lib-0.2.2}/hid_data_transfer_lib/hid_dt_lib.py

@@ -32,6 +32,15 @@ from hid_data_transfer_lib.exceptions.hid_dt_exceptions import (
 )
 from hid_data_transfer_lib.nifi.nifi_client import NIFIClient, ExecutionType
 
+KERBEROS_PRINCIPAL = "kerberos.principal"
+HDFS_DATA_FOLDER = "hdfs.data_folder"
+HDFS_FILENAME = "hdfs.filename"
+HPC_TARGET_FOLDER = "hpc.target_folder"
+HPC_DATA_FOLDER = "hpc.data_folder"
+HPC_FILENAME = "hpc.filename"
+
+HPC_ACCOUNT_MSG = "Either hpc_password or hpc_secret_key must be provided"
+
 
 class HIDDataTransfer:
     """main HiDALGO  API client class
@@ -104,12 +113,12 @@ class HIDDataTransfer:
         try:
             arguments: Dict[str, Any] = {}
             self.__nifi_client.add_default_hpc_parameters(arguments, args)
-            arguments["hpc.target_folder"] = args.data_target
-            arguments["hdfs.data_folder"] = \
+            arguments[HPC_TARGET_FOLDER] = args.data_target
+            arguments[HDFS_DATA_FOLDER] = \
                 "/".join(args.data_source.split("/")[:-1])
-            arguments["hdfs.filename"] = \
+            arguments[HDFS_FILENAME] = \
                 self.process_wildcards(args.data_source.split("/")[-1])
-            arguments["kerberos.principal"] = args.kerberos_principal
+            arguments[KERBEROS_PRINCIPAL] = args.kerberos_principal
 
             execution_type = {
                 "ListHDFS": ExecutionType.ONCE,
@@ -138,9 +147,7 @@ class HIDDataTransfer:
             )
 
             if not args.hpc_secret_key and not args.hpc_password:
-                raise HidDataTransferException(
-                    "Either hpc_password or hpc_secret_key must be provided"
-                )
+                raise HidDataTransferException(HPC_ACCOUNT_MSG)
 
             return self.__nifi_client.run_process_group(
                 "hdfs2hpc",
@@ -219,12 +226,12 @@ class HIDDataTransfer:
             hpc_filename = args.data_source[args.data_source.rfind("/") + 1 :]
             arguments: Dict[str, Any] = {}
             self.__nifi_client.add_default_hpc_parameters(arguments, args)
-            arguments["hpc.filename"] = hpc_filename if hpc_filename else ".*"
-            arguments["hpc.data_folder"] = args.data_source[
+            arguments[HPC_FILENAME] = hpc_filename if hpc_filename else ".*"
+            arguments[HPC_DATA_FOLDER] = args.data_source[
                 : args.data_source.rfind("/")
             ]
-            arguments["hdfs.data_folder"] = args.data_target
-            arguments["kerberos.principal"] = args.kerberos_principal
+            arguments[HDFS_DATA_FOLDER] = args.data_target
+            arguments[KERBEROS_PRINCIPAL] = args.kerberos_principal
 
             execution_type = {
                 "ListSFTP": ExecutionType.ONCE,
@@ -253,9 +260,7 @@ class HIDDataTransfer:
             )
 
             if not args.hpc_secret_key and not args.hpc_password:
-                raise HidDataTransferException(
-                    "Either hpc_password or hpc_secret_key must be provided"
-                )
+                raise HidDataTransferException(HPC_ACCOUNT_MSG)
 
             return self.__nifi_client.run_process_group(
                 "hpc2hdfs",
@@ -315,11 +320,11 @@ class HIDDataTransfer:
         )
         try:
             arguments: Dict[str, Any] = {}
-            arguments["hdfs.data_folder"] = \
+            arguments[HDFS_DATA_FOLDER] = \
                 "/".join(args.data_source.split("/")[:-1])
-            arguments["hdfs.filename"] = \
+            arguments[HDFS_FILENAME] = \
                 self.process_wildcards(args.data_source.split("/")[-1])
-            arguments["kerberos.principal"] = args.kerberos_principal
+            arguments[KERBEROS_PRINCIPAL] = args.kerberos_principal
             self.__nifi_client.add_default_ckan_parameters(arguments, args)
             arguments[self.__nifi_client.CKAN_RESOURCE] = \
                 self.generate_ckan_resource_name(args)
@@ -371,7 +376,6 @@ class HIDDataTransfer:
     ) -> str:
         """
         transfer data from CKAN to HDFS
-
         Args:
             ckan_host (str): The CKAN host URL.
             ckan_api_key (str): The CKAN API key.
@@ -409,9 +413,8 @@ class HIDDataTransfer:
             self.__nifi_client.add_default_ckan_parameters(arguments, args)
             arguments[self.__nifi_client.CKAN_RESOURCE] = \
                 self.process_wildcards(args.ckan_resource)
-
-            arguments["hdfs.data_folder"] = args.data_target
-            arguments["kerberos.principal"] = args.kerberos_principal
+            arguments[HDFS_DATA_FOLDER] = args.data_target
+            arguments[KERBEROS_PRINCIPAL] = args.kerberos_principal
 
             execution_type = {
                 "CKANFileDownloader": ExecutionType.ONCE,
@@ -506,7 +509,7 @@ class HIDDataTransfer:
         try:
             arguments: Dict[str, Any] = {}
             self.__nifi_client.add_default_hpc_parameters(arguments, args)
-            arguments["hpc.target_folder"] = args.data_target
+            arguments[HPC_TARGET_FOLDER] = args.data_target
             self.__nifi_client.add_default_ckan_parameters(arguments, args)
             arguments[self.__nifi_client.CKAN_RESOURCE] = \
                 self.process_wildcards(args.ckan_resource)
@@ -531,9 +534,7 @@ class HIDDataTransfer:
             )
 
             if not args.hpc_secret_key and not args.hpc_password:
-                raise HidDataTransferException(
-                    "Either hpc_password or hpc_secret_key must be provided"
-                )
+                raise HidDataTransferException(HPC_ACCOUNT_MSG)
 
             execution_type = {
                 "CKANFileDownloader": ExecutionType.ONCE,
@@ -616,8 +617,8 @@ class HIDDataTransfer:
             hpc_filename = args.data_source[args.data_source.rfind("/") + 1 :]
             arguments: Dict[str, Any] = {}
             self.__nifi_client.add_default_hpc_parameters(arguments, args)
-            arguments["hpc.filename"] = hpc_filename if hpc_filename else ".*"
-            arguments["hpc.data_folder"] = args.data_source[
+            arguments[HPC_FILENAME] = hpc_filename if hpc_filename else ".*"
+            arguments[HPC_DATA_FOLDER] = args.data_source[
                 : args.data_source.rfind("/")
             ]
             self.__nifi_client.add_default_ckan_parameters(arguments, args)
@@ -645,9 +646,7 @@ class HIDDataTransfer:
             )
 
             if not args.hpc_secret_key and not args.hpc_password:
-                raise HidDataTransferException(
-                    "Either hpc_password or hpc_secret_key must be provided"
-                )
+                raise HidDataTransferException(HPC_ACCOUNT_MSG)
 
             execution_type = {
                 "ListSFTP": ExecutionType.ONCE,
@@ -915,18 +914,14 @@ class HIDDataTransfer:
         process_group_id = args.command_id
 
         try:
-            # Get access token, do not forget to cancel
-            # the token refresh when done with it
-            token = self.__nifi_client.get_access_token()
-
             # Get all processors in the process group
             processors = self.__nifi_client.get_process_group_processors(
-                process_group_id, token
+                process_group_id
             )["processors"]
 
             # Order processor by pipeline sequence
             processors_sequence = self.__nifi_client.get_processors_sequence(
-                process_group_id, token
+                process_group_id
             )
             processors = self.__nifi_client.order_processors(
                 processors, processors_sequence
@@ -938,13 +933,14 @@ class HIDDataTransfer:
             if last_processor:
                 state = \
                     self.__nifi_client.nifi_rest_client.get_processor_state(
-                        last_processor, token
+                        last_processor, self.__nifi_client.get_access_token()
                     )
                 # Get if there remains pending incoming flowfiles
                 # to be processed
                 are_all_flowfiles_consumed = (
-                    self.__nifi_client.are_all_processor_queued_flowfiles_consumed(
-                        last_processor, process_group_id, token
+                    self.__nifi_client
+                    .are_all_processor_queued_flowfiles_consumed(
+                        last_processor, process_group_id
                     )
                 )
                 if state == "STOPPED" and are_all_flowfiles_consumed:
@@ -958,7 +954,7 @@ class HIDDataTransfer:
                 # in the process.
                 # Get processors on group and display bulletins reports
                 processors = self.__nifi_client.get_process_group_processors(
-                    process_group_id, token
+                    process_group_id
                 )
                 for processor in processors["processors"]:
                     bulletins = processor["bulletins"]

{hid_data_transfer_lib-0.2.0 → hid_data_transfer_lib-0.2.2}/hid_data_transfer_lib/keycloak/keycloak_rest.py

@@ -74,6 +74,11 @@ class KeycloakRESTClient:
         )
         self.__logger = self.__conf.logger("keycloak.keycloak_api")
 
+    def info(self, *args):
+        ''' do info logging'''
+        if self.__conf.is_logger_valid(self.__logger):
+            self.__logger.info(args)
+
     def __del__(self):
         """Destructor method"""
         self.cancel_token(do_logging=False)
@@ -81,7 +86,7 @@ class KeycloakRESTClient:
     def get_token(self):
         """Get a token that is renewed every time it expires"""
         if self.__token is None:
-            self.__logger.info(
+            self.info(
                 "Getting Keycloak token for user %s", self.__keycloak_login
             )
             self.__expires_in = self.__get_token()
@@ -97,7 +102,7 @@ class KeycloakRESTClient:
         self.__token = token
         self.__refresh_token = refresh_token
         self.__expires_in = expires_in
-        self.__logger.info(
+        self.info(
             "Setting Keycloak token for user %s with given one", user
         )
 
@@ -110,7 +115,7 @@ class KeycloakRESTClient:
     def cancel_token(self, do_logging=True):
         """Cancel the token refresh timer"""
         if do_logging:
-            self.__logger.info(
+            self.info(
                 "Cancelling the token refresh timer for user %s",
                 self.__keycloak_login,
             )
@@ -135,7 +140,7 @@ class KeycloakRESTClient:
         """Refresh the token with the expiration period
         given by the Keycloak server"""
         if self.__refresh:
-            self.__logger.info(
+            self.info(
                 "Refreshing Keycloak token for user %s", self.__keycloak_login
             )
             self.__get_refreshed_token()

{hid_data_transfer_lib-0.2.0 → hid_data_transfer_lib-0.2.2}/hid_data_transfer_lib/nifi/nifi_client.py

@@ -117,26 +117,27 @@ class NIFIClient:
 
     # GET requests
 
-    def get_process_group(self, group_id, token) -> dict:
+    def get_process_group(self, group_id) -> dict:
         """gets a process group identified by id"""
-        return self.nifi_rest_client.process_group_get_request(group_id, token)
+        return self.nifi_rest_client.process_group_get_request(
+            group_id, self.token())
 
-    def get_process_group_processors(self, group_id, token) -> dict:
+    def get_process_group_processors(self, group_id) -> dict:
         """gets the processors of a process group identified by id"""
         return self.nifi_rest_client.process_group_get_request(
-            group_id, token, "processors"
+            group_id, self.token(), "processors"
         )
 
-    def get_process_group_connections(self, group_id, token) -> dict:
+    def get_process_group_connections(self, group_id) -> dict:
         """gets the connetions of a process group identified by id"""
         return self.nifi_rest_client.process_group_get_request(
-            group_id, token, "connections"
+            group_id, self.token(), "connections"
         )
 
-    def get_processors_sequence(self, group_id, token) -> list:
+    def get_processors_sequence(self, group_id) -> list:
         """computes the sequence of processors of a process group
         identified by id"""
-        connections = self.get_process_group_connections(group_id, token)
+        connections = self.get_process_group_connections(group_id)
         connections_pairs = []
         for connection in connections["connections"]:
             connections_pairs.append(
@@ -144,10 +145,11 @@ class NIFIClient:
             )
         return order_process_group(connections_pairs)
 
-    def get_processor_error_bulletins(self, processor_id, token):
+    def get_processor_error_bulletins(self, processor_id):
         """Get the error bulletins of a processor identified by id"""
         # Retrieve current version of processor
-        processor = self.nifi_rest_client.get_processor(processor_id, token)
+        processor = self.nifi_rest_client.get_processor(
+            processor_id, self.token())
         error_bulletins = []
         bulletins = processor["bulletins"]
         for _bulletin in bulletins:
@@ -160,7 +162,7 @@ class NIFIClient:
 
     # PUT requests
 
-    def update_parameters(self, parameter_context, arguments, token) -> bool:
+    def update_parameters(self, parameter_context, arguments) -> bool:
         """updates the parameters of a parameter context
         arguments contains the parameters to inject in the context
         """
@@ -171,7 +173,8 @@ class NIFIClient:
         parameter_context["component"]["parameters"] = parameters
         modified_parameter_context_json = json.dumps(parameter_context)
         return self.nifi_rest_client.save_parameter_context(
-            modified_parameter_context_json, parameter_context["id"], token
+            modified_parameter_context_json, parameter_context["id"],
+            self.token()
         )
 
     def update_processor_property(
@@ -185,14 +188,14 @@ class NIFIClient:
 
     # MULTI-REQUESTS methods
 
-    def run_processor(self, execution_type, token, processor):
+    def run_processor(self, execution_type, processor):
         """run a processor once or forever depending on the execution type"""
         ok, response = None, None
         processor_name = processor["component"]["name"]
         if execution_type[processor_name] == ExecutionType.ONCE:
-            ok, response = self._run_processor_once(processor, token)
+            ok, response = self._run_processor_once(processor)
         elif execution_type[processor_name] == ExecutionType.FOREVER:
-            ok, response = self._run_processor_forever(processor, token)
+            ok, response = self._run_processor_forever(processor)
         else:
             raise HidDataTransferException(
                 f"Incompatible execution type {execution_type[processor_name]}"
@@ -201,31 +204,31 @@ class NIFIClient:
 
         return ok, response
 
-    def _run_processor_once(self, processor, token) -> tuple[bool, str]:
+    def _run_processor_once(self, processor) -> tuple[bool, str]:
         """run once a given processor"""
         return self.nifi_rest_client.change_processor_run_status(
-            processor, token, "RUN_ONCE"
+            processor, self.token(), "RUN_ONCE"
         )
 
-    def _run_processor_forever(self, processor, token) -> tuple[bool, str]:
+    def _run_processor_forever(self, processor) -> tuple[bool, str]:
         """run a given processor until it is stopped"""
         return self.nifi_rest_client.change_processor_run_status(
-            processor, token, "RUNNING"
+            processor, self.token(), "RUNNING"
         )
 
-    def stop_processor(self, processor, token) -> tuple[bool, str]:
+    def stop_processor(self, processor) -> tuple[bool, str]:
         """stop a given processor"""
         return self.nifi_rest_client.change_processor_run_status(
-            processor, token, "STOPPED"
+            processor, self.token(), "STOPPED"
         )
 
     def are_all_processor_queued_flowfiles_consumed(
-        self, processor, process_group_id, token
+        self, processor, process_group_id
     ) -> bool:
         """Check if all queued flowfiles have been consumed by the processor"""
         # Get the process group connections
         connections = self.get_process_group_connections(
-            process_group_id, token)[
+            process_group_id)[
             "connections"
         ]
         for connection in connections:
@@ -245,13 +248,12 @@ class NIFIClient:
         processor,
         process_group_id,
         expected_outgoing_flowfiles,
-        token,
     ) -> bool:
         """Check if all expected outgoing flowfiles
         have been generated by the processor"""
         # Get the process group connections
         connections = self.get_process_group_connections(
-            process_group_id, token)[
+            process_group_id)[
             "connections"
         ]
         for connection in connections:
@@ -262,20 +264,19 @@ class NIFIClient:
                     ] == expected_outgoing_flowfiles
                 )
         raise HidDataTransferException(
-            f"Incomming connection not found for processor {processor['id']}"
+            f"Outgoing connection not found for processor {processor['id']}"
         )
 
     def has_created_outgoing_flowfiles(
         self,
         processor,
         process_group_id,
-        token,
     ) -> bool:
         """Check if at least one outgoing flowfile
         has been generated by the processor"""
         # Get the process group connections
         connections = self.get_process_group_connections(
-            process_group_id, token)[
+            process_group_id)[
             "connections"
         ]
         for connection in connections:
@@ -284,7 +285,7 @@ class NIFIClient:
                     "flowFilesQueued"
                 ] > 0
         raise HidDataTransferException(
-            f"Incomming connection not found for processor {processor['id']}"
+            f"Outgoing connection not found for processor {processor['id']}"
         )
 
     # Class helper methods
@@ -325,20 +326,20 @@ class NIFIClient:
                 return processor
         return None
 
-    def find_flow_definition_by_name(self, flow_definition_name, token):
+    def find_flow_definition_by_name(self, flow_definition_name):
         """Find a flow definition, registered in the NIFI Registry, by name"""
         # Assuming there is only one registered Registry
-        registries = self.nifi_rest_client.get_registries(token)
+        registries = self.nifi_rest_client.get_registries(self.token())
         assert registries is not None
         registry_id = registries["registries"][0]["id"]
 
         # Assuming there is only one registered Bucket
-        buckets = self.nifi_rest_client.get_buckets(registry_id, token)
+        buckets = self.nifi_rest_client.get_buckets(registry_id, self.token())
         assert buckets is not None
         bucket_id = buckets["buckets"][0]["id"]
 
         flow_definitions = self.nifi_rest_client.get_flow_definitions(
-            registry_id, bucket_id, token
+            registry_id, bucket_id, self.token()
         )
         assert flow_definitions is not None
         flow_definition_id = None
@@ -356,7 +357,7 @@ class NIFIClient:
             )
 
         versions = self.nifi_rest_client.get_flow_definition_versions(
-            registry_id, bucket_id, flow_definition_id, token
+            registry_id, bucket_id, flow_definition_id, self.token()
         )
         assert versions is not None
         # Take the latest verstion of the flow
@@ -372,24 +373,25 @@ class NIFIClient:
         }
 
     def instantiate_process_group(
-        self, token, flow_definition_name, arguments, keys_arguments
+        self, flow_definition_name, arguments, keys_arguments
     ):
         """create a process group from a template
         and its associated parameter context"""
         try:
             # Get flow definition metadata by name
             flow_definition = self.find_flow_definition_by_name(
-                flow_definition_name, token
+                flow_definition_name
             )
 
             # Instantiate flow definition as process group
             process_group = (
-                self.nifi_rest_client.instantiate_flow_definition_as_process_group(
+                self.nifi_rest_client.
+                instantiate_flow_definition_as_process_group(
                     flow_definition["registry_id"],
                     flow_definition["bucket_id"],
                     flow_definition["flow_definition_id"],
                     flow_definition["version"],
-                    token,
+                    self.token(),
                 )
             )
 
@@ -404,11 +406,11 @@ class NIFIClient:
             # Instantiate parameter context with parameters
             parameter_context = \
                 self.nifi_rest_client.instantiate_parameter_context(
-                    token
+                    self.token()
                 )
 
             # Update parameters
-            ok = self.update_parameters(parameter_context, arguments, token)
+            ok = self.update_parameters(parameter_context, arguments)
 
             if not ok:
                 raise HidDataTransferException(
@@ -418,7 +420,7 @@ class NIFIClient:
 
             # Associate parameter context to process group
             self.nifi_rest_client.associate_parameter_context_to_group(
-                parameter_context, process_group, token
+                parameter_context, process_group, self.token()
             )
 
             return process_group
@@ -427,6 +429,10 @@ class NIFIClient:
             dtex.set_produced_object(process_group)
             raise dtex from ex
 
+    def token(self):
+        ''' Gets latest updated NIFI access token '''
+        return self.get_access_token()
+
     def run_process_group(
         self,
         template_name,
@@ -483,29 +489,24 @@ class NIFIClient:
             # Get access token, do not forget to cancel token refresh
             # when done with it
             if keycloak_token:
-                token = self.set_access_token(keycloak_token)
-            else:
-                token = self.get_access_token()
+                self.set_access_token(keycloak_token)
 
             process_group = self.instantiate_process_group(
-                token, template_name, arguments, keys_arguments
+                template_name, arguments, keys_arguments
             )
             self.__logger.info(
                 "Process group %s created\n", process_group["id"])
 
             # Get all processors in the process group
             processors = self.get_process_group_processors(
-                process_group["id"], token)[
-                    "processors"
-                ]
+                process_group["id"])["processors"]
 
             # For each processor in group ordered by the sequence,
             # execute processor once
             # Check previous processor has an running instance
             # before launching next one
             processors_sequence = self.get_processors_sequence(
-                process_group["id"], token
-            )
+                process_group["id"])
             processors = self.order_processors(processors, processors_sequence)
             index_processor = 0
 
@@ -517,19 +518,17 @@ class NIFIClient:
             for processor in processors:
                 # Set sensitive hpc parameters
                 processor = self.set_sensitive_parameters(
-                    sensitive_parameters, token, processor
+                    sensitive_parameters, processor
                 )
 
                 # Run the processor once or forever
                 # until all incoming flowfiles are consumed
                 processor_name = processor["component"]["name"]
-                ok, response = self.run_processor(
-                    execution_type, token, processor)
+                ok, response = self.run_processor(execution_type, processor)
                 index_processor += 1
                 if ok:
                     self.wait_for_processor_completion(
                         execution_type,
-                        token,
                         index_processor,
                         len(processors),
                         processor,
@@ -564,11 +563,11 @@ class NIFIClient:
             self.cancel_access_token()
             if process_group:
                 # Clean up the process group and associated parameter context
-                self.clean_process_group_up(process_group["id"], token)
+                self.clean_process_group_up(process_group["id"])
                 self.__logger.info("Process group %s cleaned up\n",
                                    process_group["id"])
 
-    def set_sensitive_parameters(self, sensitive_parameters, token, processor):
+    def set_sensitive_parameters(self, sensitive_parameters, processor):
         """Set sensitive parameters for a processor"""
         processor_name = processor["component"]["name"]
         if sensitive_parameters and processor_name in sensitive_parameters:
@@ -579,13 +578,12 @@ class NIFIClient:
                     sensitive_parameters[processor_name][key],
                 )
             processor = self.nifi_rest_client.update_processor(
-                processor, token)
+                processor, self.token())
         return processor
 
     def wait_for_processor_completion(
         self,
         execution_type,
-        token,
         index_processor,
         num_processors,
         processor,
@@ -603,7 +601,7 @@ class NIFIClient:
             if index_processor != 1:
                 are_all_flowfiles_consumed = (
                     self.are_all_processor_queued_flowfiles_consumed(
-                        processor, process_group_id, token
+                        processor, process_group_id
                     )
                 )
             else:
@@ -618,18 +616,17 @@ class NIFIClient:
                             processor,
                             process_group_id,
                             first_processor_expected_outgoing_flowfiles,
-                            token,
                         )
                     )
             status = self.nifi_rest_client.get_processor_state(
-                processor, token)
+                processor, self.token())
             # If processor is stopped and not the last one,
             # check it has produced at least one flowfile
             if (
                 status == "STOPPED"
                 and index_processor != num_processors
                 and not self.has_created_outgoing_flowfiles(
-                    processor, process_group_id, token
+                    processor, process_group_id
                 )
             ):
                 raise HidDataTransferException(
@@ -639,44 +636,45 @@ class NIFIClient:
             # Check if processor execution has failed with exception
             # Collect them all and raise a single exception
             error_bulletins = self.get_processor_error_bulletins(
-                processor["id"], token)
+                processor["id"])
             if error_bulletins:
                 # Before raising an exception, stop the processor
                 # if it is running for ever
                 self.stop_forever_running_processor(
-                    execution_type[processor_name], processor, token)
+                    execution_type[processor_name], processor)
                 # and empty the incoming queue
-                self.empty_incoming_queue(processor, process_group_id, token)
+                self.empty_incoming_queue(processor, process_group_id)
                 raise HidDataTransferException(
                     f"Processor {processor_name} has failed with exception: "
                     f"{error_bulletins}"
                 )
             sleep(self.__CHECK_STATUS_SLEEP_LAPSE)
-            self.stop_forever_running_processor(
-                execution_type[processor_name], processor, token)
+        self.stop_forever_running_processor(
+            execution_type[processor_name], processor)
 
-    def stop_forever_running_processor(self, exec_type, processor, token):
+    def stop_forever_running_processor(self, exec_type, processor):
         '''Stop a processor that is running forever'''
         # If processor has executed for ever, stop it
         if exec_type == ExecutionType.FOREVER:
             # Get current processor version
             processor = self.nifi_rest_client.get_processor(
-                processor["id"], token)
-            self.stop_processor(processor, token)
+                processor["id"], self.token())
+            self.stop_processor(processor)
 
-    def empty_incoming_queue(self, processor, process_group_id, token):
+    def empty_incoming_queue(self, processor, process_group_id):
         '''Empty the incoming queue of a processor'''
         if not self.are_all_processor_queued_flowfiles_consumed(
-                processor, process_group_id, token):
+                processor, process_group_id):
             # Get the process group connections
             connections = self.get_process_group_connections(
-                process_group_id, token)["connections"]
+                process_group_id)["connections"]
             for connection in connections:
                 if (connection["component"]["destination"]["id"]
                     == processor["id"]
                     and connection["status"]["aggregateSnapshot"]
                         ["flowFilesQueued"] != 0):  # Empty the queue
-                    self.nifi_rest_client.empty_queue(connection['id'], token)
+                    self.nifi_rest_client.empty_queue(
+                        connection['id'], self.token())
 
     def add_default_hpc_parameters(self, arguments, args):
         """adds default parameters for HPC connection"""
@@ -737,10 +735,10 @@ class NIFIClient:
                 )
         return sensitive_parameters
 
-    def clean_process_group_up(self, process_group_id, token):
+    def clean_process_group_up(self, process_group_id):
         """Clean up the process group and associated parameter context"""
         # Get process group
-        process_group = self.get_process_group(process_group_id, token)
+        process_group = self.get_process_group(process_group_id)
 
         # Remove transferred keys
         self.__logger.debug("Removing uploaded keys\n")
@@ -753,15 +751,15 @@ class NIFIClient:
                 "parameterContext"
             ]["id"]
             parameter_context = self.nifi_rest_client.get_parameter_context(
-                parameter_context_id, token
+                parameter_context_id, self.token()
             )
 
             # Remove parameter_context
             self.nifi_rest_client.remove_parameter_context(
-                parameter_context, token)
+                parameter_context, self.token())
 
         # Remove process_group
-        self.nifi_rest_client.remove_process_group(process_group, token)
+        self.nifi_rest_client.remove_process_group(process_group, self.token())
 
     def format_args_to_string(self, args):
         """Format dtcli command arguments to string for logging"""

{hid_data_transfer_lib-0.2.0 → hid_data_transfer_lib-0.2.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "hid_data_transfer_lib"
-version = "0.2.0"
+version = "0.2.2"
 description = "HiDALGO Data Transfer library provides methods to transfer data between different data providers and consumers using NIFI pipelines"
 authors = ["Jesús Gorroñogoitia <jesus.gorronogoitia@eviden.com>"]
 license = "APL-2.0"