hid_data_transfer_lib 0.4.0__tar.gz → 0.4.1__tar.gz

{hid_data_transfer_lib-0.4.0 → hid_data_transfer_lib-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hid_data_transfer_lib
-Version: 0.4.0
+Version: 0.4.1
 Summary: HiDALGO Data Transfer library provides methods to transfer data between different data providers and consumers using NIFI pipelines
 License: APL-2.0
 Author: Jesús Gorroñogoitia

{hid_data_transfer_lib-0.4.0 → hid_data_transfer_lib-0.4.1}/hid_data_transfer_lib/hid_dt_lib.py

@@ -2140,14 +2140,15 @@ class HIDDataTransfer:
                 first_processor_expected_outgoing_flowfiles=num_files,
                 keycloak_token=self.__keycloak_token,
                 accounting_target="MergeContent",
+                upload_folder=upload_folder
             )
 
         except Exception as ex:
             raise HidDataTransferException(ex) from ex
-        finally:
+        # finally:
             # Remove uploaded files
-            if upload_folder is not None:
-                self.__nifi_client.remove_upload_folder(upload_folder)
+            # if upload_folder is not None:
+            #     self.__nifi_client.remove_upload_folder(upload_folder)
 
     def ckan2local(
         self,

{hid_data_transfer_lib-0.4.0 → hid_data_transfer_lib-0.4.1}/hid_data_transfer_lib/nifi/nifi_client.py

@@ -916,6 +916,7 @@ class NIFIClient:
         ignore_bulletins_in_processors=None,
         processors_with_kerberos=None,
         kerberos_account=None,
+        upload_folder=None,
     ) -> tuple[str, AccountingInfo]:
         """MAIN method to run a processor from a template identify by name
         The procedure is as follows:
@@ -1113,7 +1114,7 @@ class NIFIClient:
                 # Refresh NIFI token before cleanup
                 self.refresh_access_token()
                 # Clean up the process group and associated parameter context
-                self.clean_process_group_up(process_group["id"])
+                self.clean_process_group_up(process_group["id"], upload_folder=upload_folder)
                 self.__logger.info("Process group %s cleaned up\n",
                                    process_group["id"])
             self.cancel_access_token()
@@ -1680,6 +1681,10 @@ class NIFIClient:
                 target_processor['status']['aggregateSnapshot']['flowFilesIn']
         }
 
+        msg = "Processor {accounting_target} has reported flowFilesIn: " \
+            f"{target_processor['status']['aggregateSnapshot']['flowFilesIn']}\n"
+        self.__logger.debug(msg)
+
         return flowfiles_sizes
 
     def find_incoming_connection_for_processor(self,
@@ -1790,7 +1795,7 @@ class NIFIClient:
             # Save changes in processor
             self.set_processor_kus(processor, kpus['id'])
 
-    def clean_process_group_up(self, process_group_id, retry=True):
+    def clean_process_group_up(self, process_group_id, upload_folder=None, retry=True):
         """Clean up the process group and associated parameter context"""
         # Get process group
         process_group = self.get_process_group(process_group_id)
@@ -1800,6 +1805,10 @@ class NIFIClient:
             self.__logger.debug("Removing uploaded keys\n")
             self.remove_uploaded_keys(process_group_id)
 
+            # Remove upload folder
+            if upload_folder is not None:
+                self.remove_upload_folder(upload_folder)
+
             # Stop process_group and wait for all processosr to be stopped
             self.stop_process_group(process_group_id)
 

{hid_data_transfer_lib-0.4.0 → hid_data_transfer_lib-0.4.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "hid_data_transfer_lib"
-version = "0.4.0"
+version = "0.4.1"
 description = "HiDALGO Data Transfer library provides methods to transfer data between different data providers and consumers using NIFI pipelines"
 authors = [
     { name = "Jesús Gorroñogoitia", email = "jesus.gorronogoitia@eviden.com" },
@@ -41,6 +41,7 @@ pytest = "^8.3.5"
 pytest-cov = "^6.1.0"
 pytest-env = "^1.1.5"
 numpy = "^2.3.4"
+python-dotenv = "^1.2.3"
 
 [build-system]
 requires = ["poetry-core"]