hid_data_transfer_lib 0.3.9__tar.gz → 0.3.11__tar.gz

{hid_data_transfer_lib-0.3.9 → hid_data_transfer_lib-0.3.11}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: hid_data_transfer_lib
-Version: 0.3.9
+Version: 0.3.11
 Summary: HiDALGO Data Transfer library provides methods to transfer data between different data providers and consumers using NIFI pipelines
 License: APL-2.0
 Author: Jesús Gorroñogoitia
@@ -11,6 +11,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Requires-Dist: paramiko (>=3.3.1)
 Requires-Dist: requests (>=2.31.0)
 Requires-Dist: setuptools (>=79.0.0,<80.0.0)
@@ -50,7 +51,7 @@ The Data Transfer Library sends requests to NIFI through its REST API.
 To use the Data Transfer library, it is required the following requirements:
  - **Python3** execution environment
  - **Poetry** python package management tool (optional)
- - **NIFI** instance, with a NIFI server SSH account (for keys transfer)
+ - **NIFI** instance, with a NIFI 2.X server SSH account (for keys transfer). Latest NIFI supported version: 2.8
  - **Keycloak** instance, with a KEYCLOAK user's account
  - **HDFS** instance, with an user's Kerberos principal account
  - **CKAN** instance, with an user APIKey

{hid_data_transfer_lib-0.3.9 → hid_data_transfer_lib-0.3.11}/README.md

@@ -32,7 +32,7 @@ The Data Transfer Library sends requests to NIFI through its REST API.
 To use the Data Transfer library, it is required the following requirements:
  - **Python3** execution environment
  - **Poetry** python package management tool (optional)
- - **NIFI** instance, with a NIFI server SSH account (for keys transfer)
+ - **NIFI** instance, with a NIFI 2.X server SSH account (for keys transfer). Latest NIFI supported version: 2.8
  - **Keycloak** instance, with a KEYCLOAK user's account
  - **HDFS** instance, with an user's Kerberos principal account
  - **CKAN** instance, with an user APIKey

{hid_data_transfer_lib-0.3.9 → hid_data_transfer_lib-0.3.11}/hid_data_transfer_lib/hid_dt_lib.py

@@ -34,7 +34,6 @@ from hid_data_transfer_lib.nifi.nifi_client import (
     NIFIClient, ExecutionType, AccountingInfo
 )
 
-KERBEROS_PRINCIPAL = "kerberos.principal"
 HDFS_DATA_FOLDER = "hdfs.data_folder"
 HDFS_FILENAME = "hdfs.filename"
 HPC_TARGET_FOLDER = "hpc.target_folder"
@@ -98,7 +97,7 @@ class HIDDataTransfer:
                     else ckan_resource
         else:
             ckan_resource = (
-                    args.data_source.replace("/", "_")
+                    args.data_source.replace("/", "_")[1:]
                     if not args.data_source.endswith("/")
                     else args.data_source[:-1].replace("/", "_")
                 )
@@ -158,9 +157,13 @@ class HIDDataTransfer:
             hpc_secret_key=hpc_secret_key_path,
             data_source=data_source,
             data_target=data_target,
+            recursive=recursive,
+        )
+
+        processors_with_kerberos = ["ListHDFS", "FetchHDFS"]
+        kerberos_account = SimpleNamespace(
             kerberos_principal=kerberos_principal,
             kerberos_password=kerberos_password,
-            recursive=recursive,
         )
 
         self.__logger.info(
@@ -175,7 +178,7 @@ class HIDDataTransfer:
                 "/".join(args.data_source.split("/")[:-1])
             arguments[HDFS_FILENAME] = \
                 self.process_wildcards(args.data_source.split("/")[-1])
-            arguments[KERBEROS_PRINCIPAL] = args.kerberos_principal
+
             arguments[RECURSIVE] = args.recursive
 
             execution_type = {
@@ -197,14 +200,6 @@ class HIDDataTransfer:
                     args
                 )
 
-            processors_with_hdfs_sensitive_parameters = \
-                ["ListHDFS", "FetchHDFS"]
-            sensitive_parameters.update(
-                self.__nifi_client.set_sensitive_hdfs_parameters(
-                    processors_with_hdfs_sensitive_parameters, args
-                )
-            )
-
             if not args.hpc_secret_key and not args.hpc_password:
                 raise HidDataTransferException(HPC_ACCOUNT_MSG)
 
@@ -217,7 +212,7 @@ class HIDDataTransfer:
                 "FetchHDFS": concurrent_tasks,
             }
 
-            return self.__nifi_client.run_process_group(
+            return self.__nifi_client.run_process_group_opt(
                 "hdfs2hpc",
                 execution_type,
                 arguments,
@@ -226,6 +221,9 @@ class HIDDataTransfer:
                 keycloak_token=self.__keycloak_token,
                 accounting_target="PutSFTP",
                 concurrent_tasks=concurrent_tasks_dict,
+                ignore_bulletins_in_processors=["PutSFTP"],
+                processors_with_kerberos=processors_with_kerberos,
+                kerberos_account=kerberos_account,
             )
 
         except Exception as ex:
@@ -386,6 +384,12 @@ class HIDDataTransfer:
             kerberos_password=kerberos_password,
         )
 
+        processors_with_kerberos = ["ListHDFS", "FetchHDFS"]
+        kerberos_account = SimpleNamespace(
+            kerberos_principal=kerberos_principal,
+            kerberos_password=kerberos_password,
+        )
+
         self.__logger.info(
             "executing hdfs2hpc_2fa command with args: %s",
             self.__nifi_client.format_args_to_string(args),
@@ -398,7 +402,6 @@ class HIDDataTransfer:
                 "/".join(args.data_source.split("/")[:-1])
             arguments[HDFS_FILENAME] = \
                 self.process_wildcards(args.data_source.split("/")[-1])
-            arguments[KERBEROS_PRINCIPAL] = args.kerberos_principal
 
             # set local.data_folder to keep dataset retrieved from HDFS
             __uuid = uuid.uuid4()
@@ -421,16 +424,6 @@ class HIDDataTransfer:
                 ],
             }
 
-            sensitive_parameters = {}
-
-            processors_with_hdfs_sensitive_parameters = \
-                ["ListHDFS", "FetchHDFS"]
-            sensitive_parameters.update(
-                self.__nifi_client.set_sensitive_hdfs_parameters(
-                    processors_with_hdfs_sensitive_parameters, args
-                )
-            )
-
             if not args.hpc_secret_key:
                 raise HidDataTransferException(HPC_ACCOUNT_MSG)
 
@@ -449,10 +442,12 @@ class HIDDataTransfer:
                 execution_type,
                 arguments,
                 keys_arguments=keys_arguments,
-                sensitive_parameters=sensitive_parameters,
+                sensitive_parameters=None,
                 keycloak_token=self.__keycloak_token,
                 accounting_target="PutFile",
                 concurrent_tasks=concurrent_tasks_dict,
+                processors_with_kerberos=processors_with_kerberos,
+                kerberos_account=kerberos_account,
             )
             process_ids.append(process_id)
 
@@ -461,7 +456,7 @@ class HIDDataTransfer:
                 execution_type,
                 arguments,
                 keys_arguments=keys_arguments,
-                sensitive_parameters=sensitive_parameters,
+                sensitive_parameters=None,
                 keycloak_token=self.__keycloak_token,
                 processor_with_2fa="SftpTransferToHPCWithVPNCode",
                 callback_2fa=callback_2fa,
@@ -535,6 +530,12 @@ class HIDDataTransfer:
             recursive=recursive,
         )
 
+        processors_with_kerberos = ["PutHDFS"]
+        kerberos_account = SimpleNamespace(
+            kerberos_principal=kerberos_principal,
+            kerberos_password=kerberos_password,
+        )
+
         self.__logger.info(
             "executing hpc2hdfs command with args: %s",
             self.__nifi_client.format_args_to_string(args),
@@ -548,7 +549,6 @@ class HIDDataTransfer:
                 : args.data_source.rfind("/")
             ]
             arguments[HDFS_DATA_FOLDER] = args.data_target
-            arguments[KERBEROS_PRINCIPAL] = args.kerberos_principal
             arguments[RECURSIVE] = args.recursive
 
             execution_type = {
@@ -571,14 +571,6 @@ class HIDDataTransfer:
                     processors_with_hpc_sensitive_parameters, args
                 )
 
-            processors_with_hdfs_sensitive_parameters = \
-                ["PutHDFS"]
-            sensitive_parameters.update(
-                self.__nifi_client.set_sensitive_hdfs_parameters(
-                    processors_with_hdfs_sensitive_parameters, args
-                )
-            )
-
             if not args.hpc_secret_key and not args.hpc_password:
                 raise HidDataTransferException(HPC_ACCOUNT_MSG)
 
@@ -591,7 +583,7 @@ class HIDDataTransfer:
                 "PutHDFS": concurrent_tasks,
             }
 
-            return self.__nifi_client.run_process_group(
+            return self.__nifi_client.run_process_group_opt(
                 "hpc2hdfs",
                 execution_type,
                 arguments,
@@ -599,7 +591,10 @@ class HIDDataTransfer:
                 sensitive_parameters,
                 keycloak_token=self.__keycloak_token,
                 accounting_target="PutHDFS",
-                concurrent_tasks=concurrent_tasks_dict
+                concurrent_tasks=concurrent_tasks_dict,
+                ignore_bulletins_in_processors=["FetchSFTP"],
+                processors_with_kerberos=processors_with_kerberos,
+                kerberos_account=kerberos_account,
             )
 
         except Exception as ex:
@@ -710,6 +705,12 @@ class HIDDataTransfer:
             kerberos_password=kerberos_password,
         )
 
+        processors_with_kerberos = ["PutHDFS"]
+        kerberos_account = SimpleNamespace(
+            kerberos_principal=kerberos_principal,
+            kerberos_password=kerberos_password,
+        )
+
         self.__logger.info(
             "executing hpc2hdfs_2fa command with args: %s",
             self.__nifi_client.format_args_to_string(args),
@@ -726,7 +727,6 @@ class HIDDataTransfer:
                 : args.data_source.rfind("/")
             ]
             arguments[HDFS_DATA_FOLDER] = args.data_target
-            arguments[KERBEROS_PRINCIPAL] = args.kerberos_principal
 
             # set local.data_folder to keep dataset retrieved from HPC
             __uuid = uuid.uuid4()
@@ -756,14 +756,6 @@ class HIDDataTransfer:
                     processors_with_hpc_sensitive_parameters, args
                 )
 
-            processors_with_hdfs_sensitive_parameters = \
-                ["PutHDFS"]
-            sensitive_parameters.update(
-                self.__nifi_client.set_sensitive_hdfs_parameters(
-                    processors_with_hdfs_sensitive_parameters, args
-                )
-            )
-
             if not args.hpc_secret_key and not args.hpc_password:
                 raise HidDataTransferException(HPC_ACCOUNT_MSG)
 
@@ -786,7 +778,6 @@ class HIDDataTransfer:
                 keycloak_token=self.__keycloak_token,
                 processor_with_2fa="SftpTransferFromHPCWithVPNCode",
                 callback_2fa=callback_2fa,
-                accounting_target="PutHDFS",
                 concurrent_tasks=concurrent_tasks_dict,
             )
             process_ids.append(process_id)
@@ -800,6 +791,8 @@ class HIDDataTransfer:
                 keycloak_token=self.__keycloak_token,
                 accounting_target="PutHDFS",
                 concurrent_tasks=concurrent_tasks_dict,
+                processors_with_kerberos=processors_with_kerberos,
+                kerberos_account=kerberos_account,
             )
             process_ids.append(process_id)
 
@@ -911,6 +904,12 @@ class HIDDataTransfer:
             recursive=recursive,
         )
 
+        processors_with_kerberos = ["ListHDFS", "FetchHDFS"]
+        kerberos_account = SimpleNamespace(
+            kerberos_principal=kerberos_principal,
+            kerberos_password=kerberos_password,
+        )
+
         self.__logger.info(
             "executing hdfs2ckan command with args: %s",
             self.__nifi_client.format_args_to_string(args),
@@ -921,7 +920,6 @@ class HIDDataTransfer:
                 "/".join(args.data_source.split("/")[:-1])
             arguments[HDFS_FILENAME] = \
                 self.process_wildcards(args.data_source.split("/")[-1])
-            arguments[KERBEROS_PRINCIPAL] = args.kerberos_principal
             self.__nifi_client.add_default_ckan_parameters(arguments, args)
             arguments[self.__nifi_client.CKAN_RESOURCE] = \
                 self.generate_ckan_resource_name(args)
@@ -941,14 +939,6 @@ class HIDDataTransfer:
                     processors_with_ckan_sensitive_parameters, args
                 )
 
-            processors_with_hdfs_sensitive_parameters = \
-                ["ListHDFS", "FetchHDFS"]
-            sensitive_parameters.update(
-                self.__nifi_client.set_sensitive_hdfs_parameters(
-                    processors_with_hdfs_sensitive_parameters, args
-                )
-            )
-
             # Set the number of concurrent tasks
             # for parallel processors in the pipeline
             if concurrent_tasks is None:
@@ -967,15 +957,21 @@ class HIDDataTransfer:
                 "CKANFileUploader": concurrent_tasks,
             }
 
-            return self.__nifi_client.run_process_group(
-                "hdfs2ckan",
-                execution_type,
-                arguments,
-                None,
-                sensitive_parameters,
+            # disabled optimized version (run process group step by step)
+            # return self.__nifi_client.run_process_group(
+            # enabled optimized version (run process group at once)
+            return self.__nifi_client.run_process_group_opt(
+                template_name="hdfs2ckan",
+                execution_type=execution_type,
+                arguments=arguments,
+                keys_arguments=None,
+                sensitive_parameters=sensitive_parameters,
                 keycloak_token=self.__keycloak_token,
                 accounting_target="MergeContent",
-                concurrent_tasks=concurrent_tasks_dict
+                concurrent_tasks=concurrent_tasks_dict,
+                synchronization_processor_name="MergeContent",
+                processors_with_kerberos=processors_with_kerberos,
+                kerberos_account=kerberos_account,
             )
 
         except Exception as ex:
@@ -1027,6 +1023,12 @@ class HIDDataTransfer:
             kerberos_password=kerberos_password,
         )
 
+        processors_with_kerberos = ["PutHDFS"]
+        kerberos_account = SimpleNamespace(
+            kerberos_principal=kerberos_principal,
+            kerberos_password=kerberos_password,
+        )
+
         self.__logger.info(
             "executing ckan2hdfs command with args: %s",
             self.__nifi_client.format_args_to_string(args),
@@ -1037,7 +1039,6 @@ class HIDDataTransfer:
             arguments[self.__nifi_client.CKAN_RESOURCE] = \
                 self.process_wildcards(args.ckan_resource)
             arguments[HDFS_DATA_FOLDER] = args.data_target
-            arguments[KERBEROS_PRINCIPAL] = args.kerberos_principal
 
             execution_type = {
                 "CKANFileDownloader": ExecutionType.ONCE,
@@ -1050,14 +1051,6 @@ class HIDDataTransfer:
                     processors_with_ckan_sensitive_parameters, args
                 )
 
-            processors_with_hdfs_sensitive_parameters = \
-                ["PutHDFS"]
-            sensitive_parameters.update(
-                self.__nifi_client.set_sensitive_hdfs_parameters(
-                    processors_with_hdfs_sensitive_parameters, args
-                )
-            )
-
             # Set the number of concurrent tasks
             # for parallel processors in the pipeline
             if concurrent_tasks is None:
@@ -1078,7 +1071,10 @@ class HIDDataTransfer:
             arguments[self.__nifi_client.CKAN_CONCURRENT_TASKS] \
                 = concurrent_tasks
 
-            return self.__nifi_client.run_process_group(
+            # disabled optimized version (run process group step by step)
+            # return self.__nifi_client.run_process_group(
+            # enabled optimized version (run process group at once)
+            return self.__nifi_client.run_process_group_opt(
                 "ckan2hdfs",
                 execution_type,
                 arguments,
@@ -1086,7 +1082,9 @@ class HIDDataTransfer:
                 sensitive_parameters,
                 keycloak_token=self.__keycloak_token,
                 accounting_target="PutHDFS",
-                concurrent_tasks=concurrent_tasks_dict
+                concurrent_tasks=concurrent_tasks_dict,
+                processors_with_kerberos=processors_with_kerberos,
+                kerberos_account=kerberos_account,
             )
 
         except Exception as ex:
@@ -1202,7 +1200,7 @@ class HIDDataTransfer:
             arguments[self.__nifi_client.CKAN_CONCURRENT_TASKS] \
                 = concurrent_tasks
 
-            return self.__nifi_client.run_process_group(
+            return self.__nifi_client.run_process_group_opt(
                 "ckan2hpc",
                 execution_type,
                 arguments,
@@ -1210,7 +1208,8 @@ class HIDDataTransfer:
                 sensitive_parameters,
                 keycloak_token=self.__keycloak_token,
                 accounting_target="PutSFTP",
-                concurrent_tasks=concurrent_tasks_dict
+                concurrent_tasks=concurrent_tasks_dict,
+                ignore_bulletins_in_processors=["PutSFTP"],
             )
 
         except Exception as ex:
@@ -1613,7 +1612,7 @@ class HIDDataTransfer:
                 "CKANFileUploader": concurrent_tasks,
             }
 
-            return self.__nifi_client.run_process_group(
+            return self.__nifi_client.run_process_group_opt(
                 "hpc2ckanv2",
                 execution_type,
                 arguments,
@@ -1621,7 +1620,9 @@ class HIDDataTransfer:
                 sensitive_parameters,
                 keycloak_token=self.__keycloak_token,
                 accounting_target="MergeContent",
-                concurrent_tasks=concurrent_tasks_dict
+                concurrent_tasks=concurrent_tasks_dict,
+                synchronization_processor_name="MergeContent",
+                ignore_bulletins_in_processors=["FetchSFTP"],
             )
 
         except Exception as ex: