data-transfer-cli 0.3.6__tar.gz → 0.3.7__tar.gz

{data_transfer_cli-0.3.6 → data_transfer_cli-0.3.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: data-transfer-cli
-Version: 0.3.6
+Version: 0.3.7
 Summary: HiDALGO Data Transfer CLI provides commands to transfer data between different data providers and consumers using NIFI pipelines
 License: APL-2.0
 Author: Jesús Gorroñogoitia
@@ -11,7 +11,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
-Requires-Dist: hid_data_transfer_lib (>=0.3.4)
+Requires-Dist: hid_data_transfer_lib (>=0.3.7)
 Requires-Dist: paramiko (>=3.3.1)
 Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
 Requires-Dist: requests (>=2.31.0)
@@ -209,6 +209,9 @@ options:
                         [Optional] Password for HPC secret key
   -2fa, --two-factor-authentication
                         [Optional] HPC requires 2FA authentication
+  -acct, --accounting   [Optional] Enable returning accounting information of data transfer
+  -ct CONCURRENT_TASKS, --concurrent-tasks CONCURRENT_TASKS
+                        [Optional] set the number of concurrent tasks for parallel data transfer
 ```
 
 A common command flow (e.g. transfer data from hdfs to hpc) would be like this:
@@ -216,6 +219,15 @@ A common command flow (e.g. transfer data from hdfs to hpc) would be like this:
 - execute *hdfs2hcp* CLI command to transfer data from an hdfs location (e.g. /users/yosu/data/genome-tags.csv) to a remote HPC (e.g. LUMI, at $HOME/data folder)
 - check status of *hdfs2hcp* transfer (and possible warnings/errors) with *check-status* CLI command
 
+If accounting report is enabled, the output of the command will include some transfer statistics:
+```
+Data transfer report:
+Transfer time: 21 s
+Transfer size: 12.86 MB
+Transfer rate: 0.61 MB/s
+Number of transferred files: 1
+```
+
 ## Support for HPC clusters that require a 2FA token
 The Data Transfer CLI tool's commands support transferring data to/from HPC clusters that require a 2FA token. These commands offer an optional flag *_2fa*. If set by the user, the command prompts the user (in the standard input) for the token when required. 
 
@@ -248,6 +260,7 @@ Note: Hidalgo2 HPDA configuration is included in the Data Transfer CLI tool impl
 
 Then, when you launch a Data Tranfer CLI tool command, any parameter not included in the command line will be retrieved from the config file if the corresponding host entry is included. After that, if the command line gets complete (i.e. all required parameters are provided), the command will be executed, otherwise the corresponding error will be triggered.
 
-
+## Data transfer optimization
+You can improve the data transfer rate by setting the optional parameter *-ct|--concurrent-tasks* (*integer*) to the number of concurrent tasks that will be used in the NIFI pipeline (default is 1). The maximum number of tasks that improve the transfer throughput depends on the physical resources of the NIFI server (consult its administrator). The parallel transfer is currently supported to/from HPC and HDFS data servers, but not to/from CKAN (under development)
 
 

{data_transfer_cli-0.3.6 → data_transfer_cli-0.3.7}/README.md

@@ -190,6 +190,9 @@ options:
                         [Optional] Password for HPC secret key
   -2fa, --two-factor-authentication
                         [Optional] HPC requires 2FA authentication
+  -acct, --accounting   [Optional] Enable returning accounting information of data transfer
+  -ct CONCURRENT_TASKS, --concurrent-tasks CONCURRENT_TASKS
+                        [Optional] set the number of concurrent tasks for parallel data transfer
 ```
 
 A common command flow (e.g. transfer data from hdfs to hpc) would be like this:
@@ -197,6 +200,15 @@ A common command flow (e.g. transfer data from hdfs to hpc) would be like this:
 - execute *hdfs2hcp* CLI command to transfer data from an hdfs location (e.g. /users/yosu/data/genome-tags.csv) to a remote HPC (e.g. LUMI, at $HOME/data folder)
 - check status of *hdfs2hcp* transfer (and possible warnings/errors) with *check-status* CLI command
 
+If accounting report is enabled, the output of the command will include some transfer statistics:
+```
+Data transfer report:
+Transfer time: 21 s
+Transfer size: 12.86 MB
+Transfer rate: 0.61 MB/s
+Number of transferred files: 1
+```
+
 ## Support for HPC clusters that require a 2FA token
 The Data Transfer CLI tool's commands support transferring data to/from HPC clusters that require a 2FA token. These commands offer an optional flag *_2fa*. If set by the user, the command prompts the user (in the standard input) for the token when required. 
 
@@ -229,5 +241,6 @@ Note: Hidalgo2 HPDA configuration is included in the Data Transfer CLI tool impl
 
 Then, when you launch a Data Tranfer CLI tool command, any parameter not included in the command line will be retrieved from the config file if the corresponding host entry is included. After that, if the command line gets complete (i.e. all required parameters are provided), the command will be executed, otherwise the corresponding error will be triggered.
 
-
+## Data transfer optimization
+You can improve the data transfer rate by setting the optional parameter *-ct|--concurrent-tasks* (*integer*) to the number of concurrent tasks that will be used in the NIFI pipeline (default is 1). The maximum number of tasks that improve the transfer throughput depends on the physical resources of the NIFI server (consult its administrator). The parallel transfer is currently supported to/from HPC and HDFS data servers, but not to/from CKAN (under development)
 

{data_transfer_cli-0.3.6 → data_transfer_cli-0.3.7}/data_transfer_cli/dtcli.py

@@ -122,11 +122,28 @@ class ThreadRaisingExceptions(threading.Thread):
     def __init__(self, *args, **kwargs):
         self._exception = None
         self._process_group_id = None
+        self.accounting = kwargs['args'][0].accounting \
+            if 'args' in kwargs and kwargs['args'] else False
         super().__init__(*args, **kwargs)
 
     def run(self):
         try:
-            self._process_group_id = self._target(*self._args, **self._kwargs)
+            self._process_group_id, accounting_info = \
+                self._target(*self._args, **self._kwargs)
+            if self.accounting:  # Report accounting information
+                transfer_time = accounting_info.pipeline_timespan
+                number_transfer_files = len(
+                    accounting_info.flowfiles_sizes)
+                transfer_size = sum(
+                    accounting_info.flowfiles_sizes.values())/(1024*1024)
+                transfer_rate = transfer_size / transfer_time
+
+                msg = "Data transfer report:\n"
+                msg += f"Transfer time: {transfer_time} s\n"
+                msg += f"Transfer size: {transfer_size:.2f} MB\n"
+                msg += f"Transfer rate: {transfer_rate:.2f} MB/s\n"
+                msg += f"Number of transferred files: {number_transfer_files}\n"
+                print(msg)
         except HidDataTransferException as e:
             self._exception = e
             raise e

{data_transfer_cli-0.3.6 → data_transfer_cli-0.3.7}/data_transfer_cli/dtproxy.py

@@ -20,7 +20,7 @@ and run processors in a process group.
 
 from hid_data_transfer_lib.exceptions.hid_dt_exceptions import HidDataTransferException
 from hid_data_transfer_lib.conf.hid_dt_configuration import HidDataTransferConfiguration
-from hid_data_transfer_lib.hid_dt_lib import HIDDataTransfer
+from hid_data_transfer_lib.hid_dt_lib import HIDDataTransfer, AccountingInfo
 
 
 class DataTransferProxy:
@@ -82,7 +82,7 @@ class DataTransferProxy:
 
     # MAIN CLI commands
 
-    def hdfs2hpc(self, args) -> str:
+    def hdfs2hpc(self, args) -> tuple[str, AccountingInfo]:
         """transfer data from HDFS to hpc using SFTP"""
         self.__logger.info(
             "executing hdfs2hpc command with args: %s", self.format_args_to_string(args)
@@ -100,6 +100,7 @@ class DataTransferProxy:
                     data_target=args.data_target,
                     kerberos_principal=args.kerberos_principal,
                     kerberos_password=args.kerberos_password,
+                    concurrent_tasks=args.concurrent_tasks,
                 )
             return self.dt_client.hdfs2hpc(
                 hpc_host=args.hpc_host,
@@ -112,12 +113,13 @@ class DataTransferProxy:
                 data_target=args.data_target,
                 kerberos_principal=args.kerberos_principal,
                 kerberos_password=args.kerberos_password,
+                concurrent_tasks=args.concurrent_tasks,
             )
 
         except Exception as ex:
             raise HidDataTransferException(ex) from ex
 
-    def hpc2hdfs(self, args) -> str:
+    def hpc2hdfs(self, args) -> tuple[str, AccountingInfo]:
         """transfer data from HPC to hdfs using SFTP"""
         self.__logger.info(
             "executing hpc2hdfs command with args: %s", self.format_args_to_string(args)
@@ -135,6 +137,7 @@ class DataTransferProxy:
                     data_target=args.data_target,
                     kerberos_principal=args.kerberos_principal,
                     kerberos_password=args.kerberos_password,
+                    concurrent_tasks=args.concurrent_tasks,
                 )
             return self.dt_client.hpc2hdfs(
                     hpc_host=args.hpc_host,
@@ -147,12 +150,13 @@ class DataTransferProxy:
                     data_target=args.data_target,
                     kerberos_principal=args.kerberos_principal,
                     kerberos_password=args.kerberos_password,
+                    concurrent_tasks=args.concurrent_tasks,
                 )
 
         except Exception as ex:
             raise HidDataTransferException(ex) from ex
 
-    def hdfs2ckan(self, args) -> str:
+    def hdfs2ckan(self, args) -> tuple[str, AccountingInfo]:
         """transfer data from HDFS to CKAN using SFTP"""
         self.__logger.info(
             "executing hpc2ckan command with args: %s", self.format_args_to_string(args)
@@ -167,12 +171,13 @@ class DataTransferProxy:
                 data_source=args.data_source,
                 kerberos_principal=args.kerberos_principal,
                 kerberos_password=args.kerberos_password,
+                concurrent_tasks=args.concurrent_tasks,
             )
 
         except Exception as ex:
             raise HidDataTransferException(ex) from ex
 
-    def ckan2hdfs(self, args) -> str:
+    def ckan2hdfs(self, args) -> tuple[str, AccountingInfo]:
         """transfer data from CKAN to HPC using SFTP"""
         self.__logger.info(
             "executing ckan2hpc command with args: %s", self.format_args_to_string(args)
@@ -188,12 +193,13 @@ class DataTransferProxy:
                 data_target=args.data_target,
                 kerberos_principal=args.kerberos_principal,
                 kerberos_password=args.kerberos_password,
+                concurrent_tasks=args.concurrent_tasks,
             )
 
         except Exception as ex:
             raise HidDataTransferException(ex) from ex
 
-    def hpc2ckan(self, args) -> str:
+    def hpc2ckan(self, args) -> tuple[str, AccountingInfo]:
         """transfer data from hpc to CKAN using SFTP"""
         self.__logger.info(
             "executing hpc2ckan command with args: %s", self.format_args_to_string(args)
@@ -208,12 +214,14 @@ class DataTransferProxy:
                     ckan_api_key=args.ckan_api_key,
                     ckan_organization=args.ckan_organization,
                     ckan_dataset=args.ckan_dataset,
+                    ckan_resource=args.ckan_resource,
                     hpc_host=args.hpc_host,
                     hpc_port=args.hpc_port,
                     hpc_username=args.hpc_username,
                     hpc_secret_key_path=args.hpc_secret_key,
                     hpc_secret_key_password=args.hpc_secret_key_password,
                     data_source=args.data_source,
+                    concurrent_tasks=args.concurrent_tasks,
                 )
             return self.dt_client.hpc2ckan(
                 ckan_host=args.ckan_host
@@ -221,6 +229,7 @@ class DataTransferProxy:
                 ckan_api_key=args.ckan_api_key,
                 ckan_organization=args.ckan_organization,
                 ckan_dataset=args.ckan_dataset,
+                ckan_resource=args.ckan_resource,
                 hpc_host=args.hpc_host,
                 hpc_port=args.hpc_port,
                 hpc_username=args.hpc_username,
@@ -228,12 +237,13 @@ class DataTransferProxy:
                 hpc_secret_key_path=args.hpc_secret_key,
                 hpc_secret_key_password=args.hpc_secret_key_password,
                 data_source=args.data_source,
+                concurrent_tasks=args.concurrent_tasks,
             )
 
         except Exception as ex:
             raise HidDataTransferException(ex) from ex
 
-    def ckan2hpc(self, args) -> str:
+    def ckan2hpc(self, args) -> tuple[str, AccountingInfo]:
         """transfer data from CKAN to hpc using SFTP"""
         self.__logger.info(
             "executing ckan2hpc command with args: %s", self.format_args_to_string(args)
@@ -255,6 +265,7 @@ class DataTransferProxy:
                     hpc_secret_key_path=args.hpc_secret_key,
                     hpc_secret_key_password=args.hpc_secret_key_password,
                     data_target=args.data_target,
+                    concurrent_tasks=args.concurrent_tasks,
                 )
             return self.dt_client.ckan2hpc(
                 ckan_host=args.ckan_host
@@ -270,12 +281,13 @@ class DataTransferProxy:
                 hpc_secret_key_path=args.hpc_secret_key,
                 hpc_secret_key_password=args.hpc_secret_key_password,
                 data_target=args.data_target,
+                concurrent_tasks=args.concurrent_tasks,
             )
 
         except Exception as ex:
             raise HidDataTransferException(ex) from ex
 
-    def local2ckan(self, args) -> str:
+    def local2ckan(self, args) -> tuple[str, AccountingInfo]:
         """transfer data from local filesystem to CKAN using SFTP"""
         self.__logger.info(
             "executing local2ckan command with args: %s",
@@ -296,7 +308,7 @@ class DataTransferProxy:
         except Exception as ex:
             raise HidDataTransferException(ex) from ex
 
-    def ckan2local(self, args) -> str:
+    def ckan2local(self, args) -> tuple[str, AccountingInfo]:
         """transfer data from CKAN to the local filesystem using SFTP"""
         self.__logger.info(
             "executing ckan2local command with args: %s",

{data_transfer_cli-0.3.6 → data_transfer_cli-0.3.7}/data_transfer_cli/parser/cli_parser.py

@@ -224,6 +224,17 @@ class CLIParser(argparse.ArgumentParser):
             required=False, action="store_true", default=False,
             help="[Optional] HPC requires 2FA authentication"
         )
+        hdfs2hpc_parser.add_argument(
+            "-acct", "--accounting",
+            required=False, action="store_true", default=False,
+            help="[Optional] Enable returning accounting information of data transfer"
+        )
+        hdfs2hpc_parser.add_argument(
+            "-ct", "--concurrent-tasks",
+            required=False, type=int, default=1,
+            help="[Optional] set the number of concurrent tasks"
+            " for parallel data transfer"
+        )
         hdfs2hpc_parser.set_defaults(func=target.hdfs2hpc)
 
         # hpc2hdfs
@@ -244,6 +255,17 @@ class CLIParser(argparse.ArgumentParser):
             required=False, action="store_true", default=False,
             help="[Optional] HPC requires 2FA authentication"
         )
+        hpc2hdfs_parser.add_argument(
+            "-acct", "--accounting",
+            required=False, action="store_true", default=False,
+            help="[Optional] Enable returning accounting information of data transfer"
+        )
+        hpc2hdfs_parser.add_argument(
+            "-ct", "--concurrent-tasks",
+            required=False, type=int, default=1,
+            help="[Optional] set the number of concurrent tasks"
+            " for parallel data transfer"
+        )
         hpc2hdfs_parser.set_defaults(func=target.hpc2hdfs)
 
         # ckan2hdfs
@@ -264,6 +286,17 @@ class CLIParser(argparse.ArgumentParser):
             "-t", "--data-target", required=False, help="[Optional] target HDFS folder"
         )
         ckan2hdfs_parser = self.add_default_kerberos_arguments(ckan2hdfs_parser)
+        ckan2hdfs_parser.add_argument(
+            "-acct", "--accounting",
+            required=False, action="store_true", default=False,
+            help="[Optional] Enable returning accounting information of data transfer"
+        )
+        ckan2hdfs_parser.add_argument(
+            "-ct", "--concurrent-tasks",
+            required=False, type=int, default=1,
+            help="[Optional] set the number of concurrent tasks"
+            " for parallel data transfer"
+        )
         ckan2hdfs_parser.set_defaults(func=target.ckan2hdfs)
 
         # hdfs2ckan
@@ -278,6 +311,17 @@ class CLIParser(argparse.ArgumentParser):
             required=True,
             help="File path to HDFS file or directory to transfer",
         )
+        hdfs2ckan_parser.add_argument(
+            "-acct", "--accounting",
+            required=False, action="store_true", default=False,
+            help="[Optional] Enable returning accounting information of data transfer"
+        )
+        hdfs2ckan_parser.add_argument(
+            "-ct", "--concurrent-tasks",
+            required=False, type=int, default=1,
+            help="[Optional] set the number of concurrent tasks"
+            " for parallel data transfer"
+        )
         hdfs2ckan_parser.set_defaults(func=target.hdfs2ckan)
 
         # ckan2hpc
@@ -303,6 +347,17 @@ class CLIParser(argparse.ArgumentParser):
             required=False, action="store_true", default=False,
             help="[Optional] HPC requires 2FA authentication"
         )
+        ckan2hpc_parser.add_argument(
+            "-acct", "--accounting",
+            required=False, action="store_true", default=False,
+            help="[Optional] Enable returning accounting information of data transfer"
+        )
+        ckan2hpc_parser.add_argument(
+            "-ct", "--concurrent-tasks",
+            required=False, type=int, default=1,
+            help="[Optional] set the number of concurrent tasks"
+            " for parallel data transfer"
+        )
         ckan2hpc_parser.set_defaults(func=target.ckan2hpc)
 
         # hpc2ckan
@@ -310,6 +365,13 @@ class CLIParser(argparse.ArgumentParser):
             "hpc2ckan", help="transfer data from HPC to a target CKAN"
         )
         hpc2ckan_parser = self.add_default_ckan_arguments(hpc2ckan_parser)
+        hpc2ckan_parser.add_argument(
+            "-r",
+            "--ckan-resource",
+            required=False,
+            help="[Optional] CKAN resource to create from transferred sources. \
+            If omitted, target resource name will adopt the source file or folder name",
+        )
         hpc2ckan_parser = self.add_default_hpc_arguments(hpc2ckan_parser)
         hpc2ckan_parser.add_argument(
             "-2fa", "--two-factor-authentication",
@@ -322,6 +384,17 @@ class CLIParser(argparse.ArgumentParser):
             required=True,
             help="File path to HPC file or directory to transfer",
         )
+        hpc2ckan_parser.add_argument(
+            "-acct", "--accounting",
+            required=False, action="store_true", default=False,
+            help="[Optional] Enable returning accounting information of data transfer"
+        )
+        hpc2ckan_parser.add_argument(
+            "-ct", "--concurrent-tasks",
+            required=False, type=int, default=1,
+            help="[Optional] set the number of concurrent tasks"
+            " for parallel data transfer"
+        )
         hpc2ckan_parser.set_defaults(func=target.hpc2ckan)
 
         # local2ckan
@@ -336,13 +409,17 @@ class CLIParser(argparse.ArgumentParser):
             help="[Optional] CKAN resource to create from transferred sources. \
             If omitted, target resource name will adopt the source file or folder name",
         )
-
         local2ckan_parser.add_argument(
             "-s",
             "--data-source",
             required=True,
             help="File path to local file or directory to transfer",
         )
+        local2ckan_parser.add_argument(
+            "-acct", "--accounting",
+            required=False, action="store_true", default=False,
+            help="[Optional] Enable returning accounting information of data transfer"
+        )
         local2ckan_parser.set_defaults(func=target.local2ckan)
 
         # ckan2local
@@ -357,7 +434,6 @@ class CLIParser(argparse.ArgumentParser):
             help="[Optional] CKAN resource to transfer. \
             If omitted, all resources in the dataset will be transferred",
         )
-
         ckan2local_parser.add_argument(
             "-t",
             "--data-target",
@@ -365,6 +441,11 @@ class CLIParser(argparse.ArgumentParser):
             help="Local directory where to transfer the data. \
             If omitted, data will be transferred to the current directory",
         )
+        ckan2local_parser.add_argument(
+            "-acct", "--accounting",
+            required=False, action="store_true", default=False,
+            help="[Optional] Enable returning accounting information of data transfer"
+        )
         ckan2local_parser.set_defaults(func=target.ckan2local)
 
         return self.parse_args(args)

{data_transfer_cli-0.3.6 → data_transfer_cli-0.3.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "data-transfer-cli"
-version = "0.3.6"
+version = "0.3.7"
 description = "HiDALGO Data Transfer CLI provides commands to transfer data between different data providers and consumers using NIFI pipelines"
 authors = [
     { name = "Jesús Gorroñogoitia", email = "jesus.gorronogoitia@eviden.com" },
@@ -11,9 +11,9 @@ requires-python = ">=3.11, <4.0"
 dependencies = [
     "requests>=2.31.0",
     "paramiko>=3.3.1",
-    "hid_data_transfer_lib>=0.3.4",
-    #"hid_data_transfer_lib @ file:///home/yosu/Projects/Hidalgo2/git/hid-data-management/data-transfer/nifi/hid_data_transfer_lib/dist/hid_data_transfer_lib-0.3.4-py3-none-any.whl",
+    "hid_data_transfer_lib>=0.3.7",
     "pyyaml (>=6.0.2,<7.0.0)",
+    #"hid-data-transfer-lib @ file:///home/yosu/Projects/Hidalgo2/git/hid-data-management/data-transfer/nifi/hid_data_transfer_lib/dist/hid_data_transfer_lib-0.3.7-py3-none-any.whl",
 ]
 
 [tool.poetry]