data-transfer-cli 0.3.7__tar.gz → 0.3.9__tar.gz

{data_transfer_cli-0.3.7 → data_transfer_cli-0.3.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: data-transfer-cli
-Version: 0.3.7
+Version: 0.3.9
 Summary: HiDALGO Data Transfer CLI provides commands to transfer data between different data providers and consumers using NIFI pipelines
 License: APL-2.0
 Author: Jesús Gorroñogoitia
@@ -11,7 +11,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
-Requires-Dist: hid_data_transfer_lib (>=0.3.7)
+Requires-Dist: hid_data_transfer_lib (>=0.3.9)
 Requires-Dist: paramiko (>=3.3.1)
 Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
 Requires-Dist: requests (>=2.31.0)
@@ -212,6 +212,7 @@ options:
   -acct, --accounting   [Optional] Enable returning accounting information of data transfer
   -ct CONCURRENT_TASKS, --concurrent-tasks CONCURRENT_TASKS
                         [Optional] set the number of concurrent tasks for parallel data transfer
+  -R, --recursive       [Optional] if True the data-source subdirectories will be transferred as well, otherwise only the root data-source folder
 ```
 
 A common command flow (e.g. transfer data from hdfs to hpc) would be like this:

{data_transfer_cli-0.3.7 → data_transfer_cli-0.3.9}/README.md

@@ -193,6 +193,7 @@ options:
   -acct, --accounting   [Optional] Enable returning accounting information of data transfer
   -ct CONCURRENT_TASKS, --concurrent-tasks CONCURRENT_TASKS
                         [Optional] set the number of concurrent tasks for parallel data transfer
+  -R, --recursive       [Optional] if True the data-source subdirectories will be transferred as well, otherwise only the root data-source folder
 ```
 
 A common command flow (e.g. transfer data from hdfs to hpc) would be like this:

{data_transfer_cli-0.3.7 → data_transfer_cli-0.3.9}/data_transfer_cli/dtproxy.py

@@ -114,6 +114,7 @@ class DataTransferProxy:
                 kerberos_principal=args.kerberos_principal,
                 kerberos_password=args.kerberos_password,
                 concurrent_tasks=args.concurrent_tasks,
+                recursive=args.recursive,
             )
 
         except Exception as ex:
@@ -151,6 +152,7 @@ class DataTransferProxy:
                     kerberos_principal=args.kerberos_principal,
                     kerberos_password=args.kerberos_password,
                     concurrent_tasks=args.concurrent_tasks,
+                    recursive=args.recursive,
                 )
 
         except Exception as ex:
@@ -168,10 +170,12 @@ class DataTransferProxy:
                 ckan_api_key=args.ckan_api_key,
                 ckan_organization=args.ckan_organization,
                 ckan_dataset=args.ckan_dataset,
+                ckan_resource=args.ckan_resource,
                 data_source=args.data_source,
                 kerberos_principal=args.kerberos_principal,
                 kerberos_password=args.kerberos_password,
                 concurrent_tasks=args.concurrent_tasks,
+                recursive=args.recursive,
             )
 
         except Exception as ex:
@@ -238,6 +242,7 @@ class DataTransferProxy:
                 hpc_secret_key_password=args.hpc_secret_key_password,
                 data_source=args.data_source,
                 concurrent_tasks=args.concurrent_tasks,
+                recursive=args.recursive,
             )
 
         except Exception as ex:

{data_transfer_cli-0.3.7 → data_transfer_cli-0.3.9}/data_transfer_cli/parser/cli_parser.py

@@ -235,6 +235,12 @@ class CLIParser(argparse.ArgumentParser):
             help="[Optional] set the number of concurrent tasks"
             " for parallel data transfer"
         )
+        hdfs2hpc_parser.add_argument(
+            "-R", "--recursive",
+            required=False, action="store_true", default=False,
+            help="[Optional] if True the data-source subdirectories"
+            " will be transferred as well, otherwise only the root data-source folder"
+        )
         hdfs2hpc_parser.set_defaults(func=target.hdfs2hpc)
 
         # hpc2hdfs
@@ -266,6 +272,12 @@ class CLIParser(argparse.ArgumentParser):
             help="[Optional] set the number of concurrent tasks"
             " for parallel data transfer"
         )
+        hpc2hdfs_parser.add_argument(
+            "-R", "--recursive",
+            required=False, action="store_true", default=False,
+            help="[Optional] if True the data-source subdirectories"
+            " will be transferred as well, otherwise only the root data-source folder"
+        )
         hpc2hdfs_parser.set_defaults(func=target.hpc2hdfs)
 
         # ckan2hdfs
@@ -304,6 +316,13 @@ class CLIParser(argparse.ArgumentParser):
             "hdfs2ckan", help="transfer data from HDFS to a target CKAN"
         )
         hdfs2ckan_parser = self.add_default_ckan_arguments(hdfs2ckan_parser)
+        hdfs2ckan_parser.add_argument(
+            "-r",
+            "--ckan-resource",
+            required=False,
+            help="[Optional] CKAN resource to create from transferred sources. \
+            If omitted, target resource name will adopt the source file or folder name",
+        )
         hdfs2ckan_parser = self.add_default_kerberos_arguments(hdfs2ckan_parser)
         hdfs2ckan_parser.add_argument(
             "-s",
@@ -322,6 +341,12 @@ class CLIParser(argparse.ArgumentParser):
             help="[Optional] set the number of concurrent tasks"
             " for parallel data transfer"
         )
+        hdfs2ckan_parser.add_argument(
+            "-R", "--recursive",
+            required=False, action="store_true", default=False,
+            help="[Optional] if True the data-source subdirectories"
+            " will be transferred as well, otherwise only the root data-source folder"
+        )
         hdfs2ckan_parser.set_defaults(func=target.hdfs2ckan)
 
         # ckan2hpc
@@ -395,6 +420,12 @@ class CLIParser(argparse.ArgumentParser):
             help="[Optional] set the number of concurrent tasks"
             " for parallel data transfer"
         )
+        hpc2ckan_parser.add_argument(
+            "-R", "--recursive",
+            required=False, action="store_true", default=False,
+            help="[Optional] if True the data-source subdirectories"
+            " will be transferred as well, otherwise only the root data-source folder"
+        )
         hpc2ckan_parser.set_defaults(func=target.hpc2ckan)
 
         # local2ckan

{data_transfer_cli-0.3.7 → data_transfer_cli-0.3.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "data-transfer-cli"
-version = "0.3.7"
+version = "0.3.9"
 description = "HiDALGO Data Transfer CLI provides commands to transfer data between different data providers and consumers using NIFI pipelines"
 authors = [
     { name = "Jesús Gorroñogoitia", email = "jesus.gorronogoitia@eviden.com" },
@@ -11,9 +11,9 @@ requires-python = ">=3.11, <4.0"
 dependencies = [
     "requests>=2.31.0",
     "paramiko>=3.3.1",
-    "hid_data_transfer_lib>=0.3.7",
+    "hid_data_transfer_lib>=0.3.9",
     "pyyaml (>=6.0.2,<7.0.0)",
-    #"hid-data-transfer-lib @ file:///home/yosu/Projects/Hidalgo2/git/hid-data-management/data-transfer/nifi/hid_data_transfer_lib/dist/hid_data_transfer_lib-0.3.7-py3-none-any.whl",
+    #"hid-data-transfer-lib @ file:///home/yosu/Projects/Hidalgo2/git/hid-data-management/data-transfer/nifi/hid_data_transfer_lib/dist/hid_data_transfer_lib-0.3.9-py3-none-any.whl",
 ]
 
 [tool.poetry]
@@ -47,3 +47,4 @@ dtcli = 'data_transfer_cli.dtcli:main'
 
 [tool.pytest.ini_options]
 pythonpath = ["data_transfer_cli"]
+addopts = "-s"