hid_data_transfer_lib 0.3.11__tar.gz → 0.4.0__tar.gz

{hid_data_transfer_lib-0.3.11 → hid_data_transfer_lib-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hid_data_transfer_lib
-Version: 0.3.11
+Version: 0.4.0
 Summary: HiDALGO Data Transfer library provides methods to transfer data between different data providers and consumers using NIFI pipelines
 License: APL-2.0
 Author: Jesús Gorroñogoitia
@@ -123,7 +123,7 @@ keycloak_login=<user_name>
 keycloak_password=<password>
 ```
 
-Under the Nifi section, you must also specify a user account (username, private_key) that grants to upload/download files to the NIFI server (as requested to upload temporary HPC keys or to support local file transfer). This user's account is provided by Hidalgo2 infrastructure provider and it is user's or service's specific. 
+Under the Nifi section, you must also specify a user account (username, private_key) that grants to upload/download files to the NIFI server (as requested to upload temporary HPC keys or to support local file transfer). This user's account is provided by Hidalgo2 infrastructure provider and it is user's or service's specific. Note: this user's account requires to belond to group *users*.
 
 Under the Keycloak section, you must specify your Keycloak account (username and password). This account granted with access to the NIFI service.
 

{hid_data_transfer_lib-0.3.11 → hid_data_transfer_lib-0.4.0}/README.md

@@ -104,7 +104,7 @@ keycloak_login=<user_name>
 keycloak_password=<password>
 ```
 
-Under the Nifi section, you must also specify a user account (username, private_key) that grants to upload/download files to the NIFI server (as requested to upload temporary HPC keys or to support local file transfer). This user's account is provided by Hidalgo2 infrastructure provider and it is user's or service's specific. 
+Under the Nifi section, you must also specify a user account (username, private_key) that grants to upload/download files to the NIFI server (as requested to upload temporary HPC keys or to support local file transfer). This user's account is provided by Hidalgo2 infrastructure provider and it is user's or service's specific. Note: this user's account requires to belond to group *users*.
 
 Under the Keycloak section, you must specify your Keycloak account (username and password). This account granted with access to the NIFI service.
 

{hid_data_transfer_lib-0.3.11 → hid_data_transfer_lib-0.4.0}/hid_data_transfer_lib/conf/hid_dt_configuration.py

@@ -24,6 +24,7 @@ from hid_data_transfer_lib.exceptions.hid_dt_exceptions import (
     HidDataTransferException
 )
 
+STATUS_SLEEP_LAPSE = 10 # seconds
 
 class HidDataTransferConfiguration:
     """This class manages the configuration of the Data Transfer library"""
@@ -44,21 +45,22 @@ class HidDataTransferConfiguration:
     __keycloak_login = None
     __keycloak_passwd = None
 
-    __check_status_sleep_lapse = 10  # seconds
+    __check_status_sleep_lapse = STATUS_SLEEP_LAPSE  # seconds
 
     def __init__(self, logging_level: Optional[int] = logging.INFO):
         self.__logging_level = logging_level
 
-    def check_keycloak_conf(self):
+    def check_keycloak_conf(self, use_keycloak_account):
         """Check if keycloak configuration is valid"""
-        if self.__keycloak_login is None:
-            raise HidDataTransferException(
-                "Keycloak login must be set in the configuration"
-            )
-        if self.__keycloak_passwd is None:
-            raise HidDataTransferException(
-                "Keycloak passwd must be set in the configuration"
-            )
+        if use_keycloak_account:
+            if self.__keycloak_login is None:
+                raise HidDataTransferException(
+                    "Keycloak login must be set in the configuration"
+                )
+            if self.__keycloak_passwd is None:
+                raise HidDataTransferException(
+                    "Keycloak passwd must be set in the configuration"
+                )
         if self.__keycloak_endpoint is None:
             raise HidDataTransferException(
                 "Keycloak endpoint must be set in the configuration"
@@ -195,6 +197,14 @@ class HidDataTransferConfiguration:
         """Returns the NIFI login password"""
         return self.__keycloak_passwd
 
+    def reset_keycloak_login(self):
+        """Reset the NIFI login name"""
+        self.__keycloak_login = None
+
+    def reset_keycloak_passwd(self):
+        """Reset the NIFI login password"""
+        self.__keycloak_passwd = None
+
     def logger(self, logger_name):
         """Returns the logging level"""
         return self.__set_logger(logger_name)
@@ -256,7 +266,8 @@ class HidDataTransferConfiguration:
         self,
         user_config_file: Optional[str] = None,
         user_config_dict: Optional[dict] = None,
-        logging_level: Optional[int] = None
+        logging_level: Optional[int] = None,
+        use_keycloak_account: Bool = True,
     ) -> HidDataTransferConfiguration:
         """
         Reads the configuration and sets this class' configuration values
@@ -276,38 +287,33 @@ class HidDataTransferConfiguration:
             self.load_nifi_configuration(config)
 
             # Keycloak section
-            self.load_keycloak_configuration(config)
-
-            # Check if either NIFI or Keycloak login and password are set
-            if (self.__nifi_login is None and self.__nifi_passwd is None) and (
-                self.__keycloak_login is None and
-                self.__keycloak_passwd is None
-            ):
-                raise HidDataTransferException(
-                    "Either (KEYCLOAK_LOGIN and KEYCLOAK_PASSWORD) "
-                    "or (NIFI_LOGIN and NIFI_PASSWORD) "
-                    "environment variables must be set"
-                )
+            self.load_keycloak_configuration(
+                config, check_configuration = use_keycloak_account)
+
+            # # Check if either NIFI or Keycloak login and password are set
+            # if not use_keycloak_token and (
+            #     self.__nifi_login is None and self.__nifi_passwd is None
+            #     ) and (
+            #     self.__keycloak_login is None and
+            #     self.__keycloak_passwd is None
+            # ):
+            #     raise HidDataTransferException(
+            #         "Either (KEYCLOAK_LOGIN and KEYCLOAK_PASSWORD) "
+            #         "or (NIFI_LOGIN and NIFI_PASSWORD) "
+            #         "environment variables must be set"
+            #     )
 
             # Logging section
             if logging_level is not None:
                 self.__logging_level = logging_level
             else:
-                logging_level_str = config.get("Logging", "logging_level")
+                logging_level_str = config.get("Logging", "logging_level", fallback="Info")
                 self.__logging_level = getattr(
                     logging, logging_level_str.upper(), logging.INFO)
 
             # Network section
-            self.__check_status_sleep_lapse = config.get(
-                "Network", "check_status_sleep_lapse")
-
-            # Network section
-            self.__check_status_sleep_lapse = config.get(
-                "Network", "check_status_sleep_lapse")
-
-            # Network section
-            self.__check_status_sleep_lapse = config.get(
-                "Network", "check_status_sleep_lapse")
+            self.__check_status_sleep_lapse = int(config.get(
+                "Network", "check_status_sleep_lapse", fallback=STATUS_SLEEP_LAPSE))
 
             return self
 
@@ -316,69 +322,67 @@ class HidDataTransferConfiguration:
                 "Error parsing CLI configuration file"
             ) from ex
 
-    def load_keycloak_configuration(self, config):
+    def load_keycloak_configuration(self, config, check_configuration = True):
         """Load Keycloak configuration from config file"""
         # Read Keycloak login and password from configuration
-        self.__keycloak_login = config.get("Keycloak", "keycloak_login")
-        self.__keycloak_passwd = config.get("Keycloak", "keycloak_password")
-        if (self.__keycloak_login and self.__keycloak_passwd is None) or (
-            self.__keycloak_login is None and self.__keycloak_passwd
-        ):
+        self.__keycloak_login = config.get("Keycloak", "keycloak_login", fallback=None)
+        self.__keycloak_passwd = config.get("Keycloak", "keycloak_password", fallback=None)
+        if check_configuration and (
+            self.__keycloak_login is None or self.__keycloak_passwd is None):
             raise HidDataTransferException(
                 "Both KEYCLOAK_LOGIN and KEYCLOAK_PASSWORD "
                 "environment variables must be set"
             )
 
         # Read other keycloak configuration from config file
-        if self.__keycloak_login or self.__keycloak_passwd:
-            self.__keycloak_endpoint = config.get(
-                "Keycloak", "keycloak_endpoint")
-            if self.__keycloak_endpoint is None:
-                raise HidDataTransferException(
-                    "Could not find Keycloak endpoint in config file."
-                )
+        self.__keycloak_endpoint = config.get(
+            "Keycloak", "keycloak_endpoint", fallback=None)
+        if self.__keycloak_endpoint is None:
+            raise HidDataTransferException(
+                "Could not find Keycloak endpoint in config file."
+            )
 
-            self.__keycloak_client_id = config.get(
-                "Keycloak", "keycloak_client_id")
-            if self.__keycloak_client_id is None:
-                raise HidDataTransferException(
-                    "Could not find Keycloak client id "
-                    "for NIFI in config file."
-                )
+        self.__keycloak_client_id = config.get(
+            "Keycloak", "keycloak_client_id", fallback=None)
+        if self.__keycloak_client_id is None:
+            raise HidDataTransferException(
+                "Could not find Keycloak client id "
+                "for NIFI in config file."
+            )
 
-            keycloak_client_secret = config.get(
-                "Keycloak", "keycloak_client_secret")
-            if keycloak_client_secret is not None:
-                self.__keycloak_client_secret = keycloak_client_secret
-            else:
-                raise HidDataTransferException(
-                    "Could not find Keycloak client secret "
-                    "for NIFI in config file."
-                )
+        keycloak_client_secret = config.get(
+            "Keycloak", "keycloak_client_secret", fallback=None)
+        if keycloak_client_secret is not None:
+            self.__keycloak_client_secret = keycloak_client_secret
+        else:
+            raise HidDataTransferException(
+                "Could not find Keycloak client secret "
+                "for NIFI in config file."
+            )
 
     def load_nifi_configuration(self, config):
         """Load NIFI configuration from config file"""
-        self.__nifi_endpoint = config.get("Nifi", "nifi_endpoint")
+        self.__nifi_endpoint = config.get("Nifi", "nifi_endpoint", fallback=None)
         if self.__nifi_endpoint is None:
             raise HidDataTransferException(
                 "Could not find Nifi endpoint in config file."
             )
 
-        nifi_secure_connection = config.get("Nifi", "nifi_secure_connection")
+        nifi_secure_connection = config.get("Nifi", "nifi_secure_connection", fallback=None)
         if nifi_secure_connection is not None:
             self.__nifi_secure_connection = \
                 nifi_secure_connection.lower() == "true"
         else:
             self.__nifi_secure_connection = False
 
-        self.__nifi_upload_folder = config.get("Nifi", "nifi_upload_folder")
+        self.__nifi_upload_folder = config.get("Nifi", "nifi_upload_folder", fallback=None)
         if self.__nifi_upload_folder is None:
             raise HidDataTransferException(
                 "Could not find Nifi nifi_upload_folder in config file"
             )
 
         self.__nifi_download_folder = config.get(
-            "Nifi", "nifi_download_folder")
+            "Nifi", "nifi_download_folder", fallback=None)
         if self.__nifi_download_folder is None:
             raise HidDataTransferException(
                 "Could not find Nifi nifi_download_folder in config file"
@@ -386,9 +390,9 @@ class HidDataTransferConfiguration:
 
         # Optional NIFI server account in case of local file transfer
         self.__nifi_server_user_name = config.get(
-            "Nifi", "nifi_server_username")
+            "Nifi", "nifi_server_username", fallback=None)
         self.__nifi_server_private_key = config.get(
-            "Nifi", "nifi_server_private_key")
+            "Nifi", "nifi_server_private_key", fallback=None)
 
         if (
             self.__nifi_server_user_name and
@@ -400,7 +404,7 @@ class HidDataTransferConfiguration:
                 "environment variables must be set"
             )
 
-        nifi_concurrent_tasks = config.get("Nifi", "nifi_concurrent_tasks")
+        nifi_concurrent_tasks = config.get("Nifi", "nifi_concurrent_tasks", fallback=None)
         if nifi_concurrent_tasks is not None:
             self.__nifi_concurrent_tasks = int(nifi_concurrent_tasks)
         else:

{hid_data_transfer_lib-0.3.11 → hid_data_transfer_lib-0.4.0}/hid_data_transfer_lib/hid_dt_lib.py

@@ -35,8 +35,10 @@ from hid_data_transfer_lib.nifi.nifi_client import (
 )
 
 HDFS_DATA_FOLDER = "hdfs.data_folder"
+HDFS_DATA_FILENAME = "hdfs.data_filename"
 HDFS_FILENAME = "hdfs.filename"
 HPC_TARGET_FOLDER = "hpc.target_folder"
+HPC_TARGET_FILENAME = "hpc.target_filename"
 HPC_DATA_FOLDER = "hpc.data_folder"
 HPC_FILENAME = "hpc.filename"
 HPC_FILENAME_REGEX = "hpc.filename.regex"
@@ -68,13 +70,16 @@ class HIDDataTransfer:
         """
         self.__conf = conf
         self.__secure = secure
-        self.__nifi_client = NIFIClient().configure(conf, secure)
+        self.use_keycloak_account = (keycloak_token is None)
+        self.__nifi_client = NIFIClient().configure(
+            conf, secure, use_keycloak_account=self.use_keycloak_account)
         self.__logger = self.__conf.logger("hid_data_transfer_lib")
         self.__keycloak_token = keycloak_token
 
     def reset(self):
         """ Reset NIFIClient configuration"""
-        self.__nifi_client = NIFIClient().configure(self.__conf, self.__secure)
+        self.__nifi_client = NIFIClient().configure(
+            self.__conf, self.__secure, use_keycloak_account=self.use_keycloak_account)
 
     # Helper methods
     def process_wildcards(self, name):
@@ -173,12 +178,26 @@ class HIDDataTransfer:
         try:
             arguments: Dict[str, Any] = {}
             self.__nifi_client.add_default_hpc_parameters(arguments, args)
-            arguments[HPC_TARGET_FOLDER] = args.data_target
+
             arguments[HDFS_DATA_FOLDER] = \
                 "/".join(args.data_source.split("/")[:-1])
             arguments[HDFS_FILENAME] = \
                 self.process_wildcards(args.data_source.split("/")[-1])
 
+            # If multiple data source files are selected, data_target represents a folder
+            # Else If data_target ends with / represents a folder, otherwise a file
+            arguments[HPC_TARGET_FILENAME] = ""
+            if args.data_source and (
+                "*" in args.data_source or
+                "?" in args.data_source
+            ):
+                arguments[HPC_TARGET_FOLDER] = args.data_target
+            elif args.data_target.endswith('/'):
+                arguments[HPC_TARGET_FOLDER] = args.data_target[:-1]
+            else:
+                arguments[HPC_TARGET_FOLDER] = "/".join(args.data_target.split("/")[:-1])
+                arguments[HPC_TARGET_FILENAME] = "/".join(args.data_target.split("/")[-1:])
+
             arguments[RECURSIVE] = args.recursive
 
             execution_type = {
@@ -548,7 +567,21 @@ class HIDDataTransfer:
             arguments[HPC_DATA_FOLDER] = args.data_source[
                 : args.data_source.rfind("/")
             ]
-            arguments[HDFS_DATA_FOLDER] = args.data_target
+
+            # If multiple data source files are selected, data_target represents a folder
+            # Else If data_target ends with / represents a folder, otherwise a file
+            arguments[HDFS_DATA_FILENAME] = ""
+            if args.data_source and (
+                "*" in args.data_source or
+                "?" in args.data_source
+            ):
+                arguments[HDFS_DATA_FOLDER] = args.data_target
+            elif args.data_target.endswith('/'):
+                arguments[HDFS_DATA_FOLDER] = args.data_target[:-1]
+            else:
+                arguments[HDFS_DATA_FOLDER] = "/".join(args.data_target.split("/")[:-1])
+                arguments[HDFS_DATA_FILENAME] = "/".join(args.data_target.split("/")[-1:])
+
             arguments[RECURSIVE] = args.recursive
 
             execution_type = {
@@ -1038,7 +1071,20 @@ class HIDDataTransfer:
             self.__nifi_client.add_default_ckan_parameters(arguments, args)
             arguments[self.__nifi_client.CKAN_RESOURCE] = \
                 self.process_wildcards(args.ckan_resource)
-            arguments[HDFS_DATA_FOLDER] = args.data_target
+
+            # If multiple ckan resource files are selected, data_target represents a folder
+            # Else If data_target ends with / represents a folder, otherwise a file
+            arguments[HDFS_DATA_FILENAME] = ""
+            if args.ckan_resource and (
+                "*" in args.ckan_resource or
+                "?" in args.ckan_resource
+            ):
+                arguments[HDFS_DATA_FOLDER] = args.data_target
+            elif args.data_target.endswith('/'):
+                arguments[HDFS_DATA_FOLDER] = args.data_target[:-1]
+            else:
+                arguments[HDFS_DATA_FOLDER] = "/".join(args.data_target.split("/")[:-1])
+                arguments[HDFS_DATA_FILENAME] = "/".join(args.data_target.split("/")[-1:])
 
             execution_type = {
                 "CKANFileDownloader": ExecutionType.ONCE,
@@ -1158,11 +1204,24 @@ class HIDDataTransfer:
         try:
             arguments: Dict[str, Any] = {}
             self.__nifi_client.add_default_hpc_parameters(arguments, args)
-            arguments[HPC_TARGET_FOLDER] = args.data_target
             self.__nifi_client.add_default_ckan_parameters(arguments, args)
             arguments[self.__nifi_client.CKAN_RESOURCE] = \
                 self.process_wildcards(args.ckan_resource)
 
+            # If multiple ckan resource files are selected, data_target represents a folder
+            # Else If data_target ends with / represents a folder, otherwise a file
+            arguments[HPC_TARGET_FILENAME] = ""
+            if args.ckan_resource and (
+                "*" in args.ckan_resource or
+                "?" in args.ckan_resource
+            ):
+                arguments[HPC_TARGET_FOLDER] = args.data_target
+            elif args.data_target.endswith('/'):
+                arguments[HPC_TARGET_FOLDER] = args.data_target[:-1]
+            else:
+                arguments[HPC_TARGET_FOLDER] = "/".join(args.data_target.split("/")[:-1])
+                arguments[HPC_TARGET_FILENAME] = "/".join(args.data_target.split("/")[-1:])
+
             keys_arguments = {
                 self.__nifi_client.HPC_SECRET_KEY_PATH: arguments[
                     self.__nifi_client.HPC_SECRET_KEY_PATH

{hid_data_transfer_lib-0.3.11 → hid_data_transfer_lib-0.4.0}/hid_data_transfer_lib/keycloak/keycloak_rest.py

@@ -69,12 +69,13 @@ class KeycloakRESTClient:
 
     def configure(
         self, conf: HidDataTransferConfiguration,
-        secure: bool = False, refresh: bool = False
+        secure: bool = False, refresh: bool = False,
+        use_keycloak_account = True,
     ) -> KeycloakRESTClient:
         """constructs a API client,
         with the Keycloak endpoint taken from configuration
         """
-        conf.check_keycloak_conf()
+        conf.check_keycloak_conf(use_keycloak_account)
         self.__conf = conf
         self.__refresh = refresh
         self.__keycloak_login = conf.keycloak_login()
@@ -117,6 +118,7 @@ class KeycloakRESTClient:
 
     def set_token(self, user, token, expires_in, refresh_token):
         """Set the given token and renews it every time it expires"""
+        self.__keycloak_login = user
         self.__token = token
         self.__refresh_token = refresh_token
         self.__set_expires_in(expires_in)

{hid_data_transfer_lib-0.3.11 → hid_data_transfer_lib-0.4.0}/hid_data_transfer_lib/nifi/nifi_client.py

@@ -135,14 +135,15 @@ class NIFIClient:
         self.check_status_sleep_lapse = 1
 
     def configure(self, conf: HidDataTransferConfiguration,
-                  secure: bool = False) -> NIFIClient:
+                  secure: bool = False, use_keycloak_account = True) -> NIFIClient:
         """constructs a NIFI client,
         with the NIFI endpoint taken from configuration
         """
         self.__conf = conf
         self.nifi_rest_client = NIFIRESTClient().configure(conf, secure)
+        self.use_keycloak_account = use_keycloak_account
         self.keycloak_rest_client = KeycloakRESTClient().configure(
-            conf, secure, refresh=True)
+            conf, secure, refresh=True, use_keycloak_account=self.use_keycloak_account)
         self.__logger = self.__conf.logger("nifi.v2.client")
         self.check_status_sleep_lapse = int(
             self.__conf.check_status_sleep_lapse())
@@ -152,14 +153,21 @@ class NIFIClient:
         """gets the NIFI access token,
         using NIFI account taken from configuration
         this access token is required for any further request sent to NIFI
+        If a taken was provided upon configuration, it gets the latest renewed
         """
+
         if self.__conf.nifi_login() and self.__conf.nifi_passwd():
             return self.nifi_rest_client.get_access_token()
-        elif self.__conf.keycloak_login() and self.__conf.keycloak_passwd():
+        # Keycloak token provided, but not an account
+        elif not self.use_keycloak_account:
             return self.keycloak_rest_client.get_token()
-        else:
+        # Keycloak token not provided, check an account is available
+        elif self.use_keycloak_account and self.__conf.keycloak_login() and \
+            self.__conf.keycloak_passwd():
+            return self.keycloak_rest_client.get_token()
+        else: # Not valid token provider configuration found
             raise HidDataTransferException(
-                "NIFI or Keycloak login and password must be provided"
+                "NIFI or Keycloak account (login and password) or Keycloak token must be provided"
             )
 
     def set_access_token(self, keycloak_token):

{hid_data_transfer_lib-0.3.11 → hid_data_transfer_lib-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "hid_data_transfer_lib"
-version = "0.3.11"
+version = "0.4.0"
 description = "HiDALGO Data Transfer library provides methods to transfer data between different data providers and consumers using NIFI pipelines"
 authors = [
     { name = "Jesús Gorroñogoitia", email = "jesus.gorronogoitia@eviden.com" },
@@ -28,6 +28,7 @@ include = ["py.typed"]
 [tool.poetry.group.dev.dependencies]
 flake8 = "^7.2.0"
 black = "^25.1.0"
+ipykernel = "^7.2.0"
 
 
 [tool.poetry.group.types.dependencies]