hid_data_transfer_lib 0.3.2__tar.gz → 0.3.4__tar.gz

{hid_data_transfer_lib-0.3.2 → hid_data_transfer_lib-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: hid_data_transfer_lib
-Version: 0.3.2
+Version: 0.3.4
 Summary: HiDALGO Data Transfer library provides methods to transfer data between different data providers and consumers using NIFI pipelines
 License: APL-2.0
 Author: Jesús Gorroñogoitia
@@ -29,8 +29,8 @@ This library is planning to support the following features:
 - transfer datasets from/to local filesystem to/from HPC
 - transfer datasets from/to local filesystem to/from CKAN
 
-## Prototype
-Current prototype of the library supports the following features:
+## Current version
+Current version of the library supports the following features:
 - transfer datasets from/to Hadoop HDFS to/from HPC
 - transfer datasets from/to Hadoop HDFS to/from CKAN
 - transfer datasets from/to a CKAN to/from HPC
@@ -41,7 +41,7 @@ Current prototype of the library supports the following features:
 This is a Python library that offers specialized API methods to transfer data from data sources to targets. 
 Each API method launches a NIFI pipeline, by instantiating a NIFI process group out of its workflow definition registered in the NIFI registry. 
 It uses the parameters given within the library method invocation to populate a NIFI parameter context that is asociated to the process group. 
-Then, processors in the process group are executed once (or until the incomining processor's flowfile queue gets empty), one after another, following the group sequence flow, until the flow is completed. 
+Then, processors in the process group are executed once (or forever until the incomining processor's flowfile queue gets empty), one after another, following the group sequence flow, until the flow is completed. 
 A processor is executed after the previous one has terminated. To check the status of the transfer command, the library offers another check-status command. 
 Upon termination, the NIFI environment is cleaned up, by removing the created entities (i.e. the process group and its paramenter context). 
 The Data Transfer Library sends requests to NIFI through its REST API. 
@@ -50,8 +50,9 @@ The Data Transfer Library sends requests to NIFI through its REST API.
 To use the Data Transfer library, it is required the following requirements:
  - **Python3** execution environment
  - **Poetry** python package management tool (optional)
- - **NIFI** instance, and either an NIFI or KEYCLOAK user's account and a NIFI server ssh account
- - **HDFS** instance
+ - **NIFI** instance, with a NIFI server SSH account (for keys transfer)
+ - **Keycloak** instance, with a KEYCLOAK user's account
+ - **HDFS** instance, with an user's Kerberos principal account
  - **CKAN** instance, with an user APIKey
 
  Python3 should be installed in the computer where Data Transfer CLI will be used.
@@ -59,7 +60,7 @@ To use the Data Transfer library, it is required the following requirements:
 
 ## Data Transfer lib configuration
 ### Configuration file
-Before using the Data Transfer library, you should configure it to point at the target NIFI. The configuration file is located, by default, at the *data_transfer_cli/conf/hid_dt.cfg* file. Otherwise, its location can be specified in the environement variable *HID_DT_CONFIG_FILE*
+Before using the Data Transfer library, you should configure it to point at the target NIFI and Keycloak services. The default configuration file is located, by default, at the *data_transfer_cli/conf/hid_dt.cfg* file. However, this default configuration should be complemented (and optionally overriden) with user's specific settings, placed in a configuration file whose location should be also specified (e.g., *~/.dtcli/dtcli.cfg*). Settings in this latter user's configuration will override those in the former default library configuration. The user should not modify the default library configuration, but the the user's specific one, including therein any additional required settings (see below) or modifications to the default ones. The location of this user's specific configuration file is passed as parameter to the library when setting it programmatically.
 
 ```
 [Nifi]
@@ -73,76 +74,119 @@ keycloak_endpoint=https://idm.hidalgo2.eu
 keycloak_client_id=nifi
 keycloak_client_secret=<keycloak_nifi_client_secret>
 
+[Logging]
+logging_level=INFO
+
 [Network]
 check_status_sleep_lapse=5
 ```
+
 Under the NIFI section, 
 - We define the url of the NIFI service (*nifi_endpoint*), 
 - We also specify a folder (*nifi_upload_folder*) in NIFI server where to upload files 
 - And another folder (*nifi_download_folder*) where from to download files. These folder must be accessible by the NIFI service (ask NIFI administrator for details). 
 - Additionally, you cat set if NIFI servers listens on a secure HTTPS connection (*nifi_secure_connection*=True) or on a non-secure HTTP (*nifi_secure_connection*=False)
 
+These default library settings works with the HiDALGO2 NIFI, so not additional modifications are required.
+
 Under the Keycloak section, you can configure the Keycloak integrated with NIFI, specifying:
 - The Keycloak service endpoint (*keycloak_endpoint*)
 - The NIFI client in Keycloak (*keycloak_client*)
-- The NIFI secret in Keycloak (*keycloak_client_secret*)
+- The NIFI secret in Keycloak (*keycloak_client_secret*). This setting must be set in the user's  configuration (e.g., *~/.dtcli/dtcli.cfg*).
+
+These default library settings works with the HiDALGO2 Keycloak, so not additional modifications are required, excepting for the NIFI client secret.
+
+Under the Logging section, you can configure the logging level. Logfile *dtcli.log" is located at the workdir of the process that executes the library.
 
-Under the Network section, you can configure the lapse time (in seconds) each processor in the NIFI pipeline is checked for complation. Most of users should leave the default value.
+Under the Network section, you can configure the lapse time (in seconds) each processor in the NIFI pipeline is checked for completion. Most of users should leave the default value.
+
+Remember that any modification or addition for the default settings must be placed on the user's specific configuration file (e.g., *~/.dtcli/dtcli.cfg*).
 
 HiDALGO2 developers can contact the Keycloak administrator for the *keycloak_client_secret*
 
-### User's accounts in environment variables
+### User's accounts
 
-You must also specify a user account (username, private_key) that grants to upload/download files to the NIFI server (as requested to upload temporary HPC keys or to support local file transfer). This user's account is provided by Hidalgo2 infrastructure provider and it is user's or service's specific. This account is set up in the following environment variables
-- NIFI_SERVER_USERNAME: `export NIFI_SERVER_USERNAME=<nifi_server_username>`
-- NIFI_SERVER_PRIVATE_KEY: `export NIFI_SERVER_PRIVATE_KEY=<path_to_private_key>`
+Additional user's accounts are specified in the user's specific configuration file (e.g., *~/.dtcli/dtcli.cfg*):
 
-Additionally, a user account granted with access to the NIFI service must be specified, either a
+```
+[Nifi]
+nifi_server_username=<user_name>
+nifi_server_private_key=<path/to/private/key>
 
-#### A) NIFI User Account
-The NIFI account must be configured in the following environment variables:
-- NIFI_LOGIN: `export NIFI_LOGIN=<nifi_login>`
-- NIFI_PASSWORD: `export NIFI_PASSWORD=<nifi_password>`
+[Keycloak]
+keycloak_login=<user_name>
+keycloak_password=<password>
+```
 
-This NIFI account is provided by the NIFI administrator. 
+Under the Nifi section, you must also specify a user account (username, private_key) that grants to upload/download files to the NIFI server (as requested to upload temporary HPC keys or to support local file transfer). This user's account is provided by Hidalgo2 infrastructure provider and it is user's or service's specific. 
 
-#### B) Keycloak Account with access to NIFI
-The Keycloak account must be configured in the following environment variables:
-- KEYCLOAK_LOGIN: `export KEYCLOAK_LOGIN=<keycloak_login>`
-- KEYCLOAK_PASSWORD: `export KEYCLOAK_PASSWORD=<keycloak_password>`
+Under the Keycloak section, you must specify your Keycloak account (username and password). This account granted with access to the NIFI service.
 
 For HiDALGO2 developers, NIFI (Service, Server) and Keycloak accounts are provided by the HiDALGO2 administrator.
 
 
 ## Usage
-The data transfer library can be invoked following this procedure:
+The data transfer library can be invoked following two procedures:
 
-- Provide NIFI server and Keycloak accounts in environment variables
+### Using user's configuration (e.g., ~/.dtcli/dtcli.cfg)
+In this case, user's configuration will be read from a give file, such as *~/.dtcli/dtcli.cfg*, whose location is programmatically passed as a parameter upon the setup of the library (see procedure below).
+
+### Providing configuration in a dictionary
+In this case, the user's configuration is provided in a dictionary, with this structure:
 ```
-NIFI_SERVER_USERNAME=<nifi_server_username>
-NIFI_SERVER_PRIVATE_KEY=<path_to_nifi_server_user_private_key>
-KEYCLOAK_LOGIN=<keycloak_username>
-KEYCLOAK_PASSWORD=<keycloak_password>
+{
+  'Nifi': 
+    {
+      'nifi_server_username': '<username>',
+      'nifi_server_private_key': '<path/to/private/key>'
+    }, 
+  'Keycloak': 
+    {
+      'keycloak_login': '<username>', 
+      'keycloak_password': '<password>',
+      'keycloak_client_secret': '<client_secret>'
+    }, 
+  'Logging': {
+    'logging_level': 'DEBUG'}, 
+  'Network': {
+    'check_status_sleep_lapse': '2'
+  }
+}
 ```
-- Customized above hid_dt.cfg and specify its path in the envirorment variable
-`HID_DT_CONFIG_FILE=<path_to_data_transfer_configuration_file`
+In this settings' dictionary you should add the user's specific accounts for Nifi and Keycloak, and optionally, other settings, as shown for the logging level or the sleep lapse time for checking the processors status on the Nifi pipeline. This dictionary is programmatically passed as parameter to the library upon its setup (see procedure below).
+
+The remaining procedure goes as follows:
 
 - In your python code, instantiate a HIDDataTransferConfiguration object and an HIDDataTranfer object
-  The HDIDataTransfer object can be created, by default, using the Keycloak account provided in the environment variables,
+  The HDIDataTransfer object can be created, by default, using the configuration read from the user's configuration file (or from the provided configuration dictionary),
   or by providing a dictionary with the Keycloak token, the refresh token, and the expiration time
 
+Example with user's configuration file
+
 ```
 from hid_data_transfer_lib.hid_dt_lib import HIDDataTransfer
 from hid_data_transfer_lib.conf.hid_dt_configuration import (
     HidDataTransferConfiguration
 )
 
-config = HidDataTransferConfiguration()
-# Create a HIDDataTransfer object that uses the Keycloak account provided in the environment variables
+# Using Keycloak configuration from users's file
+user_config_file = None
+if os.path.exists(os.path.expanduser("~/.dtcli/dtcli.cfg")):
+    user_config_file = os.path.expanduser("~/.dtcli/dtcli.cfg")
+config = HidDataTransferConfiguration().configure(
+    user_config_file=user_config_file,
+    logging_level=logging.DEBUG)
+# Create a HIDDataTransfer object that uses the Keycloak user's configuration
 dt_client = HIDDataTransfer(conf=config, secure=True)
 
 # OR
 
+user_config_file = None
+if os.path.exists(os.path.expanduser("~/.dtcli/dtcli.cfg")):
+    user_config_file = os.path.expanduser("~/.dtcli/dtcli.cfg")
+config = HidDataTransferConfiguration().configure(
+    user_config_file=user_config_file,
+    logging_level=logging.DEBUG)  
 # Create a HIDDataTransfer object that uses the provided Keycloak token dictionary
 keycloak_token = {
   "username": <keycloak_username>,
@@ -156,6 +200,17 @@ dt_client = HIDDataTransfer(
   keycloak_token=keycloak_token
 )
 ```
+
+Example with user's configuration dictionary:
+```
+user_config_dict = {...} # See example of dictionary given above
+config = HidDataTransferConfiguration().configure(
+    user_config_dict=user_config_dict,
+    logging_level=logging.DEBUG)
+dt_client = HIDDataTransfer(conf=config, secure=True)
+```
+
+
 - Invoke any data transfer library method using the created object to tranfer data
 ```
 dt_client.ckan2hpc(

{hid_data_transfer_lib-0.3.2 → hid_data_transfer_lib-0.3.4}/README.md

@@ -11,8 +11,8 @@ This library is planning to support the following features:
 - transfer datasets from/to local filesystem to/from HPC
 - transfer datasets from/to local filesystem to/from CKAN
 
-## Prototype
-Current prototype of the library supports the following features:
+## Current version
+Current version of the library supports the following features:
 - transfer datasets from/to Hadoop HDFS to/from HPC
 - transfer datasets from/to Hadoop HDFS to/from CKAN
 - transfer datasets from/to a CKAN to/from HPC
@@ -23,7 +23,7 @@ Current prototype of the library supports the following features:
 This is a Python library that offers specialized API methods to transfer data from data sources to targets. 
 Each API method launches a NIFI pipeline, by instantiating a NIFI process group out of its workflow definition registered in the NIFI registry. 
 It uses the parameters given within the library method invocation to populate a NIFI parameter context that is asociated to the process group. 
-Then, processors in the process group are executed once (or until the incomining processor's flowfile queue gets empty), one after another, following the group sequence flow, until the flow is completed. 
+Then, processors in the process group are executed once (or forever until the incomining processor's flowfile queue gets empty), one after another, following the group sequence flow, until the flow is completed. 
 A processor is executed after the previous one has terminated. To check the status of the transfer command, the library offers another check-status command. 
 Upon termination, the NIFI environment is cleaned up, by removing the created entities (i.e. the process group and its paramenter context). 
 The Data Transfer Library sends requests to NIFI through its REST API. 
@@ -32,8 +32,9 @@ The Data Transfer Library sends requests to NIFI through its REST API.
 To use the Data Transfer library, it is required the following requirements:
  - **Python3** execution environment
  - **Poetry** python package management tool (optional)
- - **NIFI** instance, and either an NIFI or KEYCLOAK user's account and a NIFI server ssh account
- - **HDFS** instance
+ - **NIFI** instance, with a NIFI server SSH account (for keys transfer)
+ - **Keycloak** instance, with a KEYCLOAK user's account
+ - **HDFS** instance, with an user's Kerberos principal account
  - **CKAN** instance, with an user APIKey
 
  Python3 should be installed in the computer where Data Transfer CLI will be used.
@@ -41,7 +42,7 @@ To use the Data Transfer library, it is required the following requirements:
 
 ## Data Transfer lib configuration
 ### Configuration file
-Before using the Data Transfer library, you should configure it to point at the target NIFI. The configuration file is located, by default, at the *data_transfer_cli/conf/hid_dt.cfg* file. Otherwise, its location can be specified in the environement variable *HID_DT_CONFIG_FILE*
+Before using the Data Transfer library, you should configure it to point at the target NIFI and Keycloak services. The default configuration file is located, by default, at the *data_transfer_cli/conf/hid_dt.cfg* file. However, this default configuration should be complemented (and optionally overriden) with user's specific settings, placed in a configuration file whose location should be also specified (e.g., *~/.dtcli/dtcli.cfg*). Settings in this latter user's configuration will override those in the former default library configuration. The user should not modify the default library configuration, but the the user's specific one, including therein any additional required settings (see below) or modifications to the default ones. The location of this user's specific configuration file is passed as parameter to the library when setting it programmatically.
 
 ```
 [Nifi]
@@ -55,76 +56,119 @@ keycloak_endpoint=https://idm.hidalgo2.eu
 keycloak_client_id=nifi
 keycloak_client_secret=<keycloak_nifi_client_secret>
 
+[Logging]
+logging_level=INFO
+
 [Network]
 check_status_sleep_lapse=5
 ```
+
 Under the NIFI section, 
 - We define the url of the NIFI service (*nifi_endpoint*), 
 - We also specify a folder (*nifi_upload_folder*) in NIFI server where to upload files 
 - And another folder (*nifi_download_folder*) where from to download files. These folder must be accessible by the NIFI service (ask NIFI administrator for details). 
 - Additionally, you cat set if NIFI servers listens on a secure HTTPS connection (*nifi_secure_connection*=True) or on a non-secure HTTP (*nifi_secure_connection*=False)
 
+These default library settings works with the HiDALGO2 NIFI, so not additional modifications are required.
+
 Under the Keycloak section, you can configure the Keycloak integrated with NIFI, specifying:
 - The Keycloak service endpoint (*keycloak_endpoint*)
 - The NIFI client in Keycloak (*keycloak_client*)
-- The NIFI secret in Keycloak (*keycloak_client_secret*)
+- The NIFI secret in Keycloak (*keycloak_client_secret*). This setting must be set in the user's  configuration (e.g., *~/.dtcli/dtcli.cfg*).
+
+These default library settings works with the HiDALGO2 Keycloak, so not additional modifications are required, excepting for the NIFI client secret.
+
+Under the Logging section, you can configure the logging level. Logfile *dtcli.log" is located at the workdir of the process that executes the library.
 
-Under the Network section, you can configure the lapse time (in seconds) each processor in the NIFI pipeline is checked for complation. Most of users should leave the default value.
+Under the Network section, you can configure the lapse time (in seconds) each processor in the NIFI pipeline is checked for completion. Most of users should leave the default value.
+
+Remember that any modification or addition for the default settings must be placed on the user's specific configuration file (e.g., *~/.dtcli/dtcli.cfg*).
 
 HiDALGO2 developers can contact the Keycloak administrator for the *keycloak_client_secret*
 
-### User's accounts in environment variables
+### User's accounts
 
-You must also specify a user account (username, private_key) that grants to upload/download files to the NIFI server (as requested to upload temporary HPC keys or to support local file transfer). This user's account is provided by Hidalgo2 infrastructure provider and it is user's or service's specific. This account is set up in the following environment variables
-- NIFI_SERVER_USERNAME: `export NIFI_SERVER_USERNAME=<nifi_server_username>`
-- NIFI_SERVER_PRIVATE_KEY: `export NIFI_SERVER_PRIVATE_KEY=<path_to_private_key>`
+Additional user's accounts are specified in the user's specific configuration file (e.g., *~/.dtcli/dtcli.cfg*):
 
-Additionally, a user account granted with access to the NIFI service must be specified, either a
+```
+[Nifi]
+nifi_server_username=<user_name>
+nifi_server_private_key=<path/to/private/key>
 
-#### A) NIFI User Account
-The NIFI account must be configured in the following environment variables:
-- NIFI_LOGIN: `export NIFI_LOGIN=<nifi_login>`
-- NIFI_PASSWORD: `export NIFI_PASSWORD=<nifi_password>`
+[Keycloak]
+keycloak_login=<user_name>
+keycloak_password=<password>
+```
 
-This NIFI account is provided by the NIFI administrator. 
+Under the Nifi section, you must also specify a user account (username, private_key) that grants to upload/download files to the NIFI server (as requested to upload temporary HPC keys or to support local file transfer). This user's account is provided by Hidalgo2 infrastructure provider and it is user's or service's specific. 
 
-#### B) Keycloak Account with access to NIFI
-The Keycloak account must be configured in the following environment variables:
-- KEYCLOAK_LOGIN: `export KEYCLOAK_LOGIN=<keycloak_login>`
-- KEYCLOAK_PASSWORD: `export KEYCLOAK_PASSWORD=<keycloak_password>`
+Under the Keycloak section, you must specify your Keycloak account (username and password). This account granted with access to the NIFI service.
 
 For HiDALGO2 developers, NIFI (Service, Server) and Keycloak accounts are provided by the HiDALGO2 administrator.
 
 
 ## Usage
-The data transfer library can be invoked following this procedure:
+The data transfer library can be invoked following two procedures:
 
-- Provide NIFI server and Keycloak accounts in environment variables
+### Using user's configuration (e.g., ~/.dtcli/dtcli.cfg)
+In this case, user's configuration will be read from a give file, such as *~/.dtcli/dtcli.cfg*, whose location is programmatically passed as a parameter upon the setup of the library (see procedure below).
+
+### Providing configuration in a dictionary
+In this case, the user's configuration is provided in a dictionary, with this structure:
 ```
-NIFI_SERVER_USERNAME=<nifi_server_username>
-NIFI_SERVER_PRIVATE_KEY=<path_to_nifi_server_user_private_key>
-KEYCLOAK_LOGIN=<keycloak_username>
-KEYCLOAK_PASSWORD=<keycloak_password>
+{
+  'Nifi': 
+    {
+      'nifi_server_username': '<username>',
+      'nifi_server_private_key': '<path/to/private/key>'
+    }, 
+  'Keycloak': 
+    {
+      'keycloak_login': '<username>', 
+      'keycloak_password': '<password>',
+      'keycloak_client_secret': '<client_secret>'
+    }, 
+  'Logging': {
+    'logging_level': 'DEBUG'}, 
+  'Network': {
+    'check_status_sleep_lapse': '2'
+  }
+}
 ```
-- Customized above hid_dt.cfg and specify its path in the envirorment variable
-`HID_DT_CONFIG_FILE=<path_to_data_transfer_configuration_file`
+In this settings' dictionary you should add the user's specific accounts for Nifi and Keycloak, and optionally, other settings, as shown for the logging level or the sleep lapse time for checking the processors status on the Nifi pipeline. This dictionary is programmatically passed as parameter to the library upon its setup (see procedure below).
+
+The remaining procedure goes as follows:
 
 - In your python code, instantiate a HIDDataTransferConfiguration object and an HIDDataTranfer object
-  The HDIDataTransfer object can be created, by default, using the Keycloak account provided in the environment variables,
+  The HDIDataTransfer object can be created, by default, using the configuration read from the user's configuration file (or from the provided configuration dictionary),
   or by providing a dictionary with the Keycloak token, the refresh token, and the expiration time
 
+Example with user's configuration file
+
 ```
 from hid_data_transfer_lib.hid_dt_lib import HIDDataTransfer
 from hid_data_transfer_lib.conf.hid_dt_configuration import (
     HidDataTransferConfiguration
 )
 
-config = HidDataTransferConfiguration()
-# Create a HIDDataTransfer object that uses the Keycloak account provided in the environment variables
+# Using Keycloak configuration from users's file
+user_config_file = None
+if os.path.exists(os.path.expanduser("~/.dtcli/dtcli.cfg")):
+    user_config_file = os.path.expanduser("~/.dtcli/dtcli.cfg")
+config = HidDataTransferConfiguration().configure(
+    user_config_file=user_config_file,
+    logging_level=logging.DEBUG)
+# Create a HIDDataTransfer object that uses the Keycloak user's configuration
 dt_client = HIDDataTransfer(conf=config, secure=True)
 
 # OR
 
+user_config_file = None
+if os.path.exists(os.path.expanduser("~/.dtcli/dtcli.cfg")):
+    user_config_file = os.path.expanduser("~/.dtcli/dtcli.cfg")
+config = HidDataTransferConfiguration().configure(
+    user_config_file=user_config_file,
+    logging_level=logging.DEBUG)  
 # Create a HIDDataTransfer object that uses the provided Keycloak token dictionary
 keycloak_token = {
   "username": <keycloak_username>,
@@ -138,6 +182,17 @@ dt_client = HIDDataTransfer(
   keycloak_token=keycloak_token
 )
 ```
+
+Example with user's configuration dictionary:
+```
+user_config_dict = {...} # See example of dictionary given above
+config = HidDataTransferConfiguration().configure(
+    user_config_dict=user_config_dict,
+    logging_level=logging.DEBUG)
+dt_client = HIDDataTransfer(conf=config, secure=True)
+```
+
+
 - Invoke any data transfer library method using the created object to tranfer data
 ```
 dt_client.ckan2hpc(

{hid_data_transfer_lib-0.3.2 → hid_data_transfer_lib-0.3.4}/hid_data_transfer_lib/conf/hid_dt_configuration.py

@@ -14,7 +14,7 @@ limitations under the License.
 
 This module manages the configuration of the Data Transfer CLI
 """
-
+from __future__ import annotations
 import configparser
 import os
 import logging
@@ -45,8 +45,8 @@ class HidDataTransferConfiguration:
 
     __check_status_sleep_lapse = 10  # seconds
 
-    def __init__(self, logging_level: Optional[int] = None) -> None:
-        self.__get_configuration(logging_level)
+    def __init__(self, logging_level: Optional[int] = logging.INFO):
+        self.__logging_level = logging_level
 
     def check_keycloak_conf(self):
         """Check if keycloak configuration is valid"""
@@ -210,27 +210,61 @@ class HidDataTransferConfiguration:
             if isinstance(handler, logging.FileHandler):
                 return handler
         return None
-    
+
     def check_status_sleep_lapse(self):
         """Returns the NIFI login name"""
         return self.__check_status_sleep_lapse
 
-    def __get_configuration(self, logging_level: Optional[int]):
+    def read_configuration(
+        self,
+        user_config_file: Optional[str] = None,
+        user_config_dict: Optional[dict] = None
+    ) -> configparser.RawConfigParser:
+        """
+        Reads the configuration file and returns a configparser object.
         """
-        Reads the configuration file and sets this class' configuration values
 
-        Try to read the configuration file
-        defined in the environment variable HID_DT_CONFIG_FILE
-        Otherwise, it reads the configuration file
-        from the default library configuration directory.
+        if user_config_file is not None and user_config_dict is not None:
+            raise HidDataTransferException(
+                (
+                    "Only one of user_config_file or user_config_dict "
+                    "should be provided"
+                )
+            )
+        if user_config_file is None and user_config_dict is None:
+            raise HidDataTransferException(
+                "One of user_config_file or user_config_dict "
+                "should be provided"
+            )
 
-        """
         config = configparser.RawConfigParser()
-        config_file = os.getenv("HID_DT_CONFIG_FILE")
-        if not config_file:
-            config_file = str(os.path.dirname(os.path.realpath(__file__))) + \
-                "/hid_dt.cfg"
+        config_file = str(os.path.dirname(os.path.realpath(__file__))) + \
+            "/hid_dt.cfg"
         config.read(config_file)
+        if user_config_file is not None:
+            config.read(user_config_file)
+        if user_config_dict is not None:
+            config.read_dict(user_config_dict)
+        return config
+
+    def configure(
+        self,
+        user_config_file: Optional[str] = None,
+        user_config_dict: Optional[dict] = None,
+        logging_level: Optional[int] = None
+    ) -> HidDataTransferConfiguration:
+        """
+        Reads the configuration and sets this class' configuration values
+        It reads the configuration
+        from the default library configuration directory.
+        If the user_config_file is provided, this files is read, and its values
+        override those in the default configuration file.
+        If the user_config_dict is provided, this dictionary is read,
+        and its values override those in the default configuration file.
+        Only one of user_config_file or user_config_dict should be provided.
+
+        """
+        config = self.read_configuration(user_config_file, user_config_dict)
 
         try:
             # NIFI section
@@ -257,11 +291,13 @@ class HidDataTransferConfiguration:
                 logging_level_str = config.get("Logging", "logging_level")
                 self.__logging_level = getattr(
                     logging, logging_level_str.upper(), logging.INFO)
-                
+
             # Network section
             self.__check_status_sleep_lapse = config.get(
                 "Network", "check_status_sleep_lapse")
 
+            return self
+
         except configparser.NoSectionError as ex:
             raise HidDataTransferException(
                 "Error parsing CLI configuration file"
@@ -269,9 +305,9 @@ class HidDataTransferConfiguration:
 
     def load_keycloak_configuration(self, config):
         """Load Keycloak configuration from config file"""
-        # Read Keycloak login and password from environment variables
-        self.__keycloak_login = os.getenv("KEYCLOAK_LOGIN")
-        self.__keycloak_passwd = os.getenv("KEYCLOAK_PASSWORD")
+        # Read Keycloak login and password from configuration
+        self.__keycloak_login = config.get("Keycloak", "keycloak_login")
+        self.__keycloak_passwd = config.get("Keycloak", "keycloak_password")
         if (self.__keycloak_login and self.__keycloak_passwd is None) or (
             self.__keycloak_login is None and self.__keycloak_passwd
         ):
@@ -315,17 +351,6 @@ class HidDataTransferConfiguration:
                 "Could not find Nifi endpoint in config file."
             )
 
-            # Read Nifi login and password from environment variables
-        self.__nifi_login = os.getenv("NIFI_LOGIN")
-        self.__nifi_passwd = os.getenv("NIFI_PASSWORD")
-        if (self.__nifi_login and self.__nifi_passwd is None) or (
-            self.__nifi_login is None and self.__nifi_passwd
-        ):
-            raise HidDataTransferException(
-                "Both NIFI_LOGIN and NIFI_PASSWORD "
-                "environment variables must be set"
-            )
-
         nifi_secure_connection = config.get("Nifi", "nifi_secure_connection")
         if nifi_secure_connection is not None:
             self.__nifi_secure_connection = \
@@ -346,9 +371,11 @@ class HidDataTransferConfiguration:
                 "Could not find Nifi nifi_download_folder in config file"
             )
 
-            # Optional NIFI server account in case of local file transfer
-        self.__nifi_server_user_name = os.getenv("NIFI_SERVER_USERNAME")
-        self.__nifi_server_private_key = os.getenv("NIFI_SERVER_PRIVATE_KEY")
+        # Optional NIFI server account in case of local file transfer
+        self.__nifi_server_user_name = config.get(
+            "Nifi", "nifi_server_username")
+        self.__nifi_server_private_key = config.get(
+            "Nifi", "nifi_server_private_key")
 
         if (
             self.__nifi_server_user_name and

{hid_data_transfer_lib-0.3.2 → hid_data_transfer_lib-0.3.4}/hid_data_transfer_lib/hid_dt_lib.py

@@ -65,7 +65,7 @@ class HIDDataTransfer:
         with the NIFI endpoint taken from configuration
         """
         self.__conf = conf
-        self.__nifi_client = NIFIClient(conf, secure)
+        self.__nifi_client = NIFIClient().configure(conf, secure)
         self.__logger = self.__conf.logger("hid_data_transfer_lib")
         self.__keycloak_token = keycloak_token
 

{hid_data_transfer_lib-0.3.2 → hid_data_transfer_lib-0.3.4}/hid_data_transfer_lib/keycloak/keycloak_rest.py

@@ -16,7 +16,7 @@ This module defines the Keycloak API client class.
 It provides methods to interface the Keycloak server to
 authenticate users and retrieve tokens.
 """
-
+from __future__ import annotations
 from typing import Optional
 from urllib.parse import urlparse, urlunparse, quote_plus
 import threading
@@ -58,10 +58,19 @@ class KeycloakRESTClient:
     __expires_in = None
     __refresh_timer = None
 
-    def __init__(
+    def __init__(self):
+        """Constructor method"""
+        self.__conf = None
+        self.__keycloak_login = None
+        self.__keycloak_endpoint = None
+        self.__logger = None
+        self.__refresh = False
+        self.__refresh_timer = None
+
+    def configure(
         self, conf: HidDataTransferConfiguration,
         secure: bool = False, refresh: bool = False
-    ) -> None:
+    ) -> KeycloakRESTClient:
         """constructs a API client,
         with the Keycloak endpoint taken from configuration
         """
@@ -73,6 +82,7 @@ class KeycloakRESTClient:
             self.__conf.keycloak_endpoint(), secure
         )
         self.__logger = self.__conf.logger("keycloak.keycloak_api")
+        return self
 
     def info(self, *args):
         ''' do info logging'''

{hid_data_transfer_lib-0.3.2 → hid_data_transfer_lib-0.3.4}/hid_data_transfer_lib/nifi/nifi_client.py

@@ -17,7 +17,7 @@ This module defines the NIFI API client class.
 It provides methods to interface the NIFI server to in instantiate templates,
 and run processors in a process group.
 """
-
+from __future__ import annotations
 import glob
 import json
 import os
@@ -99,18 +99,27 @@ class NIFIClient:
     CKAN_RESOURCE = "ckan.resource"
     KERBEROS_PASSWORD = "Kerberos Password"
 
-    def __init__(self, conf: HidDataTransferConfiguration,
-                 secure: bool = False) -> None:
+    def __init__(self):
+        """Constructor method"""
+        self.__conf = None
+        self.nifi_rest_client = None
+        self.keycloak_rest_client = None
+        self.__logger = None
+        self.check_status_sleep_lapse = 1
+
+    def configure(self, conf: HidDataTransferConfiguration,
+                  secure: bool = False) -> NIFIClient:
         """constructs a NIFI client,
         with the NIFI endpoint taken from configuration
         """
         self.__conf = conf
-        self.nifi_rest_client = NIFIRESTClient(conf, secure)
-        self.keycloak_rest_client = KeycloakRESTClient(
+        self.nifi_rest_client = NIFIRESTClient().configure(conf, secure)
+        self.keycloak_rest_client = KeycloakRESTClient().configure(
             conf, secure, refresh=True)
         self.__logger = self.__conf.logger("nifi.v2.client")
         self.check_status_sleep_lapse = int(
             self.__conf.check_status_sleep_lapse())
+        return self
 
     def get_access_token(self) -> str:
         """gets the NIFI access token,

{hid_data_transfer_lib-0.3.2 → hid_data_transfer_lib-0.3.4}/hid_data_transfer_lib/nifi/nifi_rest.py

@@ -17,7 +17,7 @@ This module defines the NIFI REST client class.
 It provides methods to interface the NIFI server to in instantiate templates,
 and run processors in a process group.
 """
-
+from __future__ import annotations
 import json
 from typing import Optional
 import uuid
@@ -59,8 +59,13 @@ class NIFIRESTClient:
     __JSON_CONTENT_TYPE = "application/json"
     __TEXT_PLAIN_CONTENT_TYPE = "text/plain"
 
-    def __init__(self, conf: HidDataTransferConfiguration,
-                 secure: bool = False) -> None:
+    def __init__(self):
+        """initializes the NIFI REST client"""
+        self.__conf = None
+        self.__nifi_endpoint = None
+
+    def configure(self, conf: HidDataTransferConfiguration,
+                  secure: bool = False) -> NIFIRESTClient:
         """constructs a NIFI REST client,
         with the NIFI endpoint taken from configuration
         """
@@ -68,6 +73,7 @@ class NIFIRESTClient:
         self.__conf = conf
         self.__nifi_endpoint = parse_base_url(
             self.__conf.nifi_endpoint(), secure)
+        return self
 
     def build_url(self, *paths: str, parameters: Optional[dict] = None) -> str:
         """constructs a NIFI query endpoint,
@@ -723,7 +729,8 @@ class NIFIRESTClient:
             return response.ok
         else:
             raise HidDataTransferException(
-                f"Delete listing request exception: {response.content.decode()}"
+                "Delete listing request exception: "
+                f"{response.content.decode()}"
             )
 
     # MULTI-REQUESTS methods

{hid_data_transfer_lib-0.3.2 → hid_data_transfer_lib-0.3.4}/pyproject.toml

@@ -1,15 +1,17 @@
 [project]
 name = "hid_data_transfer_lib"
-version = "0.3.2"
+version = "0.3.4"
 description = "HiDALGO Data Transfer library provides methods to transfer data between different data providers and consumers using NIFI pipelines"
-authors = [{name="Jesús Gorroñogoitia", email = "jesus.gorronogoitia@eviden.com"}]
+authors = [
+    { name = "Jesús Gorroñogoitia", email = "jesus.gorronogoitia@eviden.com" },
+]
 license = "APL-2.0"
 readme = "README.md"
 requires-python = ">=3.11"
-dependencies= [
+dependencies = [
     "requests>=2.31.0",
     "paramiko>=3.3.1",
-    "setuptools (>=79.0.0,<80.0.0)"
+    "setuptools (>=79.0.0,<80.0.0)",
 ]
 
 [tool.poetry]