data-transfer-cli 0.4.1__tar.gz → 0.5.0__tar.gz

{data_transfer_cli-0.4.1 → data_transfer_cli-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-transfer-cli
-Version: 0.4.1
+Version: 0.5.0
 Summary: HiDALGO Data Transfer CLI provides commands to transfer data between different data providers and consumers using NIFI pipelines
 License: APL-2.0
 Author: Jesús Gorroñogoitia
@@ -12,7 +12,8 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
-Requires-Dist: hid_data_transfer_lib (>=0.4.1)
+Classifier: Programming Language :: Python :: 3.15
+Requires-Dist: hid_data_transfer_lib (>=0.5.0)
 Requires-Dist: paramiko (>=3.3.1)
 Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
 Requires-Dist: requests (>=2.31.0)
@@ -23,8 +24,6 @@ This repository contains the implementation of the Hidalgo2 data transfer tool.
 
 ## Features
 This tool is planning to support the following features:
-- transfer datasets from Cloud Providers to HDFS
-- transfer datasets from Cloud Providers to CKAN
 - transfer datasets from/to Hadoop HDFS to/from HPC
 - transfer datasets from/to Hadoop HDFS to/from CKAN
 - transfer datasets from/to a CKAN to/from HPC
@@ -36,6 +35,8 @@ Current version supports the following features:
 - transfer datasets from/to Hadoop HDFS to/from CKAN
 - transfer datasets from/to a CKAN to/from HPC
 - transfer datasets from/to local filesystem to/from CKAN
+- transfer datasets from Cloud Providers to HDFS
+- transfer datasets from Cloud Providers to CKAN
 
 
 ## Implementation
@@ -172,6 +173,10 @@ positional arguments:
     hpc2ckan            transfer data from HPC to a target CKAN
     local2ckan          transfer data from a local filesystem to a target CKAN
     ckan2local          transfer data from CKAN to a local filesystem
+    meteofrance2ckan    download data from MeteoFrance service to a target CKAN
+    meteofrance2hdfs    download data from MeteoFrance service to a target HDFS
+    ecmwf2ckan          download data from the ECMWF open data service to a target CKAN
+    ecmwf2hdfs          download data from the ECMWF open data service to a target HDFS
 
 options:
   -h, --help            show this help message and exit
@@ -230,6 +235,105 @@ Transfer rate: 0.61 MB/s
 Number of transferred files: 1
 ```
 
+### Data Transfer from cloud providers usage
+
+The Data Transfer CLI tool supports the transfer from MeteoFrance and the ECMWF open data service to either CKAN or HDFS. These commands are and their specific usage are as follows:
+
+#### METEO FRANCE
+Transfer data from Meteo France to CKAN or HDFS. Obtaining data from Meteo France requires that the user obtains an API_KEY for Meteo France's online API (available at [METEO FRANCE – API PORTAL](https://portail-api.meteofrance.fr/web/en/). 
+
+Both options (meteofrance2ckan and meteofrance2hdfs) require the API_KEY and a JSON-formatted configuration file where the data to download needs to be detailed. To get help for one of these commands execute:
+
+`./dtcli meteofrance2ckan -h`
+
+obtaining:
+
+```
+usage: ['meteofrance2ckan', '-h'] meteofrance2ckan [-h] -c CKAN_HOST -a CKAN_API_KEY -o CKAN_ORGANIZATION -d CKAN_DATASET -ma METEOFRANCE_API_KEY -mc METEOFRANCE_CONFIG
+
+options:
+  -h, --help            show this help message and exit
+  -c CKAN_HOST, --ckan-host CKAN_HOST
+                        CKAN host endpoint
+  -a CKAN_API_KEY, --ckan-api-key CKAN_API_KEY
+                        CKAN API key
+  -o CKAN_ORGANIZATION, --ckan-organization CKAN_ORGANIZATION
+                        Identifier of the CKAN organization that hosts the dataset resource to transfer. Could be identified by the organization id, name or title
+  -d CKAN_DATASET, --ckan-dataset CKAN_DATASET
+                        Identifier of the CKAN dataset that hosts the resource to transfer. Could be identified by the dataset id, name or title
+  -ma METEOFRANCE_APIKEY, --meteofrance-api-key METEOFRANCE_API_KEY
+                        MeteoFrance API key
+  -mc METEOFRANCE_CONFIG, --meteofrance-api-key METEOFRANCE_CONFIG
+                        MeteoFrance JSON configuration file
+```
+
+The configuration file has the following structure:
+
+```
+{
+    "url": "https://public-api.meteofrance.fr/public/arpege/1.0/wcs/MF-NWP-GLOBAL-ARPEGE-01-EUROPE-WCS/GetCoverage?service=WCS&version=2.0.1&format=image/tiff&REQUEST=GetCoverage",
+    "coverages": [
+        {
+        "coverageid": "TOTAL_PRECIPITATION__GROUND_OR_WATER_SURFACE___2026-06-04T00:00:00Z_PT1H",
+        "subsets": ["lat(34.50,66)", "long(-11,42)", "time(2026-06-04T06:00:00Z)"]},
+        {
+        "coverageid": "TOTAL_PRECIPITATION__GROUND_OR_WATER_SURFACE___2026-06-04T00:00:00Z_PT1H",
+        "subsets": ["lat(34.50,66)", "long(-11,42)", "time(2026-06-04T07:00:00Z)"]}
+        ]
+}
+```
+
+It requires the URL of the API that belongs to the target model and then a list of the coverages to obtain. These coverages will need the ID of the parameter as available in the API and the corresponding subsets. A detailed description of the coverageids available and their corresponding subsets is available on the API documentation.
+
+#### ECMWF
+Transfer data from the ECMWF model to CKAN or HDFS. Obtaining data from [ECMWF open data services](https://www.ecmwf.int/en/forecasts/datasets/open-data) requires that the user creates a Python script that will connect to the service and download the data specified in it. As per the [ECMWF-opendata documentation](https://github.com/ecmwf/ecmwf-opendata)
+
+Here is an example of a simple script:
+
+```
+from ecmwf.opendata import Client
+
+client = Client(source="google", beta=False)
+client.retrieve(
+    time=0,
+    stream="oper",
+    type="fc",
+    step=24,
+    param="2t",
+    target="ecmwf_script.grib",
+)
+```
+
+Note that it is necessary to ensure that the *target* file name is the same as the one provided as **CKAN_RESOURCE** a parameter when using the command and also that the python script's name is something other than `ecmwf.py`
+
+See the help of one of these commands:
+
+`./dtcli ecmwf2ckan -h`
+
+obtaining:
+
+```
+usage: ['ecmwf2ckan', '-h'] ecmwf2ckan [-h] -c CKAN_HOST -a CKAN_API_KEY -o CKAN_ORGANIZATION -d CKAN_DATASET -r CKAN_RESOURCE -s ECMWF_SCRIPT
+
+options:
+  -h, --help            show this help message and exit
+  -c CKAN_HOST, --ckan-host CKAN_HOST
+                        CKAN host endpoint
+  -a CKAN_API_KEY, --ckan-api-key CKAN_API_KEY
+                        CKAN API key
+  -o CKAN_ORGANIZATION, --ckan-organization CKAN_ORGANIZATION
+                        Identifier of the CKAN organization that hosts the dataset resource to transfer. Could be identified by the organization id, name or title
+  -d CKAN_DATASET, --ckan-dataset CKAN_DATASET
+                        Identifier of the CKAN dataset that hosts the resource to transfer. Could be identified by the dataset id, name or title
+  -r CKAN_RESOURCE, --ckan-resource CKAN_RESOURCE
+                        CKAN resource to create from transferred sources. If omitted, target resource name will adopt the source file or folder name
+  -s ECMWF_SCRIPT, --ecmwf-script ECMWF_SCRIPT
+                        Path to ECMWF open data service python script
+```
+
+
+
+
 ## Support for HPC clusters that require a 2FA token
 The Data Transfer CLI tool's commands support transferring data to/from HPC clusters that require a 2FA token. These commands offer an optional flag *_2fa*. If set by the user, the command prompts the user (in the standard input) for the token when required. 
 

{data_transfer_cli-0.4.1 → data_transfer_cli-0.5.0}/README.md

@@ -3,8 +3,6 @@ This repository contains the implementation of the Hidalgo2 data transfer tool.
 
 ## Features
 This tool is planning to support the following features:
-- transfer datasets from Cloud Providers to HDFS
-- transfer datasets from Cloud Providers to CKAN
 - transfer datasets from/to Hadoop HDFS to/from HPC
 - transfer datasets from/to Hadoop HDFS to/from CKAN
 - transfer datasets from/to a CKAN to/from HPC
@@ -16,6 +14,8 @@ Current version supports the following features:
 - transfer datasets from/to Hadoop HDFS to/from CKAN
 - transfer datasets from/to a CKAN to/from HPC
 - transfer datasets from/to local filesystem to/from CKAN
+- transfer datasets from Cloud Providers to HDFS
+- transfer datasets from Cloud Providers to CKAN
 
 
 ## Implementation
@@ -152,6 +152,10 @@ positional arguments:
     hpc2ckan            transfer data from HPC to a target CKAN
     local2ckan          transfer data from a local filesystem to a target CKAN
     ckan2local          transfer data from CKAN to a local filesystem
+    meteofrance2ckan    download data from MeteoFrance service to a target CKAN
+    meteofrance2hdfs    download data from MeteoFrance service to a target HDFS
+    ecmwf2ckan          download data from the ECMWF open data service to a target CKAN
+    ecmwf2hdfs          download data from the ECMWF open data service to a target HDFS
 
 options:
   -h, --help            show this help message and exit
@@ -210,6 +214,105 @@ Transfer rate: 0.61 MB/s
 Number of transferred files: 1
 ```
 
+### Data Transfer from cloud providers usage
+
+The Data Transfer CLI tool supports the transfer from MeteoFrance and the ECMWF open data service to either CKAN or HDFS. These commands are and their specific usage are as follows:
+
+#### METEO FRANCE
+Transfer data from Meteo France to CKAN or HDFS. Obtaining data from Meteo France requires that the user obtains an API_KEY for Meteo France's online API (available at [METEO FRANCE – API PORTAL](https://portail-api.meteofrance.fr/web/en/). 
+
+Both options (meteofrance2ckan and meteofrance2hdfs) require the API_KEY and a JSON-formatted configuration file where the data to download needs to be detailed. To get help for one of these commands execute:
+
+`./dtcli meteofrance2ckan -h`
+
+obtaining:
+
+```
+usage: ['meteofrance2ckan', '-h'] meteofrance2ckan [-h] -c CKAN_HOST -a CKAN_API_KEY -o CKAN_ORGANIZATION -d CKAN_DATASET -ma METEOFRANCE_API_KEY -mc METEOFRANCE_CONFIG
+
+options:
+  -h, --help            show this help message and exit
+  -c CKAN_HOST, --ckan-host CKAN_HOST
+                        CKAN host endpoint
+  -a CKAN_API_KEY, --ckan-api-key CKAN_API_KEY
+                        CKAN API key
+  -o CKAN_ORGANIZATION, --ckan-organization CKAN_ORGANIZATION
+                        Identifier of the CKAN organization that hosts the dataset resource to transfer. Could be identified by the organization id, name or title
+  -d CKAN_DATASET, --ckan-dataset CKAN_DATASET
+                        Identifier of the CKAN dataset that hosts the resource to transfer. Could be identified by the dataset id, name or title
+  -ma METEOFRANCE_APIKEY, --meteofrance-api-key METEOFRANCE_API_KEY
+                        MeteoFrance API key
+  -mc METEOFRANCE_CONFIG, --meteofrance-api-key METEOFRANCE_CONFIG
+                        MeteoFrance JSON configuration file
+```
+
+The configuration file has the following structure:
+
+```
+{
+    "url": "https://public-api.meteofrance.fr/public/arpege/1.0/wcs/MF-NWP-GLOBAL-ARPEGE-01-EUROPE-WCS/GetCoverage?service=WCS&version=2.0.1&format=image/tiff&REQUEST=GetCoverage",
+    "coverages": [
+        {
+        "coverageid": "TOTAL_PRECIPITATION__GROUND_OR_WATER_SURFACE___2026-06-04T00:00:00Z_PT1H",
+        "subsets": ["lat(34.50,66)", "long(-11,42)", "time(2026-06-04T06:00:00Z)"]},
+        {
+        "coverageid": "TOTAL_PRECIPITATION__GROUND_OR_WATER_SURFACE___2026-06-04T00:00:00Z_PT1H",
+        "subsets": ["lat(34.50,66)", "long(-11,42)", "time(2026-06-04T07:00:00Z)"]}
+        ]
+}
+```
+
+It requires the URL of the API that belongs to the target model and then a list of the coverages to obtain. These coverages will need the ID of the parameter as available in the API and the corresponding subsets. A detailed description of the coverageids available and their corresponding subsets is available on the API documentation.
+
+#### ECMWF
+Transfer data from the ECMWF model to CKAN or HDFS. Obtaining data from [ECMWF open data services](https://www.ecmwf.int/en/forecasts/datasets/open-data) requires that the user creates a Python script that will connect to the service and download the data specified in it. As per the [ECMWF-opendata documentation](https://github.com/ecmwf/ecmwf-opendata)
+
+Here is an example of a simple script:
+
+```
+from ecmwf.opendata import Client
+
+client = Client(source="google", beta=False)
+client.retrieve(
+    time=0,
+    stream="oper",
+    type="fc",
+    step=24,
+    param="2t",
+    target="ecmwf_script.grib",
+)
+```
+
+Note that it is necessary to ensure that the *target* file name is the same as the one provided as **CKAN_RESOURCE** a parameter when using the command and also that the python script's name is something other than `ecmwf.py`
+
+See the help of one of these commands:
+
+`./dtcli ecmwf2ckan -h`
+
+obtaining:
+
+```
+usage: ['ecmwf2ckan', '-h'] ecmwf2ckan [-h] -c CKAN_HOST -a CKAN_API_KEY -o CKAN_ORGANIZATION -d CKAN_DATASET -r CKAN_RESOURCE -s ECMWF_SCRIPT
+
+options:
+  -h, --help            show this help message and exit
+  -c CKAN_HOST, --ckan-host CKAN_HOST
+                        CKAN host endpoint
+  -a CKAN_API_KEY, --ckan-api-key CKAN_API_KEY
+                        CKAN API key
+  -o CKAN_ORGANIZATION, --ckan-organization CKAN_ORGANIZATION
+                        Identifier of the CKAN organization that hosts the dataset resource to transfer. Could be identified by the organization id, name or title
+  -d CKAN_DATASET, --ckan-dataset CKAN_DATASET
+                        Identifier of the CKAN dataset that hosts the resource to transfer. Could be identified by the dataset id, name or title
+  -r CKAN_RESOURCE, --ckan-resource CKAN_RESOURCE
+                        CKAN resource to create from transferred sources. If omitted, target resource name will adopt the source file or folder name
+  -s ECMWF_SCRIPT, --ecmwf-script ECMWF_SCRIPT
+                        Path to ECMWF open data service python script
+```
+
+
+
+
 ## Support for HPC clusters that require a 2FA token
 The Data Transfer CLI tool's commands support transferring data to/from HPC clusters that require a 2FA token. These commands offer an optional flag *_2fa*. If set by the user, the command prompts the user (in the standard input) for the token when required. 
 

{data_transfer_cli-0.4.1 → data_transfer_cli-0.5.0}/data_transfer_cli/dtproxy.py

@@ -63,6 +63,11 @@ class DataTransferProxy:
                     f"data_target={args.data_target},"
                     if hasattr(args, "data_target")
                     else ""
+                ),
+                                (
+                    f"data_script={args.data_script},"
+                    if hasattr(args, "data_script")
+                    else ""
                 ),
                 f"hpc_host={args.hpc_host}," if hasattr(args, "hpc_host") else "",
                 f"hpc_port={args.hpc_port}," if hasattr(args, "hpc_port") else "",
@@ -333,6 +338,86 @@ class DataTransferProxy:
         except Exception as ex:
             raise HidDataTransferException(ex) from ex
 
+    def meteofrance2ckan(self, args) -> tuple[str, AccountingInfo]:
+        """transfer data from meteofrance to CKAN"""
+        self.__logger.info(
+            "executing meteofrance2ckan command with args: %s",
+            self.format_args_to_string(args),
+        )
+
+        try:
+            return self.dt_client.meteofrance2ckan(
+                ckan_host=args.ckan_host
+                if args.ckan_host.startswith("https") else f"https://{args.ckan_host}",
+                ckan_api_key=args.ckan_api_key,
+                ckan_organization=args.ckan_organization,
+                ckan_dataset=args.ckan_dataset,
+                meteofrance_apikey=args.meteofrance_apikey,
+                meteofrance_config=args.meteofrance_config,
+            )
+
+        except Exception as ex:
+            raise HidDataTransferException(ex) from ex
+
+    def meteofrance2hdfs(self, args) -> tuple[str, AccountingInfo]:
+        """transfer data from meteofrance to HDFS"""
+        self.__logger.info(
+            "executing meteofrance2hdfs command with args: %s",
+            self.format_args_to_string(args),
+        )
+
+        try:
+            return self.dt_client.meteofrance2hdfs(
+                data_target=args.data_target,
+                kerberos_principal=args.kerberos_principal,
+                kerberos_password=args.kerberos_password,
+                meteofrance_apikey=args.meteofrance_apikey,
+                meteofrance_config=args.meteofrance_config,
+            )
+
+        except Exception as ex:
+            raise HidDataTransferException(ex) from ex
+
+    def ecmwf2ckan(self, args) -> tuple[str, AccountingInfo]:
+        """transfer data from ECMWF to CKAN"""
+        self.__logger.info(
+            "executing ecmwf2ckan command with args: %s",
+            self.format_args_to_string(args),
+        )
+
+        try:
+            return self.dt_client.ecmwf2ckan(
+                data_script=args.data_script,
+                ckan_host=args.ckan_host
+                if args.ckan_host.startswith("https") else f"https://{args.ckan_host}",
+                ckan_api_key=args.ckan_api_key,
+                ckan_organization=args.ckan_organization,
+                ckan_dataset=args.ckan_dataset,
+                ckan_resource=args.ckan_resource,
+            )
+
+        except Exception as ex:
+            raise HidDataTransferException(ex) from ex
+
+    def ecmwf2hdfs(self, args) -> tuple[str, AccountingInfo]:
+        """transfer data from ECMWF to HDFS"""
+        self.__logger.info(
+            "executing ecmwf2hdfs command with args: %s",
+            self.format_args_to_string(args),
+        )
+
+        try:
+            return self.dt_client.ecmwf2hdfs(
+                data_script=args.data_script,
+                data_target=args.data_target,
+                data_source=args.data_source,
+                kerberos_principal=args.kerberos_principal,
+                kerberos_password=args.kerberos_password,
+            )
+
+        except Exception as ex:
+            raise HidDataTransferException(ex) from ex
+
     def check_command_status(self, args):
         """Checks the status of a CLI command by ID"""
         # Check process group state by id

{data_transfer_cli-0.4.1 → data_transfer_cli-0.5.0}/data_transfer_cli/parser/cli_parser.py

@@ -178,6 +178,32 @@ class CLIParser(argparse.ArgumentParser):
         )
         return parser
 
+    def add_default_meteofrance_arguments(self, parser):
+        """Add default MeteoFrance arguments to the parser"""
+        parser.add_argument(
+            "-ma",
+            "--meteofrance-apikey",
+            required=True,
+            help="MeteoFrance API key",
+        )
+        parser.add_argument(
+            "-mc",
+            "--meteofrance-config",
+            required=True,
+            help="MeteoFrance JSON configuration file",
+        )
+        return parser
+
+    def add_default_ecmwf_arguments(self, parser):
+        """Add default ECMWF arguments to the parser"""
+        parser.add_argument(
+            "-s",
+            "--data-script",
+            required=True,
+            help="ECMWF data download script full path",
+        )
+        return parser
+
     def parse_arguments(self, args, target):
         """parse the command line arguments
 
@@ -479,4 +505,83 @@ class CLIParser(argparse.ArgumentParser):
         )
         ckan2local_parser.set_defaults(func=target.ckan2local)
 
+        # meteofrance2ckan
+        meteofrance2ckan_parser = commands_parsers.add_parser(
+            "meteofrance2ckan", help="transfer data from MeteoFrance to a target CKAN"
+        )
+        meteofrance2ckan_parser = self.add_default_meteofrance_arguments(meteofrance2ckan_parser)
+        meteofrance2ckan_parser = self.add_default_ckan_arguments(meteofrance2ckan_parser)
+
+        meteofrance2ckan_parser.add_argument(
+            "-acct", "--accounting",
+            required=False, action="store_true", default=False,
+            help="[Optional] Enable returning accounting information of data transfer"
+        )
+
+        meteofrance2ckan_parser.set_defaults(func=target.meteofrance2ckan)
+        
+        #meteofrance2hdfs
+        meteofrance2hdfs_parser = commands_parsers.add_parser(
+            "meteofrance2hdfs", help="transfer data from MeteoFrance to target HDFS"
+        )
+        
+        meteofrance2hdfs_parser.add_argument(
+            "-t", "--data-target", required=True, help="HDFS folder"
+        )
+        meteofrance2hdfs_parser = self.add_default_kerberos_arguments(meteofrance2hdfs_parser)
+        meteofrance2hdfs_parser = self.add_default_meteofrance_arguments(meteofrance2hdfs_parser)
+        meteofrance2hdfs_parser.add_argument(
+            "-acct", "--accounting",
+            required=False, action="store_true", default=False,
+            help="[Optional] Enable returning accounting information of data transfer"
+        )
+
+        meteofrance2hdfs_parser.set_defaults(func=target.meteofrance2hdfs)
+
+        # ecmwf2ckan
+        ecmwf2ckan_parser = commands_parsers.add_parser(
+            "ecmwf2ckan", help="transfer data from ECMWF to a target CKAN"
+        )
+        ecmwf2ckan_parser = self.add_default_ecmwf_arguments(ecmwf2ckan_parser)
+        ecmwf2ckan_parser = self.add_default_ckan_arguments(ecmwf2ckan_parser)
+        ecmwf2ckan_parser.add_argument(
+            "-r",
+            "--ckan-resource",
+            required=False,
+            help="CKAN resource to create from transferred sources. \
+            It has to be the same name as the one configured in the download script.",
+        )
+        ecmwf2ckan_parser.add_argument(
+            "-acct", "--accounting",
+            required=False, action="store_true", default=False,
+            help="[Optional] Enable returning accounting information of data transfer"
+        )
+
+        ecmwf2ckan_parser.set_defaults(func=target.ecmwf2ckan)
+
+        # ecmwf2hdfs
+        ecmwf2hdfs_parser = commands_parsers.add_parser(
+            "ecmwf2hdfs", help="transfer data from ECMWF to a target HDFS"
+        )
+        ecmwf2hdfs_parser = self.add_default_ecmwf_arguments(ecmwf2hdfs_parser)
+        ecmwf2hdfs_parser.add_argument(
+            "-t", "--data-target", required=True, help="HDFS folder"
+        )
+        ecmwf2hdfs_parser.add_argument(
+            "-r",
+            "--data-source",
+            required=False,
+            help="Filename of the downloaded data. \
+            It has to be the same name as the one configured in the download script.",
+        )
+        ecmwf2hdfs_parser = self.add_default_kerberos_arguments(ecmwf2hdfs_parser)
+
+        ecmwf2hdfs_parser.add_argument(
+            "-acct", "--accounting",
+            required=False, action="store_true", default=False,
+            help="[Optional] Enable returning accounting information of data transfer"
+        )
+
+        ecmwf2hdfs_parser.set_defaults(func=target.ecmwf2hdfs)
+
         return self.parse_args(args)

{data_transfer_cli-0.4.1 → data_transfer_cli-0.5.0}/pyproject.toml

@@ -1,9 +1,10 @@
 [project]
 name = "data-transfer-cli"
-version = "0.4.1"
+version = "0.5.0"
 description = "HiDALGO Data Transfer CLI provides commands to transfer data between different data providers and consumers using NIFI pipelines"
 authors = [
     { name = "Jesús Gorroñogoitia", email = "jesus.gorronogoitia@eviden.com" },
+    { name = "Angela Rivera", email = "angela@meteogrid.com" },
 ]
 license = "APL-2.0"
 readme = "README.md"
@@ -11,9 +12,9 @@ requires-python = ">=3.11, <4.0"
 dependencies = [
     "requests>=2.31.0",
     "paramiko>=3.3.1",
-    "hid_data_transfer_lib>=0.4.1",
+    "hid_data_transfer_lib>=0.5.0",
     "pyyaml (>=6.0.2,<7.0.0)",
-    # "hid-data-transfer-lib @ file:///home/yosu/Projects/Hidalgo2/git/hid-data-management/data-transfer/nifi/hid_data_transfer_lib/dist/hid_data_transfer_lib-0.3.11-py3-none-any.whl",
+    # "hid-data-transfer-lib @ file:///home/yosu/Projects/Hidalgo2/git/hid-data-management/data-transfer/nifi/hid_data_transfer_lib",
 ]
 
 [tool.poetry]
@@ -25,7 +26,7 @@ include = ["dtcli", "setenv.sh"]
 
 [tool.poetry.dependencies]
 # Comment out when using the version installed in Pypi
-# hid_data_transfer_lib = { path = "../hid_data_transfer_lib/dist/hid_data_transfer_lib-0.3.2-py3-none-any.whl", develop = true }
+# hid_data_transfer_lib = { path = "../hid_data_transfer_lib", develop = true }
 
 [tool.poetry.group.dev.dependencies]
 flake8 = "^6.1.0"
@@ -33,6 +34,7 @@ black = "^23.9.1"
 pytest = "^7.4.2"
 pytest-cov = "^5.0.0"
 pytest-env = "^1.1.3"
+python-dotenv = "^1.2.4"
 
 [tool.poetry.group.types.dependencies]
 types-paramiko = "^3.4.0.20240311"