data_pipeline_api 0.7.5__tar.gz → 0.7.7__tar.gz

{data_pipeline_api-0.7.5 → data_pipeline_api-0.7.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: data-pipeline-api
-Version: 0.7.5
+Version: 0.7.7
 Summary: Python api to interact with the Fair Data Pipeline
 Home-page: https://www.fairdatapipeline.org/
 License: GNU General Public License v3.0
@@ -56,9 +56,10 @@ Assume FDP_CONFIG_DIR, storage_locations and objects have been set by CLI tool
 import os
 import fairdatapipeline as pipeline
 
-token = os.environ.get('FDP_LOCAL_TOKEN')
-config_path = os.environ.get('FDP_CONFIG_DIR') + '/config.yaml'
-script_path = os.environ.get('FDP_CONFIG_DIR') + '/script.sh'
+token = os.environ['FDP_LOCAL_TOKEN']
+config_dir = os.environ['FDP_CONFIG_DIR']
+config_path = os.path.join(config_dir, 'config.yaml')
+script_path = os.path.join(config_dir, 'script.sh')
 
 handle = pipeline.initialise(token, config_path, script_path)
 

{data_pipeline_api-0.7.5 → data_pipeline_api-0.7.7}/README.md

@@ -33,9 +33,10 @@ Assume FDP_CONFIG_DIR, storage_locations and objects have been set by CLI tool
 import os
 import fairdatapipeline as pipeline
 
-token = os.environ.get('FDP_LOCAL_TOKEN')
-config_path = os.environ.get('FDP_CONFIG_DIR') + '/config.yaml'
-script_path = os.environ.get('FDP_CONFIG_DIR') + '/script.sh'
+token = os.environ['FDP_LOCAL_TOKEN']
+config_dir = os.environ['FDP_CONFIG_DIR']
+config_path = os.path.join(config_dir, 'config.yaml')
+script_path = os.path.join(config_dir, 'script.sh')
 
 handle = pipeline.initialise(token, config_path, script_path)
 

{data_pipeline_api-0.7.5 → data_pipeline_api-0.7.7}/data_pipeline_api/fdp_utils.py

@@ -225,10 +225,30 @@ def post_storage_root(
     """
     if "local" in data and data["local"]:
         data["root"] = "file://" + data["root"]
-    if data["root"][-1] != "/":
+        if data["root"][-1] != os.sep:
+            data["root"] = data["root"] + os.sep
+    elif data["root"][-1] != "/":
         data["root"] = data["root"] + "/"
     return post_entry(url, "storage_root", data, token, api_version)
 
+def post_file_type(
+    url: str, data: dict, token: str, api_version: str = "1.0.0"
+) -> dict:
+    """
+    Internal wrapper function to return check if a file_type already exists and return it.
+    """
+    if not "extension" in data and data["extension"]:
+        raise ValueError("error file_type name not specified")
+    file_type_exists = get_entry(url= url, 
+        endpoint= "file_type", 
+        query= {"extension": data["extension"]}, 
+        api_version= api_version)
+    if file_type_exists : return file_type_exists[0]
+    return post_entry(url= url,
+        endpoint= "file_type",
+        data= data,
+        token= token,
+        api_version= api_version)
 
 def remove_local_from_root(root: str) -> str:
     """

{data_pipeline_api-0.7.5 → data_pipeline_api-0.7.7}/data_pipeline_api/pipeline.py

@@ -82,11 +82,10 @@ def initialise(token: str, config: str, script: str) -> dict:
     config_location_url = config_location_response["url"]
 
     # Configure Yaml File Type
-    config_filetype_response = fdp_utils.post_entry(
+    config_filetype_response = fdp_utils.post_file_type(
         token=token,
         url=registry_url,
-        endpoint="file_type",
-        data={"name": "yaml", "extension": "yaml"},
+        data={"name": "YAML Document", "extension": "yaml"},
         api_version=api_version,
     )
     config_filetype_url = config_filetype_response["url"]
@@ -174,14 +173,14 @@ def initialise(token: str, config: str, script: str) -> dict:
     )
 
     script_location_url = script_location_response["url"]
+    script_file_type = os.path.basename(script).split(".")[-1]
 
     # TODO: Change to Batch?
     # Create Script File Type
-    script_filetype_response = fdp_utils.post_entry(
+    script_filetype_response = fdp_utils.post_file_type(
         token=token,
         url=registry_url,
-        endpoint="file_type",
-        data={"name": "py", "extension": "py"},
+        data={"name": "python submission script", "extension": script_file_type},
         api_version=api_version,
     )
 
@@ -478,23 +477,11 @@ def finalise(token: str, handle: dict) -> None:
 
             file_type = os.path.basename(new_path).split(".")[-1]
 
-            file_type_exists = fdp_utils.get_entry(
+            file_type_url = fdp_utils.post_file_type(
+                token=token,
                 url=registry_url,
-                endpoint="file_type",
-                query={"extension": file_type},
+                data={"name": file_type, "extension": file_type},
                 api_version=api_version,
-            )
-
-            if file_type_exists:
-                entry = fdp_utils.get_first_entry(file_type_exists)
-                file_type_url = entry["url"]
-            else:
-                file_type_url = fdp_utils.post_entry(
-                    token=token,
-                    url=registry_url,
-                    endpoint="file_type",
-                    data={"name": file_type, "extension": file_type},
-                    api_version=api_version,
                 )["url"]
 
             data_product_exists = fdp_utils.get_entry(

{data_pipeline_api-0.7.5 → data_pipeline_api-0.7.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "data_pipeline_api"
-version = "0.7.5"
+version = "0.7.7"
 description = "Python api to interact with the Fair Data Pipeline"
 authors = [
     "Ryan J Field <ryan.field@glasgow.ac.uk>",
@@ -77,7 +77,7 @@ line_length = 79
 line-length = 79
 
 [tool.pytest.ini_options]
-addopts = '-s -v --cov=fairdatapipeline  --cov-report=html --cov-report=term'
+addopts = '-s -v --cov=data_pipeline_api  --cov-report=html --cov-report=term'
 markers = [
     "pipeline: tests for 'pipeline' module ",
     "issue: tests for raising issues ",

{data_pipeline_api-0.7.5 → data_pipeline_api-0.7.7}/setup.py

@@ -16,9 +16,9 @@ install_requires = \
 
 setup_kwargs = {
     'name': 'data-pipeline-api',
-    'version': '0.7.5',
+    'version': '0.7.7',
     'description': 'Python api to interact with the Fair Data Pipeline',
-    'long_description': "# pyDataPipeline\n\n[![pyDataPipeline](https://github.com/FAIRDataPipeline/pyDataPipeline/actions/workflows/pyDataPipeline.yaml/badge.svg?branch=dev)](https://github.com/FAIRDataPipeline/pyDataPipeline/actions/workflows/pyDataPipeline.yaml)\n[![codecov](https://codecov.io/gh/FAIRDataPipeline/pyDataPipeline/branch/dev/graph/badge.svg?token=Eax5AmrDxx)](https://codecov.io/gh/FAIRDataPipeline/pyDataPipeline)\n[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.5562602.svg)](https://doi.org/10.5281/zenodo.5562602)\n[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/5461/badge)](https://bestpractices.coreinfrastructure.org/projects/5461)\n\nWelcome to pyDataPipeline a Python api to interact with the Fair Data Pipeline.\n\nFull documention of the pyDataPipeline is avaialable at [https://www.fairdatapipeline.org/pyDataPipeline/](https://www.fairdatapipeline.org/pyDataPipeline/)\n\n## Installation\npyDataPipeline can be installed from PyPi:\n```\npip3 install data-pipeline-api\n```\n\nOr from the Repository:\n```\ngit clone https://github.com/FAIRDataPipeline/pythonFDP.git\n\ngit checkout dev\n\npip3 install -e .\n```\n**NB. PyDataPipeline requires Python3.**\n\n## Example submission_script\n\nAssume FDP_CONFIG_DIR, storage_locations and objects have been set by CLI tool\n\n```\nimport os\nimport fairdatapipeline as pipeline\n\ntoken = os.environ.get('FDP_LOCAL_TOKEN')\nconfig_path = os.environ.get('FDP_CONFIG_DIR') + '/config.yaml'\nscript_path = os.environ.get('FDP_CONFIG_DIR') + '/script.sh'\n\nhandle = pipeline.initialise(token, config_path, script_path)\n\npipeline.finalise(token, handle)\n\n```\n\n## SEIRS Model Example\n\nThe SEIRS Model Example is available at: [https://www.fairdatapipeline.org/pyDataPipeline/examples/SEIRS.html](https://www.fairdatapipeline.org/pyDataPipeline/examples/SEIRS.html)\n",
+    'long_description': "# pyDataPipeline\n\n[![pyDataPipeline](https://github.com/FAIRDataPipeline/pyDataPipeline/actions/workflows/pyDataPipeline.yaml/badge.svg?branch=dev)](https://github.com/FAIRDataPipeline/pyDataPipeline/actions/workflows/pyDataPipeline.yaml)\n[![codecov](https://codecov.io/gh/FAIRDataPipeline/pyDataPipeline/branch/dev/graph/badge.svg?token=Eax5AmrDxx)](https://codecov.io/gh/FAIRDataPipeline/pyDataPipeline)\n[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.5562602.svg)](https://doi.org/10.5281/zenodo.5562602)\n[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/5461/badge)](https://bestpractices.coreinfrastructure.org/projects/5461)\n\nWelcome to pyDataPipeline a Python api to interact with the Fair Data Pipeline.\n\nFull documention of the pyDataPipeline is avaialable at [https://www.fairdatapipeline.org/pyDataPipeline/](https://www.fairdatapipeline.org/pyDataPipeline/)\n\n## Installation\npyDataPipeline can be installed from PyPi:\n```\npip3 install data-pipeline-api\n```\n\nOr from the Repository:\n```\ngit clone https://github.com/FAIRDataPipeline/pythonFDP.git\n\ngit checkout dev\n\npip3 install -e .\n```\n**NB. PyDataPipeline requires Python3.**\n\n## Example submission_script\n\nAssume FDP_CONFIG_DIR, storage_locations and objects have been set by CLI tool\n\n```\nimport os\nimport fairdatapipeline as pipeline\n\ntoken = os.environ['FDP_LOCAL_TOKEN']\nconfig_dir = os.environ['FDP_CONFIG_DIR']\nconfig_path = os.path.join(config_dir, 'config.yaml')\nscript_path = os.path.join(config_dir, 'script.sh')\n\nhandle = pipeline.initialise(token, config_path, script_path)\n\npipeline.finalise(token, handle)\n\n```\n\n## SEIRS Model Example\n\nThe SEIRS Model Example is available at: [https://www.fairdatapipeline.org/pyDataPipeline/examples/SEIRS.html](https://www.fairdatapipeline.org/pyDataPipeline/examples/SEIRS.html)\n",
     'author': 'Ryan J Field',
     'author_email': 'ryan.field@glasgow.ac.uk',
     'maintainer': None,