data_pipeline_api 0.9.0__tar.gz → 0.9.2__tar.gz

{data_pipeline_api-0.9.0 → data_pipeline_api-0.9.2}/PKG-INFO

@@ -1,22 +1,24 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: data_pipeline_api
-Version: 0.9.0
+Version: 0.9.2
 Summary: Python api to interact with the Fair Data Pipeline
-Home-page: https://www.fairdatapipeline.org/
 License: GNU General Public License v3.0
+License-File: LICENSE
 Keywords: FAIR Data Pipeline,FAIR,Data Management,Provenance
 Author: Ryan J Field
 Author-email: ryan.field@glasgow.ac.uk
-Requires-Python: >=3.9,<4.0
+Requires-Python: >=3.10,<4.0
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3.15
 Requires-Dist: PyYAML (>=6.0,<7.0)
 Requires-Dist: requests (>=2.23.0,<3.0.0)
+Project-URL: Homepage, https://www.fairdatapipeline.org/
 Project-URL: Issue Tracker, https://github.com/FAIRDataPipeline/pyDataPipeline/issues
 Project-URL: Repository, https://github.com/FAIRDataPipeline/pyDataPipeline
 Description-Content-Type: text/markdown

{data_pipeline_api-0.9.0 → data_pipeline_api-0.9.2}/data_pipeline_api/fdp_utils.py

@@ -163,7 +163,9 @@ def post_entry(
         return existing_entry[0]
 
     if response.status_code != 201:
-        raise ValueError(SERVER_RESPONSE_STR + str(response.status_code))
+        raise ValueError(
+            f"{SERVER_RESPONSE_STR}{response.status_code}: {response.text}"
+        )
 
     return response.json()
 
@@ -188,7 +190,9 @@ def patch_entry(
 
     response = requests.patch(url, data_json, headers=headers)
     if response.status_code != 200:
-        raise ValueError(SERVER_RESPONSE_STR + str(response.status_code))
+        raise ValueError(
+            f"{SERVER_RESPONSE_STR}{response.status_code}: {response.text}"
+        )
 
     return response.json()
 

{data_pipeline_api-0.9.0 → data_pipeline_api-0.9.2}/data_pipeline_api/pipeline.py

@@ -347,6 +347,47 @@ def finalise(token: str, handle: dict) -> None:
     datastore = handle["yaml"]["run_metadata"]["write_data_store"]
     api_version = handle["yaml"]["run_metadata"]["api_version"]
 
+    # Refuse outputs whose data product version is already registered before
+    # anything is posted or moved: the registry allows one producing code run
+    # per object, so the existing object cannot be claimed by this run
+    for output in handle.get("output", {}):
+        if "${{RUN_ID}}" in handle["output"][output]["use_data_product"]:
+            handle["output"][output]["use_data_product"] = handle["output"][
+                output
+            ]["use_data_product"].replace("${{RUN_ID}}", handle["code_run_uuid"])
+
+        namespace_name = handle["output"][output]["use_namespace"]
+        data_product_name = handle["output"][output]["use_data_product"]
+        version = handle["output"][output]["use_version"]
+
+        existing_namespace = fdp_utils.get_entry(
+            url=registry_url,
+            endpoint="namespace",
+            query={"name": namespace_name},
+            api_version=api_version,
+        )
+        if not existing_namespace:
+            continue
+
+        data_product_exists = fdp_utils.get_entry(
+            url=registry_url,
+            endpoint="data_product",
+            query={
+                "name": data_product_name,
+                "version": version,
+                "namespace": fdp_utils.get_first_entry(existing_namespace)[
+                    "url"
+                ],
+            },
+            api_version=api_version,
+        )
+        if data_product_exists:
+            raise ValueError(
+                f"Data product {namespace_name}:{data_product_name}@{version} "
+                "already exists; bump the version in the write block of your "
+                "config"
+            )
+
     datastore = fdp_utils.remove_local_from_root(datastore)
     datastore_root = fdp_utils.get_entry(
         url=registry_url,
@@ -372,13 +413,6 @@ def finalise(token: str, handle: dict) -> None:
 
     if "output" in handle:
         for output in handle["output"]:
-
-            if "${RUN_ID}" in handle["output"][output]["use_data_product"]:
-                handle["output"][output]["use_data_product"] = handle[
-                    "output"
-                ][output]["use_data_product"].replace(
-                    "${RUN_ID}", handle["code_run_uuid"]
-                )
             write_namespace = fdp_utils.get_entry(
                 url=registry_url,
                 endpoint="namespace",
@@ -492,82 +526,56 @@ def finalise(token: str, handle: dict) -> None:
                 api_version=api_version,
                 )["url"]
 
-            data_product_exists = fdp_utils.get_entry(
+            object_url = fdp_utils.post_entry(
+                token=token,
                 url=registry_url,
-                endpoint="data_product",
-                query={
-                    "name": handle["output"][output]["use_data_product"],
-                    "version": handle["output"][output]["use_version"],
-                    "namespace": write_namespace_url,
+                endpoint="object",
+                data={
+                    "description": handle["output"][output][
+                        "data_product_description"
+                    ],
+                    "storage_location": storage_location_url,
+                    "authors": [handle["author"]],
+                    "file_type": file_type_url,
                 },
                 api_version=api_version,
-            )
+            )["url"]
 
-            if data_product_exists:
-                data_product_exists_dict = fdp_utils.get_first_entry(
-                    data_product_exists
-                )
-                data_product_url = data_product_exists_dict["url"]
-                object_url = data_product_exists_dict["object"]
-                obj = fdp_utils.get_entity(
-                    url=registry_url,
-                    endpoint="object",
-                    id=int(fdp_utils.extract_id(object_url)),
-                    api_version=api_version,
-                )
-                component_url = obj["components"][0]
+            component_url = None
 
-            else:
-                object_url = fdp_utils.post_entry(
+            if handle["output"][output]["use_component"]:
+                component_url = fdp_utils.post_entry(
                     token=token,
                     url=registry_url,
-                    endpoint="object",
+                    endpoint="object_component",
                     data={
-                        "description": handle["output"][output][
-                            "data_product_description"
-                        ],
-                        "storage_location": storage_location_url,
-                        "authors": [handle["author"]],
-                        "file_type": file_type_url,
+                        "object": object_url,
+                        "name": handle["output"][output]["use_component"],
                     },
                     api_version=api_version,
                 )["url"]
-
-                component_url = None
-
-                if handle["output"][output]["use_component"]:
-                    component_url = fdp_utils.post_entry(
-                        token=token,
-                        url=registry_url,
-                        endpoint="object_component",
-                        data={
-                            "object": object_url,
-                            "name": handle["output"][output]["use_component"],
-                        },
-                        api_version=api_version,
-                    )["url"]
-                else:
-                    component_url = fdp_utils.get_entry(
-                        url=registry_url,
-                        endpoint="object_component",
-                        query={
-                            "object": fdp_utils.extract_id(object_url),
-                        },
-                        api_version=api_version,
-                    )[0]["url"]
-
-                data_product_url = fdp_utils.post_entry(
-                    token=token,
+            else:
+                component_url = fdp_utils.get_entry(
                     url=registry_url,
-                    endpoint="data_product",
-                    data={
-                        "name": handle["output"][output]["use_data_product"],
-                        "version": handle["output"][output]["use_version"],
-                        "object": object_url,
-                        "namespace": write_namespace_url,
+                    endpoint="object_component",
+                    query={
+                        "object": fdp_utils.extract_id(object_url),
                     },
                     api_version=api_version,
-                )["url"]
+                )[0]["url"]
+
+            data_product_url = fdp_utils.post_entry(
+                token=token,
+                url=registry_url,
+                endpoint="data_product",
+                data={
+                    "name": handle["output"][output]["use_data_product"],
+                    "version": handle["output"][output]["use_version"],
+                    "object": object_url,
+                    "namespace": write_namespace_url,
+                },
+                api_version=api_version,
+            )["url"]
 
             handle["output"][output]["component_url"] = component_url
             handle["output"][output]["data_product_url"] = data_product_url

{data_pipeline_api-0.9.0 → data_pipeline_api-0.9.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "data_pipeline_api"
-version = "0.9.0"
+version = "0.9.2"
 description = "Python api to interact with the Fair Data Pipeline"
 authors = [
     "Ryan J Field <ryan.field@glasgow.ac.uk>",
@@ -31,7 +31,7 @@ keywords = [
 ]
 
 [tool.poetry.dependencies]
-python = ">=3.9,<4.0"
+python = ">=3.10,<4.0"
 requests = "^2.23.0"
 PyYAML = "^6.0"
 
@@ -40,12 +40,12 @@ pytest = "^8.3.4"
 black = "*"
 mypy = "*"
 flake8 = "*"
-poetry = "^1.1.12"
+poetry = "^2.0.0"
 pytest-mock = "*"
 pytest-dependency = "*"
 pytest-cov = "*"
 pre-commit = "^4.0.1"
-isort = "^5.10.1"
+isort = "^6.0.0"
 Sphinx = "*"
 sphinx-rtd-theme = "*"