fair-cli 0.9.5__tar.gz → 0.9.6__tar.gz

{fair_cli-0.9.5 → fair_cli-0.9.6}/CITATION.cff

@@ -26,7 +26,7 @@ authors:
   affiliation: University of Glasgow
   orcid: https://orcid.org/0000-0002-4424-9890
 
-date-released: 2023-09-07
+date-released: 2024-08-04
 keywords:
 - FAIR Data Pipeline
 - FAIR
@@ -36,4 +36,4 @@ license: BSD-2-Clause
 message: If you use this software, please cite it using these metadata.
 repository-code: https://github.com/FAIRDataPipeline/FAIR-CLI/
 title: "The FAIR Data Pipeline command line tool"
-version: 0.9.4
+version: 0.9.6

{fair_cli-0.9.5 → fair_cli-0.9.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fair-cli
-Version: 0.9.5
+Version: 0.9.6
 Summary: Synchronization interface for the SCRC FAIR Data Pipeline registry
 Home-page: https://www.fairdatapipeline.org/
 License: BSD-2-Clause

{fair_cli-0.9.5 → fair_cli-0.9.6}/fair/registry/sync.py

@@ -146,8 +146,7 @@ def sync_dependency_chain(
     dest_token: str,
     origin_token: str,
     local_data_store: str = None,
-    public: bool = False,
-    remote_author_url = None
+    public: bool = False
 ) -> typing.Dict[str, str]:
     """Push an object and all of its dependencies to the remote registry
 
@@ -187,7 +186,7 @@ def sync_dependency_chain(
     )
 
     _new_urls: typing.Dict[str, str] = {k: "" for k in _dependency_chain}
-    _writable_fields: typing.Dict[str, str] = {}
+    _writable_fields: typing.Dict[str, str] = {}    
 
     # For every object (and the order) in the dependency chain
     # post the object then store the URL so it can be used to assemble those
@@ -224,9 +223,9 @@ def sync_dependency_chain(
             object_url=object_url,
             new_urls=_new_urls,
             writable_data=_writable_data,
+            object_data = _obj_data,
             local_data_store = local_data_store,
-            public = public,
-            remote_author_url = remote_author_url
+            public = public
         )
 
         if not fdp_util.is_api_url(dest_uri, _new_url):
@@ -248,9 +247,9 @@ def _get_new_url(
     object_url: str,
     new_urls: typing.Dict,
     writable_data: typing.Dict,
+    object_data = {},
     local_data_store = None,
-    public = False,
-    remote_author_url = None
+    public = False
 ) -> typing.Tuple[typing.Dict, typing.List]:
     """Internal Function to return a resgistry entry from the remote registry given an origin entry URL
     If the entry does not exist it will be created
@@ -303,6 +302,7 @@ def _get_new_url(
     # (as remote URL will never match local)
 
     _obj_type = fdp_req.get_obj_type_from_url(object_url, token=origin_token)
+    _obj_id = fdp_req.get_obj_id_from_url(object_url)
 
     _filters = {
         k: v
@@ -317,19 +317,25 @@ def _get_new_url(
     if dest_uri == origin_uri:
         raise fdp_exc.InternalError("Cannot push object to its source address")
 
-     # If public replace storage root, path
+    # If Public is true then any files will be uploaded to the remote registry
+    # If local_data_store is set we're pulling to a local registry
     if public and not local_data_store:
+        # The remote data_store storage_root URL should always be the 1st storage_root
         _remote_storage_root_url = urllib.parse.urljoin(dest_uri, "storage_root/1/")
-        if _obj_type == "storage_root":
+        # If the current objects a storage_root and the root is the first (data_store)
+        # then simply return the remote
+        if _obj_type == "storage_root" and _obj_id == '1':
             return _remote_storage_root_url
-        if "path" in _new_obj_data:
-            _new_obj_data["path"] = _filters["path"] = _new_obj_data["hash"]
-        if "storage_root" in _new_obj_data:
-            _new_obj_data["storage_root"] = _remote_storage_root_url
-            _filters["storage_root"] = fdp_req.get_obj_id_from_url(_remote_storage_root_url)
-
-    # if remote_author_url and _obj_type == "object":
-    #     _new_obj_data["authors"] = _filters["authors"] = [remote_author_url]
+        # If the current object is a storage_location
+        elif _obj_type == "storage_location":
+            # Again if the storage_root is the data_store (storage_root 1)
+            if fdp_req.get_obj_id_from_url(object_data.get("storage_root", "/2")) == '1':
+                # Update the new object path and filter path
+                _new_obj_data["path"] = _filters["path"] = _new_obj_data["hash"]
+                # Update the new object storage_root
+                _new_obj_data["storage_root"] = _remote_storage_root_url
+                # Update the filters for storage_root to the ID
+                _filters["storage_root"] = fdp_req.get_obj_id_from_url(_remote_storage_root_url)
 
     if _obj_type == "object":
         _new_obj_data["authors"] = []
@@ -433,13 +439,13 @@ def get_new_author_url(
             if k in _writable_fields
         }
     return _get_new_url(
-        origin_uri,
-        origin_token,
-        dest_uri,
-        dest_token,
-        author_url,
-        {},
-        _writable_data
+        origin_uri= origin_uri,
+        origin_token= origin_token,
+        dest_uri= dest_uri,
+        dest_token= dest_token,
+        object_url= author_url,
+        new_urls= {},
+        writable_data= _writable_data
     )
 
 def sync_data_products(
@@ -450,8 +456,7 @@ def sync_data_products(
     remote_label: str,
     data_products: typing.List[str],
     local_data_store: str = None,
-    force: bool = False,
-    remote_author_url = None
+    force: bool = False
 ) -> None:
     """Transfer data products from one registry to another
 
@@ -545,8 +550,7 @@ def sync_data_products(
             dest_token=dest_token,
             origin_token=origin_token,
             local_data_store=local_data_store,
-            public=_is_public,
-            remote_author_url = remote_author_url
+            public=_is_public
         )
         # If local_data_store assume we're syncing from remote to local
         if local_data_store:
@@ -568,8 +572,8 @@ def sync_data_products(
                 for origin_input_code_run in origin_input_code_runs:
                     dest_component_url = get_dest_component_url(origin_component_url, dest_uri, dest_token, origin_token)
                     dest_inputs = [dest_component_url]
-                    dest_inputs += get_dest_inputs(origin_input_code_run["inputs"], origin_uri, dest_uri, dest_token, origin_token, remote_label, remote_author_url)
-                    sync_code_run(origin_uri, dest_uri, dest_token, origin_token, origin_input_code_run["uuid"], inputs= dest_inputs, remote_author_url = remote_author_url)
+                    dest_inputs += get_dest_inputs(origin_input_code_run["inputs"], origin_uri, dest_uri, dest_token, origin_token, remote_label)
+                    sync_code_run(origin_uri, dest_uri, dest_token, origin_token, origin_input_code_run["uuid"], inputs= dest_inputs)
                 
                 origin_output_code_runs = fdp_req.get(
                         origin_uri,
@@ -579,9 +583,10 @@ def sync_data_products(
                     )
                 for origin_output_code_run in origin_output_code_runs:
                     dest_component_url = get_dest_component_url(origin_component_url, dest_uri, dest_token, origin_token)
-                    dest_inputs = get_dest_inputs(origin_output_code_run["inputs"], origin_uri, dest_uri, dest_token, origin_token, remote_label, force, remote_author_url)
-                    sync_code_run(origin_uri, dest_uri, dest_token, origin_token, origin_output_code_run["uuid"], inputs= dest_inputs, outputs= [dest_component_url], remote_author_url = remote_author_url)
+                    dest_inputs = get_dest_inputs(origin_output_code_run["inputs"], origin_uri, dest_uri, dest_token, origin_token, remote_label, force)
+                    sync_code_run(origin_uri, dest_uri, dest_token, origin_token, origin_output_code_run["uuid"], inputs= dest_inputs, outputs= [dest_component_url])
         logger.info(f"Synced Data Product: {data_product}")
+
 def get_dest_component_url(
     origin_component_url: str,
     dest_uri: str,
@@ -626,8 +631,7 @@ def sync_code_runs(
     remote_label: str,
     code_runs: typing.List[str],
     local_data_store: str = None,
-    force: bool = False,
-    remote_author_url = None
+    force: bool = False
 ) -> None:
     """Transfer data code_run(s) from one registry to another
 
@@ -696,8 +700,7 @@ def sync_code_runs(
             remote_label, 
             _origin_data_products_formatted, 
             local_data_store,
-            force,
-            remote_author_url)
+            force)
 
         # Iterate through formatted objects and get their new values from the remote registry
         for _origin_data_product_formatted in _origin_data_products_formatted:
@@ -725,7 +728,7 @@ def sync_code_runs(
             if _origin_data_product_formatted in _outputs_data_products:
                 _dest_outputs += _dest_object["components"]
         logger.debug(f'attempting to sync coderun {code_run_uuid} with inputs {_dest_inputs} and outputs {_dest_outputs}')
-        sync_code_run(origin_uri, dest_uri, dest_token, origin_token, code_run_uuid, _dest_inputs, _dest_outputs, remote_author_url = remote_author_url)
+        sync_code_run(origin_uri, dest_uri, dest_token, origin_token, code_run_uuid, _dest_inputs, _dest_outputs)
         logger.info(f"Synced Code Run: {code_run_uuid}")
 
 # Internal function to return the (remote) object associated with a code_run field containing and object url
@@ -772,8 +775,7 @@ def sync_code_run(
     origin_token: str,
     code_run_uuid: str,
     inputs = [],
-    outputs = [],
-    remote_author_url = None) -> typing.Dict:
+    outputs = []) -> typing.Dict:
     """_summary_
 
     Args:
@@ -820,8 +822,7 @@ def sync_code_run(
         origin_uri=origin_uri,
         dest_token=dest_token,
         origin_token=origin_token,
-        public= True,
-        remote_author_url = remote_author_url
+        public= True
         )
         _dest_code_run_model_config = get_dest_object_url(code_run["model_config"], dest_uri, dest_token, origin_token)
         upload_object(origin_uri, dest_uri, dest_token, origin_token, code_run["model_config"])
@@ -834,8 +835,7 @@ def sync_code_run(
             origin_uri=origin_uri,
             dest_token=dest_token,
             origin_token=origin_token,
-            public= True,
-            remote_author_url = remote_author_url
+            public= True
             )
             _dest_code_run_code_repo = get_dest_object_url(code_run["code_repo"],  dest_uri, dest_token, origin_token)
         # Sync Submision Script
@@ -845,8 +845,7 @@ def sync_code_run(
         origin_uri=origin_uri,
         dest_token=dest_token,
         origin_token=origin_token,
-        public= True,
-        remote_author_url = remote_author_url
+        public= True
         )
         _dest_code_run_submission_script = get_dest_object_url(code_run["submission_script"], dest_uri, dest_token, origin_token)
         upload_object(origin_uri, dest_uri, dest_token, origin_token, code_run["submission_script"])
@@ -941,8 +940,7 @@ def get_dest_inputs(origin_inputs: typing.List,
     dest_token: str,
     origin_token: str,
     remote_label: str,
-    force: bool = False,
-    remote_author_url = None) -> list:
+    force: bool = False) -> list:
     """Returns a list of input component urls on the destination registry
      from a given list of input component urls from the origin registry
      assumes the destination componets already exists in the remote registry
@@ -963,7 +961,7 @@ def get_dest_inputs(origin_inputs: typing.List,
     for origin_input in origin_inputs:
         component_data_products = get_data_products_from_component(origin_input, origin_token)
         if component_data_products:
-            sync_data_products(origin_uri, dest_uri, dest_token, origin_token, remote_label, format_data_product_list(component_data_products, origin_token), force= force, remote_author_url = remote_author_url)
+            sync_data_products(origin_uri, dest_uri, dest_token, origin_token, remote_label, format_data_product_list(component_data_products, origin_token), force= force)
             for data_product_url in component_data_products:
                 origin_data_product_object_url = fdp_req.url_get(data_product_url, origin_token)["object"]
                 dest_object_url = get_dest_object_url(origin_data_product_object_url, dest_uri, dest_token, origin_token)
@@ -976,8 +974,7 @@ def get_dest_inputs(origin_inputs: typing.List,
                 origin_uri=origin_uri,
                 dest_token=dest_token,
                 origin_token=origin_token,
-                public= is_component_public(origin_input, origin_token),
-                remote_author_url = remote_author_url
+                public= is_component_public(origin_input, origin_token)
                 )
             dest_inputs.append(get_dest_component_url(origin_input, dest_uri, dest_token, origin_token))
     return dest_inputs

{fair_cli-0.9.5 → fair_cli-0.9.6}/fair/session.py

@@ -411,8 +411,7 @@ class FAIR:
             ),
             origin_token=fdp_req.local_token(),
             remote_label=remote,
-            code_runs=_staged_code_runs,
-            remote_author_url = remote_author_url
+            code_runs=_staged_code_runs
         )
 
         fdp_sync.sync_data_products(
@@ -423,8 +422,7 @@ class FAIR:
             ),
             origin_token=fdp_req.local_token(),
             remote_label=remote,
-            data_products=_staged_data_products,
-            remote_author_url = remote_author_url
+            data_products=_staged_data_products
         )
 
         self._session_config.write_log_lines(
@@ -495,7 +493,7 @@ class FAIR:
 
         self._logger.debug("Pulling data products locally")
 
-        # Only push data products if there are any to do so, this covers the
+        # Only pull data products if there are any to do so, this covers the
         # case whereby no remote has been setup and we just want to register
         # items on the local registry
         if _readables:

{fair_cli-0.9.5 → fair_cli-0.9.6}/pyproject.toml

@@ -24,7 +24,7 @@ classifiers = [
 ]
 description = "Synchronization interface for the SCRC FAIR Data Pipeline registry"
 name = "fair-cli"
-version = "0.9.5"
+version = "0.9.6"
 
 homepage = "https://www.fairdatapipeline.org/"