fair-cli 0.10.0__tar.gz → 0.10.1__tar.gz

{fair_cli-0.10.0 → fair_cli-0.10.1}/CHANGELOG.md

@@ -1,3 +1,49 @@
+# 2026-10-05 [v0.10.1](https://github.com/FAIRDataPipeline/FAIR-CLI/releases/tag/v0.10.1)
+
+## Changed behaviour
+- A wildcard entry keeps to one namespace: the one its `use:` names, else the default for its block. It
+  used to match names in every namespace, so a `write:` pattern could write a name into a namespace
+  other than its own.
+- A wildcard entry gives one entry for each name it matches: for a `read:`, the version its `use:` names,
+  else the highest; for a `write:`, the version to be written. It used to give one for every version, and
+  to ignore a version on a `read:`. A `read:` pattern that matches nothing is not an error.
+- `fair run` refuses a `read:` of a data product, or of a version of one, that is not in the registry,
+  naming the namespace and registry it looked in. Such a read used to pass unchecked, or as version 0.0.0.
+- `fair push` fails if a file cannot be uploaded. It used to warn and carry on, leaving the remote with
+  the file's records and no file. A file is now uploaded before its records are written, so a push that
+  failed can be run again.
+- `fair pull` checks each file it fetches against the hash the remote registry holds for it, and refuses
+  one that differs.
+
+## Added
+- Registering an external object records where its file was fetched from: the `root` and `path` of its
+  `register:` entry, as the object's `original_store`, with the hash of the file found there. `fair push`
+  takes that record to the remote. A file registered from the machine itself has no such record.
+- `fair push` takes a refusal of an upload address with HTTP 409 to mean that the remote's store already
+  holds the file, sends nothing, and carries on. No released registry answers so yet.
+
+## Fixed
+- `fair push` sent no value that was false, leaving the remote to apply its default: an external object
+  registered with `primary: false` arrived as primary, and a location with `public: false` as public.
+- A list from a registry stopped at its first 100 rows, so wildcards, staging and version lookups went
+  wrong, with no message, in a registry holding more.
+- `fair pull` looked for the version of a `read:` entry in the local registry only, so an entry at the
+  default version, for something not yet local, asked the remote for version 0.0.0 and failed.
+- `fair push` held each file whole in memory, `fair pull` more than twice over, and both left a copy of
+  it in the temporary directory. Files are now sent and received a block at a time, whatever their
+  size, and nothing is left behind.
+- An upload of 2 GiB or more over plain http failed on macOS.
+- An external object could not be registered with a `unique_name` and `alternate_identifier_type` in
+  place of an `identifier`: `fair pull` stopped, reporting that the object already existed.
+- `fair identify` looked at only the newest of a file's storage locations, so it could miss the data
+  products of a file the registry records in more than one place.
+- `fair run` could not start the script when the path of the data store, which holds the job
+  directory, had a space in it.
+
+## Development
+- The Python job of the implementations workflow starts an object store for its remote registry, so
+  that its `fair push` uploads the files. It had none, and passed while a failed upload was a warning.
+
 # 2026-10-03 [v0.10.0](https://github.com/FAIRDataPipeline/FAIR-CLI/releases/tag/v0.10.0)
 
 ## Changed behaviour

{fair_cli-0.10.0 → fair_cli-0.10.1}/CITATION.cff

@@ -26,7 +26,7 @@ authors:
   affiliation: University of Glasgow
   orcid: https://orcid.org/0000-0002-4424-9890
 
-date-released: 2026-10-03
+date-released: 2026-10-05
 keywords:
 - FAIR Data Pipeline
 - FAIR
@@ -36,4 +36,4 @@ license: BSD-2-Clause
 message: If you use this software, please cite it using these metadata.
 repository-code: https://github.com/FAIRDataPipeline/FAIR-CLI/
 title: "The FAIR Data Pipeline command line tool"
-version: 0.10.0
+version: 0.10.1

{fair_cli-0.10.0 → fair_cli-0.10.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fair-cli
-Version: 0.10.0
+Version: 0.10.1
 Summary: Synchronization interface for the SCRC FAIR Data Pipeline registry
 Home-page: https://www.fairdatapipeline.org/
 License: BSD-2-Clause

{fair_cli-0.10.0 → fair_cli-0.10.1}/fair/registry/requests.py

@@ -84,6 +84,44 @@ def local_token(registry_dir: str = None) -> str:
     return _file_lines[0].strip()
 
 
+# The rest of a list, which a registry gives a page at a time, each page with
+# the address of the next. Only the cursor is taken from that address: the
+# rest of it is the registry's own idea of where it is, which behind a proxy
+# need not be where it is reached
+def _later_pages(
+    url: str, headers: typing.Dict, params: typing.Dict, page: typing.Dict
+) -> typing.List:
+    _results: typing.List = []
+    _status = 200
+
+    while page.get("next"):
+        try:
+            _next = urllib.parse.urlparse(page["next"])
+            _cursor = urllib.parse.parse_qs(_next.query)["cursor"][0]
+            _request = requests.get(
+                url, headers=headers, params={**params, "cursor": _cursor}
+            )
+            _status = _request.status_code
+            page = _request.json()
+            _results += page["results"]
+        except requests.exceptions.ConnectionError as e:
+            raise fdp_exc.UnexpectedRegistryServerState(
+                f"Failed to make registry API request '{url}'",
+                hint="Is this remote correct and the server running?",
+            ) from e
+        except (
+            json.JSONDecodeError,
+            simplejson.errors.JSONDecodeError,
+            KeyError,
+        ) as exc:
+            raise fdp_exc.RegistryAPICallError(
+                f"Failed to retrieve every page of results from '{url}'",
+                error_code=_status,
+            ) from exc
+
+    return _results
+
+
 def _access(
     uri: str,
     method: str = None,
@@ -180,6 +218,10 @@ def _access(
             f"Request failed with status code {_request.status_code}: {_info}",
             error_code=_request.status_code,
         )
+
+    if method == "get" and "results" in _json_req:
+        _result += _later_pages(_url, _headers, params, _json_req)
+
     return _result
 
 
@@ -215,8 +257,10 @@ def post(
 
     headers.update({"Content-Type": "application/json"})
 
+    # False and 0 are values, not gaps: left out, the registry would apply
+    # its own default, which for 'public' and 'primary_not_supplement' is true
     for param, value in data.copy().items():
-        if not value:
+        if not value and not isinstance(value, (bool, int, float)):
             logger.debug(
                 f"Key in post data '{param}' \
                          has no value so will be ignored"
@@ -483,7 +527,13 @@ def put_file(upload_url: str, file_loc: str) -> bool:
         bool: Will return True if the upload succeeded.
     """
     s = requests.Session()
-    _req = s.put(upload_url, data=open(file_loc, mode="rb").read())
+    # Sent from the open file, a block at a time, whatever its size. An empty
+    # file is sent as no bytes: given a file with nothing to read, requests
+    # would not state a length, which an object store insists on
+    with open(file_loc, mode="rb") as _file:
+        _req = s.put(
+            upload_url, data=_file if os.path.getsize(file_loc) else b""
+        )
     if _req.status_code not in [200, 201]:
         raise fdp_exc.RegistryError(
             f"File: {file_loc} could not be uploaded,\
@@ -492,7 +542,7 @@ def put_file(upload_url: str, file_loc: str) -> bool:
     return True
 
 
-def download_file(url: str, chunk_size: int = 8192) -> str:
+def download_file(url: str, chunk_size: int = 1024 * 1024) -> str:
     """Download a file from a given URL
 
     Parameters
@@ -500,7 +550,7 @@ def download_file(url: str, chunk_size: int = 8192) -> str:
     url : str
         address of remote file
     chunk_size : int, optional
-        chunk size for download, by default 8192
+        chunk size for download, by default 1 MiB
 
     Returns
     -------
@@ -511,10 +561,15 @@ def download_file(url: str, chunk_size: int = 8192) -> str:
     ------
     requests.HTTPError
         if the server answers with an error status
+    fdp_exc.FAIRCLIException
+        if the file could not be fetched, or not all of it
     """
     # Save the data to a temporary file so we can calculate the hash
     _file = tempfile.NamedTemporaryFile(delete=False)
     _fname = _file.name
+    # Only the name is wanted: the file is written, and on failure removed,
+    # by name, which Windows refuses while this handle is open
+    _file.close()
 
     # Copy File if local
     if "file://" in url:
@@ -522,6 +577,7 @@ def download_file(url: str, chunk_size: int = 8192) -> str:
         try:
             shutil.copy2(_local_fname, _fname)
         except Exception as e:
+            os.remove(_fname)
             raise fdp_exc.FAIRCLIException(
                 f"Failed to download file '{url}'"
                 f" due to connection error: {traceback.format_exc()}"
@@ -531,17 +587,37 @@ def download_file(url: str, chunk_size: int = 8192) -> str:
             requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
             headers = {"User-Agent": str(UserAgent().chrome)}
             response = requests.get(
-                url, allow_redirects=True, verify=False, headers=headers
+                url,
+                allow_redirects=True,
+                verify=False,
+                headers=headers,
+                stream=True,
             )
         except Exception as e:
+            os.remove(_fname)
             raise fdp_exc.FAIRCLIException(
                 f"Failed to download file '{url}'"
                 f" due to connection error: {traceback.format_exc()}"
             ) from e
         # Outside the try, so that callers can tell an HTTP error from a
         # failure to connect, and an error page is never saved as the data
-        response.raise_for_status()
-        open(_fname, mode="wb").write(response.content)
+        try:
+            response.raise_for_status()
+            # Written as it arrives, a block at a time, whatever its size
+            with open(_fname, mode="wb") as _out:
+                for _chunk in response.iter_content(chunk_size=chunk_size):
+                    _out.write(_chunk)
+        except requests.HTTPError:
+            os.remove(_fname)
+            raise
+        except requests.RequestException as e:
+            os.remove(_fname)
+            raise fdp_exc.FAIRCLIException(
+                f"Failed to download all of file '{url}'"
+                f" due to connection error: {traceback.format_exc()}"
+            ) from e
+        finally:
+            response.close()
 
     return _fname
 

{fair_cli-0.10.0 → fair_cli-0.10.1}/fair/registry/storage.py

@@ -473,6 +473,13 @@ def store_data_file(
         registry_uri=uri,
         registry_token=token,
         data_product_url=_data_prod_url,
+        original_store_url=_get_url_from_original_store(
+            data=data,
+            registry_uri=uri,
+            registry_token=token,
+            storage_loc_url=_post_store_loc,
+            is_public=public,
+        ),
     )
 
 
@@ -493,7 +500,12 @@ def _get_url_from_storage_loc(
         "hash": _hash,
     }
 
-    _search_data = {"hash": _hash}
+    # The same file may be recorded under another root as well, as where an
+    # external object was fetched from
+    _search_data = {
+        "hash": _hash,
+        "storage_root": fdp_req.get_obj_id_from_url(root_store_url),
+    }
 
     return fdp_req.post_else_get(
         registry_uri,
@@ -504,6 +516,73 @@ def _get_url_from_storage_loc(
     )
 
 
+# The storage location of the place an external object's file was fetched
+# from: the 'root' and 'path' of its 'register' entry, with the hash its
+# location in the data store was given. None if the entry names no place
+# that anyone else could fetch it from: a file on this machine, or (a root
+# that is a bare path) one beside the registry itself.
+def _get_url_from_original_store(
+    data: typing.Dict,
+    registry_uri: str,
+    registry_token: str,
+    storage_loc_url: str,
+    is_public: bool,
+) -> typing.Optional[str]:
+    _root = data.get("root")
+    _path = data.get("path")
+
+    if not _root or not _path or _root.startswith(("/", "file://")):
+        return None
+
+    _root_url = fdp_req.post_else_get(
+        registry_uri,
+        "storage_root",
+        registry_token,
+        data={"root": _root, "local": False},
+        params={"root": _root},
+    )
+    _hash = fdp_req.url_get(storage_loc_url, registry_token)["hash"]
+
+    # A registry records a file once under a root, whatever its path there
+    _recorded = [
+        location
+        for location in fdp_req.get(
+            registry_uri,
+            "storage_location",
+            registry_token,
+            params={
+                "hash": _hash,
+                "storage_root": fdp_req.get_obj_id_from_url(_root_url),
+            },
+        )
+        if str(location["public"]).lower() == str(is_public).lower()
+    ]
+
+    if _recorded:
+        if _recorded[0]["path"] != _path:
+            logger.warning(
+                "The file at '%s%s' is already recorded at '%s%s', "
+                "which is given as where it came from",
+                _root,
+                _path,
+                _root,
+                _recorded[0]["path"],
+            )
+        return _recorded[0]["url"]
+
+    return fdp_req.post(
+        registry_uri,
+        "storage_location",
+        registry_token,
+        data={
+            "path": _path,
+            "storage_root": _root_url,
+            "public": str(is_public).lower(),
+            "hash": _hash,
+        },
+    )["url"]
+
+
 def _get_url_from_file_type(
     data: typing.Dict, local_file: str, registry_uri: str, registry_token: str
 ) -> str:
@@ -539,6 +618,7 @@ def _get_url_from_external_obj(
     registry_uri: str,
     registry_token: str,
     data_product_url: str,
+    original_store_url: str = None,
 ) -> typing.Dict:
     _expected_ext_obj_keys = ("release_date", "primary", "title")
 
@@ -553,6 +633,7 @@ def _get_url_from_external_obj(
         "title": data["title"],
         "primary_not_supplement": f'{data["primary"]}',
         "release_date": data["release_date"],
+        "original_store": original_store_url,
     }
     _external_obj_data.update(_get_identifier_from_data(data, local_file))
 
@@ -672,10 +753,10 @@ def _get_identifier_from_data(data: typing.Dict, label: str) -> typing.Dict[str,
             raise fdp_exc.UserConfigError(
                 f"No identifier/alternate_identifier given for item '{label}'",
                 hint="You must provide either a URL 'identifier', or "
-                "'unique_name' and 'source_name' keys",
+                "'unique_name' and 'alternate_identifier_type' keys",
             ) from e
 
-        _identifier["alternate_identifier"] = data.get(
+        _identifier["alternate_identifier_type"] = data.get(
             "alternate_identifier_type", "local source descriptor"
         )
 
@@ -868,6 +949,14 @@ def upload_remote_file(
     """
     if not os.path.exists(file_loc):
         raise fdp_exc.FileNotFoundError(f"File: {file_loc} does not exist")
-    _upload_url = get_upload_url(file_loc, remote_uri, remote_token)
+    try:
+        _upload_url = get_upload_url(file_loc, remote_uri, remote_token)
+    except fdp_exc.RegistryAPICallError as e:
+        # A registry refuses an address to upload a file it already holds,
+        # found by its hash: there is then nothing to send
+        if e.error_code != 409:
+            raise
+        logger.debug(f"File {file_loc} is already in the registry's store")
+        return
     logger.debug(f"Uploading {file_loc} to URL: {_upload_url}")
     fdp_req.put_file(_upload_url, file_loc)

{fair_cli-0.10.0 → fair_cli-0.10.1}/fair/registry/sync.py

@@ -567,6 +567,23 @@ def sync_data_products(
         if result["external_object"]:
             result = fdp_req.url_get(result["external_object"], token=origin_token)
 
+        # The file is moved before its records are written, so that a failure
+        # leaves no record, and the next attempt does not take this data
+        # product for one already there. If local_data_store assume we're
+        # syncing from remote to local, else a public file is uploaded to
+        # object storage
+        if local_data_store:
+            logger.debug("Retrieving files from remote registry data storage")
+            fetch_data_product(origin_token, local_data_store, _data_product)
+        elif _is_public:
+            upload_object(
+                origin_uri,
+                dest_uri,
+                dest_token,
+                origin_token,
+                result_object["url"],
+            )
+
         sync_dependency_chain(
             object_url=result["url"],
             dest_uri=dest_uri,
@@ -576,18 +593,8 @@ def sync_data_products(
             local_data_store=local_data_store,
             public=_is_public,
         )
-        # If local_data_store assume we're syncing from remote to local
-        if local_data_store:
-            logger.debug("Retrieving files from remote registry data storage")
-            fetch_data_product(origin_token, local_data_store, _data_product)
-        # Else going from local to remote
-        else:
-            # If the storage location is public upload the files to object storage
-            if _is_public:
-                upload_object(
-                    origin_uri, dest_uri, dest_token, origin_token, result_object["url"]
-                )
-
+        # Going from local to remote
+        if not local_data_store:
             for origin_component_url in result_object["components"]:
                 origin_input_code_runs = fdp_req.get(
                     origin_uri,
@@ -925,7 +932,10 @@ def sync_code_run(
             data={"inputs": list(set(inputs)), "outputs": list(set(outputs))},
         )
     else:
-        # Get and sync model config
+        # Get and sync model config, the file before its records
+        upload_object(
+            origin_uri, dest_uri, dest_token, origin_token, code_run["model_config"]
+        )
         sync_dependency_chain(
             object_url=code_run["model_config"],
             dest_uri=dest_uri,
@@ -937,9 +947,6 @@ def sync_code_run(
         _dest_code_run_model_config = get_dest_object_url(
             code_run["model_config"], dest_uri, dest_token, origin_token
         )
-        upload_object(
-            origin_uri, dest_uri, dest_token, origin_token, code_run["model_config"]
-        )
         # If theres a code_repo sync it
         _dest_code_run_code_repo = None
         if code_run["code_repo"]:
@@ -954,7 +961,14 @@ def sync_code_run(
             _dest_code_run_code_repo = get_dest_object_url(
                 code_run["code_repo"], dest_uri, dest_token, origin_token
             )
-        # Sync Submision Script
+        # Sync Submision Script, the file before its records
+        upload_object(
+            origin_uri,
+            dest_uri,
+            dest_token,
+            origin_token,
+            code_run["submission_script"],
+        )
         sync_dependency_chain(
             object_url=code_run["submission_script"],
             dest_uri=dest_uri,
@@ -966,13 +980,6 @@ def sync_code_run(
         _dest_code_run_submission_script = get_dest_object_url(
             code_run["submission_script"], dest_uri, dest_token, origin_token
         )
-        upload_object(
-            origin_uri,
-            dest_uri,
-            dest_token,
-            origin_token,
-            code_run["submission_script"],
-        )
 
         # If the code run is not in the remote registry post the coderun
         dest_code_run = fdp_req.post(
@@ -1157,18 +1164,10 @@ def fetch_data_product(
             "as there is no physical storage location",
             data_product,
         )
+        return
 
     _storage_loc = fdp_req.url_get(_object["storage_location"], remote_token)
 
-    _path = _storage_loc["path"]
-    _path = urllib.parse.quote(_path)
-    _root = fdp_req.url_get(_storage_loc["storage_root"], remote_token)
-
-    _reg_parse = urllib.parse.urlparse(_endpoint)
-    _reg_url = f"{_reg_parse.scheme}://{_reg_parse.netloc}"
-
-    _downloaded_file = download_from_registry(_reg_url, _root["root"], _path)
-
     _namespace = fdp_req.url_get(data_product["namespace"], remote_token)
 
     if _file_type_url := _object.get("file_type", None):
@@ -1180,15 +1179,36 @@ def fetch_data_product(
         local_data_store, _namespace["name"], data_product["name"]
     )
 
-    os.makedirs(_local_dir, exist_ok=True)
-
     _out_file = os.path.join(_local_dir, f'{data_product["version"]}{_file_type}')
 
     if os.path.exists(_out_file):
         logger.debug("File '%s' already exists skipping download", _out_file)
         return
 
-    shutil.copy(_downloaded_file, _out_file)
+    _path = _storage_loc["path"]
+    _path = urllib.parse.quote(_path)
+    _root = fdp_req.url_get(_storage_loc["storage_root"], remote_token)
+
+    _reg_parse = urllib.parse.urlparse(_endpoint)
+    _reg_url = f"{_reg_parse.scheme}://{_reg_parse.netloc}"
+
+    _downloaded_file = download_from_registry(_reg_url, _root["root"], _path)
+
+    # A file is kept only if it is the one the registry describes
+    _hash = fdp_store.calculate_file_hash(_downloaded_file)
+    if _hash != _storage_loc["hash"]:
+        os.remove(_downloaded_file)
+        raise fdp_exc.SynchronisationError(
+            f"Data product '{_namespace['name']}:{data_product['name']}"
+            f"@v{data_product['version']}' was not fetched: the file at "
+            f"'{_root['root']}{_path}' has hash '{_hash}', and the registry "
+            f"records '{_storage_loc['hash']}'",
+            error_code=1,
+        )
+
+    os.makedirs(_local_dir, exist_ok=True)
+
+    shutil.move(_downloaded_file, _out_file)
 
 
 def download_from_registry(registry_url: str, root: str, path: str) -> str:
@@ -1249,8 +1269,6 @@ def upload_object(
     """
     Upload a file from the remote registry given the object url.
 
-    This function only preduces a warning if the file cannot be uploaded, allowing the file to be uploaded manually afterwards
-
     Parameters
     ----------
     origin_url : str
@@ -1267,8 +1285,12 @@ def upload_object(
     Returns
     -------
     bool
-        was the file successfully uploaded
+        whether there was a file to upload
 
+    Raises
+    ------
+    fdp_exc.SynchronisationError
+        if the file was not uploaded
     """
     _object = fdp_req.url_get(object_url, origin_token)
     if not _object["storage_location"]:
@@ -1282,18 +1304,31 @@ def upload_object(
     _object_storage_location_root = fdp_req.url_get(
         _object_storage_location["storage_root"], origin_token
     )
-    _file_loc = download_from_registry(
-        origin_uri,
-        _object_storage_location_root["root"],
-        _object_storage_location["path"],
-    )
+    # A file on this machine is uploaded from where it is. Any other is
+    # fetched first, to a temporary file which is removed afterwards
+    _is_local = _is_local_root(_object_storage_location_root)
+    if _is_local:
+        _file_loc = (
+            _object_storage_location_root["root"]
+            + _object_storage_location["path"]
+        ).replace("file://", "")
+    else:
+        _file_loc = download_from_registry(
+            origin_uri,
+            _object_storage_location_root["root"],
+            _object_storage_location["path"],
+        )
     try:
         fdp_store.upload_remote_file(_file_loc, dest_uri, dest_token)
-        logger.debug(f"File {_file_loc} Uploaded Successfully")
-        return True
-    except Exception:
-        logger.warning(
-            f'File upload error: {_object["description"]} was not uploaded to remote registry please upload the file manually'
-        )
+    except Exception as e:
         logger.debug(f"{traceback.format_exc()}")
-        return False
+        raise fdp_exc.SynchronisationError(
+            f"File '{_file_loc}' ({_object['description']}) was not uploaded "
+            f"to registry '{dest_uri}': {e}",
+            error_code=1,
+        ) from e
+    finally:
+        if not _is_local:
+            os.remove(_file_loc)
+    logger.debug(f"File {_file_loc} Uploaded Successfully")
+    return True

{fair_cli-0.10.0 → fair_cli-0.10.1}/fair/session.py

@@ -1378,24 +1378,26 @@ class FAIR:
             raise fdp_exc.FileNotFoundError(f"File: {file_path} does not exist")
         click.echo(f"Information about file: {file_path}")
         _hash = fdp_store.calculate_file_hash(file_path)
-        _storage_location = fdp_req.get(
+        _storage_locations = fdp_req.get(
             _api_url, "storage_location", _token, params={"hash": _hash}
         )
-        if _storage_location:
-            _storage_location = _storage_location[0]
-        else:
+        if not _storage_locations:
             click.echo(f"File: {file_path} could not be found on the {_registry}")
             return
-        _objects = fdp_req.get(
-            _api_url,
-            "object",
-            _token,
-            params={
-                "storage_location": fdp_req.get_obj_id_from_url(
-                    _storage_location["url"]
-                )
-            },
-        )
+        # A file may be recorded in more than one place: where it is stored,
+        # and where an external object was fetched from
+        _objects = []
+        for _storage_location in _storage_locations:
+            _objects += fdp_req.get(
+                _api_url,
+                "object",
+                _token,
+                params={
+                    "storage_location": fdp_req.get_obj_id_from_url(
+                        _storage_location["url"]
+                    )
+                },
+            )
         for _object in _objects:
             if _object["data_products"]:
                 for _data_product_url in _object["data_products"]:

{fair_cli-0.10.0 → fair_cli-0.10.1}/fair/user_config/__init__.py

@@ -72,24 +72,26 @@ JOB2CLI_MAPPINGS = {
     "run_metadata.write_data_store": "registries.local.data_store",
 }
 
+# The command for each shell is a list of its arguments, so that the path of
+# the script stays one argument when it has a space in it
 SHELLS: typing.Dict[str, str] = {
-    "pwsh": {"exec": "pwsh -command \". '{0}'\"", "extension": "ps1"},
-    "batch": {"exec": "{0}", "extension": "bat"},
+    "pwsh": {"exec": ["pwsh", "-command", ". '{0}'"], "extension": "ps1"},
+    "batch": {"exec": ["{0}"], "extension": "bat"},
     "powershell": {
-        "exec": "powershell -command \". '{0}'\"",
+        "exec": ["powershell", "-command", ". '{0}'"],
         "extension": "ps1",
     },
-    "python2": {"exec": "python2 {0}", "extension": "py"},
-    "python3": {"exec": "python3 {0}", "extension": "py"},
-    "python": {"exec": "python {0}", "extension": "py"},
-    "R": {"exec": "R -f {0}", "extension": "R"},
-    "julia": {"exec": "julia {0}", "extension": "jl"},
+    "python2": {"exec": ["python2", "{0}"], "extension": "py"},
+    "python3": {"exec": ["python3", "{0}"], "extension": "py"},
+    "python": {"exec": ["python", "{0}"], "extension": "py"},
+    "R": {"exec": ["R", "-f", "{0}"], "extension": "R"},
+    "julia": {"exec": ["julia", "{0}"], "extension": "jl"},
     "bash": {
-        "exec": "bash -eo pipefail {0}",
+        "exec": ["bash", "-eo", "pipefail", "{0}"],
         "extension": "sh",
     },
-    "java": {"exec": "java {0}", "extension": "java"},
-    "sh": {"exec": "sh -e {0}", "extension": "sh"},
+    "java": {"exec": ["java", "{0}"], "extension": "java"},
+    "sh": {"exec": ["sh", "-e", "{0}"], "extension": "sh"},
 }
 
 
@@ -416,14 +418,29 @@ class JobConfiguration(MutableMapping):
             )
 
         _search_key = SEARCH_KEYS[_obj_type]
+        _search = {_search_key: block_entry[_obj_type]}
 
         try:
+            # A data product pattern stands for names in one namespace: the
+            # one its entry names, else the default for its block
+            if _obj_type == "data_product":
+                _namespace = block_entry.get("use", {}).get("namespace") or (
+                    self.default_input_namespace
+                    if block_type == "read"
+                    else self.default_output_namespace
+                )
+                _search["namespace"] = fdp_reg.convert_key_value_to_id(
+                    registry_uri, "namespace", _namespace, registry_token
+                )
             _results_local = fdp_req.get(
                 registry_uri,
                 _obj_type,
                 registry_token,
-                params={_search_key: block_entry[_obj_type]},
+                params=_search,
             )
+        except fdp_exc.RegistryError:
+            # The registry has no such namespace, so nothing in it to match
+            _results_local = []
         except fdp_exc.RegistryAPICallError as e:
             raise fdp_exc.UserConfigError(
                 f"Failed to retrieve entries on local registry for {_obj_type}"
@@ -460,6 +477,13 @@ class JobConfiguration(MutableMapping):
                 "version", block_entry.get("version", None)
             )
 
+            # One entry for each name. A read takes the version its entry
+            # names, as a read of a single name does, else the highest; a
+            # write is given its version when versions are filled
+            _results_local = fdp_glob.get_one_version_per_name(
+                _results_local, _version if block_type == "read" else None
+            )
+
             _new_entries = fdp_glob.get_data_product_objects(
                 registry_token, _results_local, block_type, _version
             )
@@ -804,9 +828,15 @@ class JobConfiguration(MutableMapping):
         else:
             self._expand_wildcards(self.local_uri, fdp_req.local_token())
 
+        # What 'pull' fetches is on the remote registry, so that is where the
+        # version of a read is looked for first
+        _remote = (None, None)
+        if job_mode == CMD_MODE.PULL and not local:
+            _remote = (remote_uri, remote_token)
+
         for block_type in self._block_types:
             if block_type in self:
-                self[block_type] = self._fill_versions(block_type)
+                self[block_type] = self._fill_versions(block_type, *_remote)
 
         if job_mode == CMD_MODE.PULL and "register" in self:
             self._logger.debug("Fetching registrations")
@@ -1029,7 +1059,9 @@ class JobConfiguration(MutableMapping):
 
         return _new_config
 
-    def _fill_versions(self, block_type: str) -> typing.List[typing.Dict]:
+    def _fill_versions(
+        self, block_type: str, remote_uri: str = None, remote_token: str = None
+    ) -> typing.List[typing.Dict]:
         self._logger.debug("Filling version information")
         _entries: typing.List[typing.Dict] = []
 
@@ -1075,45 +1107,62 @@ class JobConfiguration(MutableMapping):
             _new_item["use"]["data_product"] = _name
             _namespace = item["use"]["namespace"]
 
-            # If no ID exists for the namespace then this object has not yet
-            # been written to the target registry and so a version number
-            # cannot be deduced this way
-            try:
-                _id_namespace = fdp_reg.convert_key_value_to_id(
-                    self.local_uri,
-                    "namespace",
-                    _namespace,
-                    fdp_req.local_token(),
-                )
-                if "${{" in _version:
+            # A read to be fetched is looked for where it will come from,
+            # and only then in the local registry
+            _registries = [(self.local_uri, fdp_req.local_token())]
+            if block_type == "read" and remote_uri:
+                _registries.insert(0, (remote_uri, remote_token))
+
+            for _uri, _token in _registries:
+                # If no ID exists for the namespace then this object has not
+                # yet been written to the target registry and so a version
+                # number cannot be deduced this way
+                try:
+                    _search = {
+                        "name": _name,
+                        "namespace": fdp_reg.convert_key_value_to_id(
+                            _uri, "namespace", _namespace, _token
+                        ),
+                    }
+                    if "${{" not in _version:
+                        _search["version"] = _version
                     _results = fdp_req.get(
-                        self.local_uri,
-                        "data_product",
-                        fdp_req.local_token(),
-                        params={"name": _name, "namespace": _id_namespace},
+                        _uri, "data_product", _token, params=_search
                     )
-                    if "LATEST" in _version:
-                        _version = fdp_ver.get_latest_version(_results)
-                else:
-                    _results = fdp_req.get(
-                        self.local_uri,
-                        "data_product",
-                        fdp_req.local_token(),
-                        params={
-                            "name": _name,
-                            "namespace": _id_namespace,
-                            "version": _version,
-                        },
-                    )
-            except fdp_exc.RegistryError:
-                _results = []
+                except fdp_exc.RegistryError:
+                    _results = []
+
+                if "*" in _name:
+                    _results = [
+                        result
+                        for result in _results
+                        if fdp_glob.matches_wildcard(_name, result["name"])
+                    ]
+
+                if _results:
+                    break
+
+            # A read of one name must be of something. With nothing found,
+            # the latest version would be 0.0.0, and a version given would
+            # stand unchecked
+            if (
+                block_type == "read"
+                and "*" not in _name
+                and "cache" not in item["use"]
+                and not _results
+            ):
+                _wanted = f"version '{_version}'"
+                if "${{" in _version:
+                    _wanted = "any version"
+                raise fdp_exc.UserConfigError(
+                    f"Cannot read data product '{_name}': {_wanted} of it "
+                    f"was not found in namespace '{_namespace}' on registry "
+                    + " or ".join(f"'{_uri}'" for _uri, _ in _registries),
+                    hint="Has a run written it, or 'fair pull' fetched it?",
+                )
 
-            if "*" in _name:
-                _results = [
-                    result
-                    for result in _results
-                    if fdp_glob.matches_wildcard(_name, result["name"])
-                ]
+            if "LATEST" in _version:
+                _version = fdp_ver.get_latest_version(_results)
 
             try:
                 _version = fdp_ver.get_correct_version(
@@ -1353,14 +1402,16 @@ class JobConfiguration(MutableMapping):
         if not self.env:
             raise fdp_exc.InternalError("Command execution environment setup failed")
 
-        _exec = SHELLS[self.shell]["exec"].format(self.script)
+        _exec = [arg.format(self.script) for arg in SHELLS[self.shell]["exec"]]
 
-        self._logger.debug("Executing command: %s", _exec)
+        self._logger.debug(
+            "Executing command: %s", subprocess.list2cmdline(_exec)
+        )
 
         _log_tail: typing.List[str] = []
 
         _process = subprocess.Popen(
-            _exec.split(),
+            _exec,
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,
             universal_newlines=True,
@@ -1392,7 +1443,7 @@ class JobConfiguration(MutableMapping):
             self.close_log()
             self._logger.error(
                 "Command '%s' failed with exit code %s, log tail:\n\t%s",
-                _exec,
+                subprocess.list2cmdline(_exec),
                 _process.returncode,
                 "\n\t".join(_log_tail),
             )

{fair_cli-0.10.0 → fair_cli-0.10.1}/fair/user_config/globbing.py

@@ -14,9 +14,12 @@ Constants
     - DISPOSABLES: tuple of keys to be removed before adding to config.yaml
 
 """
+import contextlib
 import re
 import typing
 
+import semver
+
 import fair.exceptions as fdp_exc
 import fair.registry.requests as fdp_req
 from fair.registry import SEARCH_KEYS
@@ -62,6 +65,44 @@ def matches_wildcard(pattern: str, name: str) -> bool:
     return re.fullmatch(_regex, name) is not None
 
 
+def get_one_version_per_name(
+    results_list: typing.List[typing.Dict], version: str = None
+) -> typing.List[typing.Dict]:
+    """Return one of the results of a data product search for each name
+
+    Parameters
+    ----------
+    results_list : typing.List[typing.Dict]
+        results of registry search for the wildcard
+    version : str, optional
+        version to take of each name, leaving out a name that has none such;
+        by default, or if this is not a version, the highest of each name
+
+    Returns
+    -------
+    typing.List[typing.Dict]
+        a result for each name, in the order the names first appear
+    """
+    _wanted = None
+    if version:
+        with contextlib.suppress(ValueError):
+            _wanted = semver.VersionInfo.parse(version)
+
+    _versions: typing.Dict[str, semver.VersionInfo] = {}
+    _results: typing.Dict[str, typing.Dict] = {}
+
+    for result in results_list:
+        _name = result[SEARCH_KEYS["data_product"]]
+        _version = semver.VersionInfo.parse(result["version"])
+        if _wanted is not None and _version != _wanted:
+            continue
+        if _name not in _versions or _version > _versions[_name]:
+            _versions[_name] = _version
+            _results[_name] = result
+
+    return list(_results.values())
+
+
 def get_single_layer_objects(
     results_list: typing.List[typing.Dict], object_type: str
 ) -> typing.List[typing.Dict]:

{fair_cli-0.10.0 → fair_cli-0.10.1}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
 ]
 description = "Synchronization interface for the SCRC FAIR Data Pipeline registry"
 name = "fair-cli"
-version = "0.10.0"
+version = "0.10.1"
 
 homepage = "https://www.fairdatapipeline.org/"