fair-cli 0.10.0__tar.gz → 0.10.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. fair_cli-0.10.2/CHANGELOG.md +192 -0
  2. {fair_cli-0.10.0 → fair_cli-0.10.2}/CITATION.cff +2 -2
  3. {fair_cli-0.10.0 → fair_cli-0.10.2}/PKG-INFO +83 -7
  4. {fair_cli-0.10.0 → fair_cli-0.10.2}/README.md +77 -3
  5. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/identifiers.py +53 -3
  6. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/register.py +50 -32
  7. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/registry/requests.py +83 -7
  8. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/registry/server.py +31 -7
  9. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/registry/storage.py +206 -9
  10. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/registry/sync.py +304 -84
  11. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/session.py +22 -14
  12. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/user_config/__init__.py +105 -54
  13. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/user_config/globbing.py +41 -0
  14. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/user_config/validation.py +5 -0
  15. fair_cli-0.10.2/fair/virtualenv.py +141 -0
  16. {fair_cli-0.10.0 → fair_cli-0.10.2}/pyproject.toml +2 -5
  17. fair_cli-0.10.0/CHANGELOG.md +0 -86
  18. fair_cli-0.10.0/fair/virtualenv.py +0 -45
  19. {fair_cli-0.10.0 → fair_cli-0.10.2}/LICENSE +0 -0
  20. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/__init__.py +0 -0
  21. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/cli.py +0 -0
  22. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/common.py +0 -0
  23. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/configuration/__init__.py +0 -0
  24. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/configuration/validation.py +0 -0
  25. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/exceptions.py +0 -0
  26. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/files.txt +0 -0
  27. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/history.py +0 -0
  28. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/logging.py +0 -0
  29. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/registry/__init__.py +0 -0
  30. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/registry/file_formats.json +0 -0
  31. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/registry/file_types.py +0 -0
  32. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/registry/versioning.py +0 -0
  33. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/run.py +0 -0
  34. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/staging.py +0 -0
  35. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/templates/__init__.py +0 -0
  36. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/templates/config.jinja +0 -0
  37. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/templates/hist.jinja +0 -0
  38. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/testing.py +0 -0
  39. {fair_cli-0.10.0 → fair_cli-0.10.2}/fair/utilities.py +0 -0
fair_cli-0.10.2/CHANGELOG.md ADDED
@@ -0,0 +1,192 @@
1
+ # 2026-10-05 [v0.10.2](https://github.com/FAIRDataPipeline/FAIR-CLI/releases/tag/v0.10.2)
2
+
3
+ ## Changed behaviour
4
+ - `fair pull` refuses a `register:` entry for an external object whose identifier or unique name, title and
5
+ release version are those of a source already registered from a different file: a source is one file. Give
6
+ the entry a `title` of its own, or its `release_version`; the same file under another name still registers.
7
+ With a local registry before data-registry v1.4.0, where each data product has its own source, nothing is
8
+ checked.
9
+ - `fair init` in a repository that is already initialised registers its user in the local registry, where it
10
+ did nothing. A registry reinstalled since the repository was initialised has no record of the user, and a
11
+ run failed until the repository was purged and initialised afresh. The repository itself is left as it is.
12
+
13
+ ## Added
14
+ - A `register:` entry for an external object may give `release_version`, the version of the source it was taken
15
+ from, beside `release_date`. A local registry at data-registry v1.4.0 or later records it as the external
16
+ object's version, 1.0.0 without it; an earlier one records the data product's version there whatever is
17
+ given. It is not the entry's `version`, which is the data product's.
18
+
19
+ ## Fixed
20
+ - A GitHub author or user was refused as "not a recognised github" once the GitHub API's limit for
21
+ anonymous requests (60 an hour per address) was spent. GitHub lookups now send the token in `GITHUB_TOKEN`
22
+ or `GITHUB_PAT` if set, retry without it if GitHub refuses it (expired or revoked), and say when the rate
23
+ limit is the problem.
24
+ - The FAIR-CLI binaries could not install a local registry, so `fair init` and `fair registry install` failed:
25
+ the registry's virtual environment was built from the binary in place of a Python. They now use a Python
26
+ >= 3.10 from `FAIR_PYTHON` or the `PATH`, or have `uv` fetch one, and say so if there is none. A missing
27
+ Python is found before anything is installed.
28
+ - `fair push` failed, with "Failed to access [] on remote registry", for a data product registered from a web
29
+ address once a code run had used it, when the local registry was data-registry v1.4.0. The remote records
30
+ such a file in two places with one hash - where it is stored and where it came from - and the wrong one
31
+ could be taken.
32
+ - `fair registry install --force` refused an existing install, as it does without `--force`, instead of
33
+ replacing it.
34
+ - A file that was already registered was not registered again under a second data product name, or in a
35
+ second namespace: `fair pull` passed over the entry without a word, and a run that read it then failed. An
36
+ entry is now passed over only when that data product, in that namespace and at that version, already holds
37
+ the file.
38
+ - `fair push` and `fair pull` left out a data product that shares its external object with another, as
39
+ data-registry v1.4.0 allows, while reporting it synchronised: its file was moved and its record was not made.
40
+ - Starting the local registry reported success when another registry already held its port, and the commands
41
+ that followed went to that one. The start now fails, naming the address: the server that answers there
42
+ must accept the installed registry's token.
43
+ - `fair push` and `fair pull` stopped with a Python error at a data product that has no file - one whose
44
+ object has no storage location, as an entry for a deposit as a whole may be - and so at a code run that
45
+ read one. Its records are now synchronised, and a `register:` entry of the same name is told it exists.
46
+ - `fair push` gave a code run an input it did not read when two data products held the same file, as one
47
+ file registered under two names does: a run that read one of them and wrote something new arrived on the
48
+ remote as having read both. A run pushed before this keeps the extra input.
49
+ - A file registered under a second data product name, or in a second namespace, was copied into the data store
50
+ again, though the registry records a file once there and nothing referred to the copy. The copy is no
51
+ longer kept: the data product's file is the one already in the store.
52
+ - A data product pulled from a remote could not be read through an API. `fair pull` copied the remote's record
53
+ of where the file is - the remote's own store - and fetched the file to a place no record named, so a model
54
+ was handed an address and no file. A pulled file is now recorded at its place in the local data store, and
55
+ a file that two data products share is fetched once. Data products pulled with an earlier version keep
56
+ the old record.
57
+
58
+ ## Development
59
+ - The test of a data product's dependencies allows for any a registry adds; data-registry v1.4.0 adds one.
60
+
61
+ # 2026-10-05 [v0.10.1](https://github.com/FAIRDataPipeline/FAIR-CLI/releases/tag/v0.10.1)
62
+
63
+ ## Changed behaviour
64
+ - A wildcard entry keeps to one namespace: the one its `use:` names, else the default for its block. It
65
+ used to match names in every namespace, so a `write:` pattern could write a name into a namespace
66
+ other than its own.
67
+ - A wildcard entry gives one entry for each name it matches: for a `read:`, the version its `use:` names,
68
+ else the highest; for a `write:`, the version to be written. It used to give one for every version, and
69
+ to ignore a version on a `read:`. A `read:` pattern that matches nothing is not an error.
70
+ - `fair run` refuses a `read:` of a data product, or of a version of one, that is not in the registry,
71
+ naming the namespace and registry it looked in. Such a read used to pass unchecked, or as version 0.0.0.
72
+ - `fair push` fails if a file cannot be uploaded. It used to warn and carry on, leaving the remote with
73
+ the file's records and no file. A file is now uploaded before its records are written, so a push that
74
+ failed can be run again.
75
+ - `fair pull` checks each file it fetches against the hash the remote registry holds for it, and refuses
76
+ one that differs.
77
+
78
+ ## Added
79
+ - Registering an external object records where its file was fetched from: the `root` and `path` of its
80
+ `register:` entry, as the object's `original_store`, with the hash of the file found there. `fair push`
81
+ takes that record to the remote. A file registered from the machine itself has no such record.
82
+ - `fair push` takes a refusal of an upload address with HTTP 409 to mean that the remote's store already
83
+ holds the file, sends nothing, and carries on. No released registry answers so yet.
84
+
85
+ ## Fixed
86
+ - `fair push` sent no value that was false, leaving the remote to apply its default: an external object
87
+ registered with `primary: false` arrived as primary, and a location with `public: false` as public.
88
+ - A list from a registry stopped at its first 100 rows, so wildcards, staging and version lookups went
89
+ wrong, with no message, in a registry holding more.
90
+ - `fair pull` looked for the version of a `read:` entry in the local registry only, so an entry at the
91
+ default version, for something not yet local, asked the remote for version 0.0.0 and failed.
92
+ - `fair push` held each file whole in memory, `fair pull` more than twice over, and both left a copy of
93
+ it in the temporary directory. Files are now sent and received a block at a time, whatever their
94
+ size, and nothing is left behind.
95
+ - An upload of 2 GiB or more over plain http failed on macOS.
96
+ - An external object could not be registered with a `unique_name` and `alternate_identifier_type` in
97
+ place of an `identifier`: `fair pull` stopped, reporting that the object already existed.
98
+ - `fair identify` looked at only the newest of a file's storage locations, so it could miss the data
99
+ products of a file the registry records in more than one place.
100
+ - `fair run` could not start the script when the path of the data store, which holds the job
101
+ directory, had a space in it.
102
+
103
+ ## Development
104
+ - The Python job of the implementations workflow starts an object store for its remote registry, so
105
+ that its `fair push` uploads the files. It had none, and passed while a failed upload was a warning.
106
+
107
+ # 2026-10-03 [v0.10.0](https://github.com/FAIRDataPipeline/FAIR-CLI/releases/tag/v0.10.0)
108
+
109
+ ## Changed behaviour
110
+ - `fair remote add` takes the remote's token file with `--token FILE`, and prompts for it if omitted.
111
+ The command never worked before: the label was stored as the token path.
112
+ - `fair run --ci` prepares the job directory and prints it without running the script, as documented.
113
+ - `fair init --ci` leaves an existing `.fair/` alone, as `fair init` does; it used to delete it, data
114
+ store included. Use `fair purge` to start afresh.
115
+ - Wildcards: each `*` matches one segment of a name (not `/`), anchored at both ends, in `read:` and
116
+ `write:` alike, matching DataPipeline.jl. `data_product/*` no longer matches `data_product/1/2`.
117
+ - `${{GIT_TAG}}` is the nearest tag in the history of the current commit (`git describe`), not the
118
+ last tag by name; with no such tag it is an error rather than a crash.
119
+ - `run_metadata.shell` now chooses the interpreter the script runs with.
120
+
121
+ ## Fixed
122
+ - `write:` entries with a wildcard were dropped from the working config (#140). The pattern is now kept
123
+ for new names, and existing matches are written as the pattern entry describes them (its `use: version`,
124
+ `file_type`, `description`).
125
+ - `fair push` from a project whose data store was not the local registry's first storage root recorded
126
+ its files on the remote under the pusher's `file://` path (data-registry #229).
127
+ - `fair pull` of a `read:` entry from a remote registry failed.
128
+ - A DOI whose publisher refuses automated requests (e.g. Wiley, HTTP 403) was rejected as an invalid
129
+ identifier. An identifier that resolves is now accepted; one that does not (404) is still refused.
130
+ - `${{DATETIME-<format>}}` always failed with "Failed to parse formatted datetime variable".
131
+ - Substituting a path with backslashes, such as `${{CONFIG_DIR}}` on Windows, failed `fair pull` and `fair run`
132
+ (#267). Variables are now substituted into the configuration's values, so any text is inserted as it is.
133
+ - `fair remote remove` crashed.
134
+ - A ROR or GRID ID the ROR API could not parse (e.g. one containing `!`) crashed with `KeyError`
135
+ instead of being reported as not found.
136
+ - Run from a subdirectory of a project, commands lost configuration changes and left a stale session
137
+ file, so the next command did not start the registry and `fair registry stop` needed `--force`.
138
+ - Staging ignored the configured local registry and always used port 8000.
139
+ - A download that failed with an HTTP error registered the error page as the data.
140
+ - `fair run` could crash on Windows when the model printed a character the console cannot encode.
141
+ - `fair run` in a git repository without the configured remote gave a bare `IndexError`; it now says
142
+ which remote is missing, or reads `run_metadata: remote_repo:` if given.
143
+ - `fair init` in a subdirectory of a FAIR repository named it as `'True'`.
144
+
145
+ ## Development
146
+ - CI ran no tests: an unused dev dependency (`pylama`) crashed `pytest --markers`, leaving the test loop
147
+ empty. It is removed, and a failure to list the tests now fails the job; all 8 jobs run the full suite.
148
+ - numpy is chosen per Python version (>= 2.3 from 3.11), so Python 3.14 gets a wheel, not a source build.
149
+ - Download tests use a local HTTP server rather than github.com.
150
+ - `test_configuration.py::test_local_config_query` is marked `xfail`: it has never passed, and its intent
151
+ is unknown.
152
+
153
+ # 2026-09-22 [v0.9.9](https://github.com/FAIRDataPipeline/FAIR-CLI/releases/tag/v0.9.9)
154
+ - Requires Python >= 3.10, in step with data-registry v1.2.2.
155
+ - `${{RUN_ID}}` in a `write:` name is left for the language API to fill in at `finalise`.
156
+ - ROR/GRID organisation lookups failed after the ROR API moved to its v2 schema.
157
+
158
+ # Between v0.2.3 and v0.9.8
159
+ Not recorded release by release; see the [releases](https://github.com/FAIRDataPipeline/FAIR-CLI/releases)
160
+ and the git log.
161
+ - Fix virtual environment bug in `fair` binaries where `venv` assumes the main executable is `python`.
162
+ - Registration of `author` and `namespace` objects.
163
+ - Allow specification of tag to install registry from during `fair registry install`.
164
+ - Wildcard '*' parsing introduced for data products.
165
+ - Ability to `push` to a registry added.
166
+ - Added `--dirty` option to `fair run` to allow running with uncommitted changes.
167
+ - Added `config.yaml` file validation.
168
+ - Added initialisation from existing registry.
169
+ - Switch to setting port not local URI during initialisation.
170
+ - Added option to specify port on `fair registry start`.
171
+ - Added `cli-config.yaml` file validation during initialisation.
172
+ - Added `fair reset` to list of commands.
173
+ - Add tab completion for staging data products.
174
+
175
+ # 2021-11-17 [v0.2.3](https://github.com/FAIRDataPipeline/FAIR-CLI/releases/tag/v0.2.3)
176
+ - Move handling of the user `config.yaml` file to a separate class `JobConfiguration`.
177
+ - Added various fixes to improve functionality within Windows.
178
+ - Move registry installation from script execution to internal function which sets up virtual environment etc.
179
+ - Added a test suite to the project.
180
+ - Added additional recognised identifiers for author setup from an organisation GRID and ROR.
181
+
182
+ # 2021-10-06 [v0.2.2](https://github.com/FAIRDataPipeline/FAIR-CLI/releases/tag/v0.2.2)
183
+ - Update to package metadata for PyPi
184
+
185
+ # 2021-10-06 [v0.2.1](https://github.com/FAIRDataPipeline/FAIR-CLI/releases/tag/v0.2.1)
186
+ - Automatic starter `config.yaml` generation.
187
+ - Local and Global CLI configurations.
188
+ - Start/stop local registry either explicitly or during synchronisations.
189
+ - Run logs available via git-like interface.
190
+ - Added ability to add/remove files.
191
+ - Repository style handling, acts like another git-like tool per project.
192
+ - Creation of an interface for `fair` using `click`.
{fair_cli-0.10.0 → fair_cli-0.10.2}/CITATION.cff RENAMED
@@ -26,7 +26,7 @@ authors:
26
26
  affiliation: University of Glasgow
27
27
  orcid: https://orcid.org/0000-0002-4424-9890
28
28
 
29
- date-released: 2026-10-03
29
+ date-released: 2026-10-05
30
30
  keywords:
31
31
  - FAIR Data Pipeline
32
32
  - FAIR
@@ -36,4 +36,4 @@ license: BSD-2-Clause
36
36
  message: If you use this software, please cite it using these metadata.
37
37
  repository-code: https://github.com/FAIRDataPipeline/FAIR-CLI/
38
38
  title: "The FAIR Data Pipeline command line tool"
39
- version: 0.10.0
39
+ version: 0.10.2
{fair_cli-0.10.0 → fair_cli-0.10.2}/PKG-INFO RENAMED
@@ -1,9 +1,9 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: fair-cli
3
- Version: 0.10.0
3
+ Version: 0.10.2
4
4
  Summary: Synchronization interface for the SCRC FAIR Data Pipeline registry
5
- Home-page: https://www.fairdatapipeline.org/
6
5
  License: BSD-2-Clause
6
+ License-File: LICENSE
7
7
  Keywords: FAIR Data Pipeline,FAIR,Data Management,Provenance
8
8
  Author: Richard Reeve
9
9
  Author-email: richard.reeve@glasgow.ac.uk
@@ -20,9 +20,10 @@ Classifier: Programming Language :: Python :: 3.10
20
20
  Classifier: Programming Language :: Python :: 3.11
21
21
  Classifier: Programming Language :: Python :: 3.12
22
22
  Classifier: Programming Language :: Python :: 3.13
23
+ Classifier: Programming Language :: Python :: 3.14
24
+ Classifier: Programming Language :: Python :: 3.15
23
25
  Classifier: Topic :: Database :: Front-Ends
24
26
  Classifier: Topic :: Scientific/Engineering
25
- Provides-Extra: all
26
27
  Requires-Dist: GitPython (>=3.1.18,<4.0.0)
27
28
  Requires-Dist: Jinja2 (>=3.0.1,<4.0.0)
28
29
  Requires-Dist: PyYAML (>=5.4.1,<7.0.0)
@@ -38,6 +39,7 @@ Requires-Dist: simplejson (>=3.17.5,<4.0.0)
38
39
  Requires-Dist: toml (>=0.10.2,<0.11.0)
39
40
  Requires-Dist: validators (>=0.18.2,<0.19.0)
40
41
  Project-URL: Documentation, https://www.fairdatapipeline.org/docs/interface/fdp/
42
+ Project-URL: Homepage, https://www.fairdatapipeline.org/
41
43
  Project-URL: Issue Tracker, https://github.com/FAIRDataPipeline/FAIR-CLI/issues
42
44
  Project-URL: Repository, https://github.com/FAIRDataPipeline/FAIR-CLI
43
45
  Description-Content-Type: text/markdown
@@ -58,6 +60,8 @@ The package is installed using Pip:
58
60
  pip install fair-cli
59
61
  ```
60
62
 
63
+ It needs Python 3.10 or later. A virtual environment is recommended: the local registry is installed with the same Python as the CLI, in an environment of its own.
64
+
61
65
  To enable tab completion you need to modify your shell:
62
66
 
63
67
  ### Bash
@@ -78,12 +82,81 @@ _FAIR_COMPLETE=bash_source fair > ~/.config/fish/.fair-complete.fish
78
82
  echo '. ~/.config/fish/.fair-complete.fish' >> ~/.bashrc
79
83
  ```
80
84
 
85
+ ## Upgrading and reinstalling
86
+
87
+ The CLI and the local registry are installed separately - the CLI by `pip`, the registry by the CLI - so upgrading one does not upgrade the other.
88
+
89
+ ### The CLI
90
+
91
+ To move to the newest release, or to a particular one:
92
+
93
+ ```sh
94
+ pip install --upgrade fair-cli
95
+ pip install fair-cli==0.10.1
96
+ ```
97
+
98
+ To install a branch of this repository, give `pip` its name after an `@`. A tag or a commit works the same way:
99
+
100
+ ```sh
101
+ pip install git+https://github.com/FAIRDataPipeline/FAIR-CLI@<branch>
102
+ ```
103
+
104
+ or clone the repository, check the branch out and install from the clone:
105
+
106
+ ```sh
107
+ git clone -b <branch> https://github.com/FAIRDataPipeline/FAIR-CLI
108
+ pip install ./FAIR-CLI
109
+ ```
110
+
111
+ A branch keeps its version number from one commit to the next, and `pip` does not replace an installed package with another of the same version. To bring an installed branch up to date, or to install one branch over another of the same version, add `--force-reinstall --no-deps`:
112
+
113
+ ```sh
114
+ pip install --force-reinstall --no-deps git+https://github.com/FAIRDataPipeline/FAIR-CLI@<branch>
115
+ ```
116
+
117
+ `fair --version` gives the version number alone, which a branch may share with a release. `pip freeze` says what is installed: `fair-cli==0.10.1` for a release, and `fair-cli @ git+https://github.com/FAIRDataPipeline/FAIR-CLI@<commit>` for a branch.
118
+
119
+ ### The local registry
120
+
121
+ To replace the local registry with the newest release of it, stop it if it is running, install with `--force`, and start it again:
122
+
123
+ ```sh
124
+ fair registry stop
125
+ fair registry install --force
126
+ fair registry start
127
+ ```
128
+
129
+ `--version` installs a tag or a branch of the [registry's repository](https://github.com/FAIRDataPipeline/data-registry) in place of the newest release:
130
+
131
+ ```sh
132
+ fair registry install --force --version v1.3.0
133
+ fair registry install --force --version <branch>
134
+ ```
135
+
136
+ Without `--force` an existing registry is left as it is and the command stops. `fair registry uninstall`, which asks first, followed by `fair registry install` does the same in two steps, and is the way with `fair-cli` 0.10.1 or earlier, where `--force` is refused.
137
+
138
+ Check the name given to `--version` before using it with `--force`: the old registry is removed before the name is looked for, so one that does not exist leaves no working registry until the command is run again with one that does.
139
+
140
+ The registry is installed in `~/.fair/registry`, or in the directory given with `--directory`, which has to be given again to replace a registry that was installed elsewhere. A reinstall replaces that directory:
141
+
142
+ - every record in the local registry goes with its database: the data products, the code runs, and what `fair init` put there;
143
+ - the local registry has a new token, written to `token` in its directory when it is next started, and anything else that was kept in that directory is gone;
144
+ - the data store, the projects and `~/.fair/cli` are not touched.
145
+
146
+ The new registry does not know the user of a project that was initialised before the reinstall, and a run in that project fails until it does. There are two ways to put that right:
147
+
148
+ - run `fair init` in the project again, which leaves the project as it is - its configuration and any data store kept inside it - and registers its user in the new registry. This needs `fair-cli` 0.10.2 or later: before that, `fair init` does nothing in a project that is already initialised;
149
+ - or start the project afresh: run `fair purge` in it, which removes its `.fair` folder and any data store kept inside it, and then `fair init`.
150
+
151
+ After either, `fair pull` registers again what the project registers or reads. What had been pushed to a remote registry can be pulled back from it. The records of anything that had not been pushed cannot be recovered, though its files are still in the data store unless that was removed with the project's `.fair` folder.
152
+
81
153
  ## Uninstallation
82
154
  To uninstall the CLI run:
83
155
  ```
84
156
  fair purge --all
85
- pip uninstall fair
157
+ pip uninstall fair-cli
86
158
  ```
159
+ `fair purge --all` removes the `.fair` folder of the current project and the whole of `~/.fair`: the local registry, the default data store, the CLI's configuration and any token kept there. Leave it out to keep them.
87
160
 
88
161
  ## The User Configuration File
89
162
  Job runs are configured via `config.yaml` files. Upon initialisation of a project, FAIR-CLI automatically generates a starter configuration file with all requirements in place. To execute a process (e.g. perform a model run from a compiled binary/script) an additional key of either `script` or `script_path` must be provided. Alternatively the command `fair run bash` can be used to append the key and run a command directly.
@@ -112,6 +185,8 @@ A full description of `config.yaml` files can be found [here](https://www.fairda
112
185
 
113
186
  Initialises a new FAIR repository within the given directory. This should ideally be the same location as the `.git` folder for the current project, however during setup an option is given to specify an alternative. The command will ask the user a series of questions which will provide metadata for tracking run authors, and also allow for the creation of a starter `config.yaml` file. Initialisation will also configure the CLI itself.
114
187
 
188
+ In a repository that is already initialised the command asks nothing and leaves the repository as it is. It registers the repository's user in the local registry, which a registry [reinstalled](#upgrading-and-reinstalling) since the repository was initialised no longer holds.
189
+
115
190
  #### Custom CLI Configuration
116
191
  After setup is complete, the current CLI configuration can also be saved using the command:
117
192
  ```
@@ -270,10 +345,11 @@ The registry can be installed using the CLI as well by running:
270
345
  ```sh
271
346
  fair registry install
272
347
  ```
273
- with the additional options to specify the installation location, and the data registry repository tag to install from:
348
+ with the additional options to specify the installation location, and the data registry repository tag or branch to install from:
274
349
  ```sh
275
- fair registry install --directory ~/.fair/my_registry --version v1.0-rc5
350
+ fair registry install --directory ~/.fair/my_registry --version v1.4.0
276
351
  ```
352
+ To replace a registry that is already installed, see [Upgrading and reinstalling](#upgrading-and-reinstalling).
277
353
 
278
354
  ### `log`
279
355
 
{fair_cli-0.10.0 → fair_cli-0.10.2}/README.md RENAMED
@@ -14,6 +14,8 @@ The package is installed using Pip:
14
14
  pip install fair-cli
15
15
  ```
16
16
 
17
+ It needs Python 3.10 or later. A virtual environment is recommended: the local registry is installed with the same Python as the CLI, in an environment of its own.
18
+
17
19
  To enable tab completion you need to modify your shell:
18
20
 
19
21
  ### Bash
@@ -34,12 +36,81 @@ _FAIR_COMPLETE=bash_source fair > ~/.config/fish/.fair-complete.fish
34
36
  echo '. ~/.config/fish/.fair-complete.fish' >> ~/.bashrc
35
37
  ```
36
38
 
39
+ ## Upgrading and reinstalling
40
+
41
+ The CLI and the local registry are installed separately - the CLI by `pip`, the registry by the CLI - so upgrading one does not upgrade the other.
42
+
43
+ ### The CLI
44
+
45
+ To move to the newest release, or to a particular one:
46
+
47
+ ```sh
48
+ pip install --upgrade fair-cli
49
+ pip install fair-cli==0.10.1
50
+ ```
51
+
52
+ To install a branch of this repository, give `pip` its name after an `@`. A tag or a commit works the same way:
53
+
54
+ ```sh
55
+ pip install git+https://github.com/FAIRDataPipeline/FAIR-CLI@<branch>
56
+ ```
57
+
58
+ or clone the repository, check the branch out and install from the clone:
59
+
60
+ ```sh
61
+ git clone -b <branch> https://github.com/FAIRDataPipeline/FAIR-CLI
62
+ pip install ./FAIR-CLI
63
+ ```
64
+
65
+ A branch keeps its version number from one commit to the next, and `pip` does not replace an installed package with another of the same version. To bring an installed branch up to date, or to install one branch over another of the same version, add `--force-reinstall --no-deps`:
66
+
67
+ ```sh
68
+ pip install --force-reinstall --no-deps git+https://github.com/FAIRDataPipeline/FAIR-CLI@<branch>
69
+ ```
70
+
71
+ `fair --version` gives the version number alone, which a branch may share with a release. `pip freeze` says what is installed: `fair-cli==0.10.1` for a release, and `fair-cli @ git+https://github.com/FAIRDataPipeline/FAIR-CLI@<commit>` for a branch.
72
+
73
+ ### The local registry
74
+
75
+ To replace the local registry with the newest release of it, stop it if it is running, install with `--force`, and start it again:
76
+
77
+ ```sh
78
+ fair registry stop
79
+ fair registry install --force
80
+ fair registry start
81
+ ```
82
+
83
+ `--version` installs a tag or a branch of the [registry's repository](https://github.com/FAIRDataPipeline/data-registry) in place of the newest release:
84
+
85
+ ```sh
86
+ fair registry install --force --version v1.3.0
87
+ fair registry install --force --version <branch>
88
+ ```
89
+
90
+ Without `--force` an existing registry is left as it is and the command stops. `fair registry uninstall`, which asks first, followed by `fair registry install` does the same in two steps, and is the way with `fair-cli` 0.10.1 or earlier, where `--force` is refused.
91
+
92
+ Check the name given to `--version` before using it with `--force`: the old registry is removed before the name is looked for, so one that does not exist leaves no working registry until the command is run again with one that does.
93
+
94
+ The registry is installed in `~/.fair/registry`, or in the directory given with `--directory`, which has to be given again to replace a registry that was installed elsewhere. A reinstall replaces that directory:
95
+
96
+ - every record in the local registry goes with its database: the data products, the code runs, and what `fair init` put there;
97
+ - the local registry has a new token, written to `token` in its directory when it is next started, and anything else that was kept in that directory is gone;
98
+ - the data store, the projects and `~/.fair/cli` are not touched.
99
+
100
+ The new registry does not know the user of a project that was initialised before the reinstall, and a run in that project fails until it does. There are two ways to put that right:
101
+
102
+ - run `fair init` in the project again, which leaves the project as it is - its configuration and any data store kept inside it - and registers its user in the new registry. This needs `fair-cli` 0.10.2 or later: before that, `fair init` does nothing in a project that is already initialised;
103
+ - or start the project afresh: run `fair purge` in it, which removes its `.fair` folder and any data store kept inside it, and then `fair init`.
104
+
105
+ After either, `fair pull` registers again what the project registers or reads. What had been pushed to a remote registry can be pulled back from it. The records of anything that had not been pushed cannot be recovered, though its files are still in the data store unless that was removed with the project's `.fair` folder.
106
+
37
107
  ## Uninstallation
38
108
  To uninstall the CLI run:
39
109
  ```
40
110
  fair purge --all
41
- pip uninstall fair
111
+ pip uninstall fair-cli
42
112
  ```
113
+ `fair purge --all` removes the `.fair` folder of the current project and the whole of `~/.fair`: the local registry, the default data store, the CLI's configuration and any token kept there. Leave it out to keep them.
43
114
 
44
115
  ## The User Configuration File
45
116
  Job runs are configured via `config.yaml` files. Upon initialisation of a project, FAIR-CLI automatically generates a starter configuration file with all requirements in place. To execute a process (e.g. perform a model run from a compiled binary/script) an additional key of either `script` or `script_path` must be provided. Alternatively the command `fair run bash` can be used to append the key and run a command directly.
@@ -68,6 +139,8 @@ A full description of `config.yaml` files can be found [here](https://www.fairda
68
139
 
69
140
  Initialises a new FAIR repository within the given directory. This should ideally be the same location as the `.git` folder for the current project, however during setup an option is given to specify an alternative. The command will ask the user a series of questions which will provide metadata for tracking run authors, and also allow for the creation of a starter `config.yaml` file. Initialisation will also configure the CLI itself.
70
141
 
142
+ In a repository that is already initialised the command asks nothing and leaves the repository as it is. It registers the repository's user in the local registry, which a registry [reinstalled](#upgrading-and-reinstalling) since the repository was initialised no longer holds.
143
+
71
144
  #### Custom CLI Configuration
72
145
  After setup is complete, the current CLI configuration can also be saved using the command:
73
146
  ```
@@ -226,10 +299,11 @@ The registry can be installed using the CLI as well by running:
226
299
  ```sh
227
300
  fair registry install
228
301
  ```
229
- with the additional options to specify the installation location, and the data registry repository tag to install from:
302
+ with the additional options to specify the installation location, and the data registry repository tag or branch to install from:
230
303
  ```sh
231
- fair registry install --directory ~/.fair/my_registry --version v1.0-rc5
304
+ fair registry install --directory ~/.fair/my_registry --version v1.4.0
232
305
  ```
306
+ To replace a registry that is already installed, see [Upgrading and reinstalling](#upgrading-and-reinstalling).
233
307
 
234
308
  ### `log`
235
309
 
{fair_cli-0.10.0 → fair_cli-0.10.2}/fair/identifiers.py RENAMED
@@ -18,6 +18,7 @@ Functions
18
18
 
19
19
  __date__ = "2021-07-01"
20
20
 
21
+ import os
21
22
  import time
22
23
  import typing
23
24
  import urllib.parse
@@ -28,6 +29,8 @@ import logging
28
29
  from urllib3.exceptions import InsecureRequestWarning
29
30
  from fake_useragent import UserAgent
30
31
 
32
+ import fair.exceptions as fdp_exc
33
+
31
34
  logger = logging.getLogger("FAIRDataPipeline.Identifiers")
32
35
 
33
36
  JSON_MIME_TYPE = "application/json"
@@ -98,10 +101,28 @@ def check_github(github: str) -> typing.Dict:
98
101
  typing.Dict
99
102
  metadata from the given ID
100
103
  """
101
- _header = JSON_HEADERS
102
104
  _url = urllib.parse.urljoin(QUERY_URLS["github"], github)
103
105
  requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
104
- _response = requests.get(_url, headers=_header, verify=False, allow_redirects=True)
106
+
107
+ # Requests without a token share a limit of 60 an hour per IP address
108
+ _token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GITHUB_PAT")
109
+ _auth = {"Authorization": f"Bearer {_token}"} if _token else {}
110
+ _response = requests.get(
111
+ _url, headers={**JSON_HEADERS, **_auth}, verify=False, allow_redirects=True
112
+ )
113
+
114
+ # An expired or revoked token is refused outright, where no token is not
115
+ if _response.status_code == 401 and _token:
116
+ logger.warning(
117
+ "GitHub refused the token in GITHUB_TOKEN/GITHUB_PAT, "
118
+ "retrying without it"
119
+ )
120
+ _auth = {}
121
+ _response = requests.get(
122
+ _url, headers=JSON_HEADERS, verify=False, allow_redirects=True
123
+ )
124
+
125
+ _check_github_rate_limit(_response, bool(_auth))
105
126
 
106
127
  _result_dict: typing.Dict[str, typing.Any] = {}
107
128
 
@@ -109,8 +130,9 @@ def check_github(github: str) -> typing.Dict:
109
130
  time.sleep(3)
110
131
  _header = {"Accept": JSON_MIME_TYPE, "User-Agent": str(UserAgent().chrome)}
111
132
  _response = requests.get(
112
- _url, headers=_header, verify=False, allow_redirects=True
133
+ _url, headers={**_header, **_auth}, verify=False, allow_redirects=True
113
134
  )
135
+ _check_github_rate_limit(_response, bool(_auth))
114
136
 
115
137
  if _response.status_code != 200:
116
138
  logger.debug(f"{_url} Responded with {_response.status_code}")
@@ -129,6 +151,34 @@ def check_github(github: str) -> typing.Dict:
129
151
  return _result_dict
130
152
 
131
153
 
154
+ def _check_github_rate_limit(response: requests.Response, token: bool) -> None:
155
+ """Raise if the GitHub API refused a request for exceeding its rate limit
156
+
157
+ Parameters
158
+ ----------
159
+ response : requests.Response
160
+ response from the GitHub API
161
+ token : bool
162
+ whether the request was sent with a token GitHub accepted
163
+ """
164
+ if response.status_code not in (403, 429):
165
+ return
166
+ if response.headers.get("X-RateLimit-Remaining") != "0":
167
+ return
168
+ _reset = response.headers.get("X-RateLimit-Reset")
169
+ _when = (
170
+ f" until {time.strftime('%H:%M:%S', time.localtime(int(_reset)))}"
171
+ if _reset and _reset.isdigit()
172
+ else ""
173
+ )
174
+ raise fdp_exc.FAIRCLIException(
175
+ f"The GitHub API rate limit is exhausted{_when}, so GitHub usernames "
176
+ "cannot be checked",
177
+ hint="Wait for the limit to reset"
178
+ + ("" if token else ", or set GITHUB_TOKEN to a valid GitHub token"),
179
+ )
180
+
181
+
132
182
  def check_gitlab(gitlab: str, gitlab_url: str = "https://gitlab.com/") -> typing.Dict:
133
183
  """Checks if valid GitLab Username using Gitlab profile address
134
184