fair-cli 0.9.4__tar.gz → 0.9.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fair_cli-0.9.4 → fair_cli-0.9.6}/CITATION.cff +2 -2
- {fair_cli-0.9.4 → fair_cli-0.9.6}/PKG-INFO +2 -2
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/configuration/__init__.py +64 -19
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/configuration/validation.py +2 -2
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/identifiers.py +38 -1
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/registry/requests.py +66 -2
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/registry/server.py +6 -6
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/registry/sync.py +51 -53
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/session.py +6 -8
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/testing.py +1 -1
- {fair_cli-0.9.4 → fair_cli-0.9.6}/pyproject.toml +10 -11
- {fair_cli-0.9.4 → fair_cli-0.9.6}/CHANGELOG.md +0 -0
- {fair_cli-0.9.4 → fair_cli-0.9.6}/LICENSE +0 -0
- {fair_cli-0.9.4 → fair_cli-0.9.6}/README.md +0 -0
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/__init__.py +0 -0
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/cli.py +0 -0
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/common.py +0 -0
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/exceptions.py +0 -0
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/files.txt +0 -0
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/history.py +0 -0
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/logging.py +0 -0
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/register.py +0 -0
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/registry/__init__.py +0 -0
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/registry/file_formats.json +0 -0
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/registry/file_types.py +0 -0
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/registry/storage.py +0 -0
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/registry/versioning.py +0 -0
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/run.py +0 -0
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/staging.py +0 -0
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/templates/__init__.py +0 -0
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/templates/config.jinja +0 -0
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/templates/hist.jinja +0 -0
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/user_config/__init__.py +0 -0
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/user_config/globbing.py +0 -0
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/user_config/validation.py +0 -0
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/utilities.py +0 -0
- {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/virtualenv.py +0 -0
|
@@ -26,7 +26,7 @@ authors:
|
|
|
26
26
|
affiliation: University of Glasgow
|
|
27
27
|
orcid: https://orcid.org/0000-0002-4424-9890
|
|
28
28
|
|
|
29
|
-
date-released:
|
|
29
|
+
date-released: 2024-08-04
|
|
30
30
|
keywords:
|
|
31
31
|
- FAIR Data Pipeline
|
|
32
32
|
- FAIR
|
|
@@ -36,4 +36,4 @@ license: BSD-2-Clause
|
|
|
36
36
|
message: If you use this software, please cite it using these metadata.
|
|
37
37
|
repository-code: https://github.com/FAIRDataPipeline/FAIR-CLI/
|
|
38
38
|
title: "The FAIR Data Pipeline command line tool"
|
|
39
|
-
version: 0.9.
|
|
39
|
+
version: 0.9.6
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: fair-cli
|
|
3
|
-
Version: 0.9.
|
|
3
|
+
Version: 0.9.6
|
|
4
4
|
Summary: Synchronization interface for the SCRC FAIR Data Pipeline registry
|
|
5
5
|
Home-page: https://www.fairdatapipeline.org/
|
|
6
6
|
License: BSD-2-Clause
|
|
@@ -30,7 +30,7 @@ Requires-Dist: PyYAML (>=5.4.1,<7.0.0)
|
|
|
30
30
|
Requires-Dist: click (>=8.0.0,<9.0.0)
|
|
31
31
|
Requires-Dist: email-validator (>=1.1.3,<2.0.0)
|
|
32
32
|
Requires-Dist: fake-useragent (>=1,<2)
|
|
33
|
-
Requires-Dist: pre-commit (>=
|
|
33
|
+
Requires-Dist: pre-commit (>=3,<4)
|
|
34
34
|
Requires-Dist: pydantic[email] (>=1.9.0,<2.0.0)
|
|
35
35
|
Requires-Dist: requests (>=2.23.0,<3.0.0)
|
|
36
36
|
Requires-Dist: rich (>=10.2.3,<12.0.0)
|
|
@@ -379,22 +379,22 @@ def get_current_user_uri(repo_loc: str) -> str:
|
|
|
379
379
|
raise fdp_exc.CLIConfigurationError("No user URI identifier defined.")
|
|
380
380
|
return _uri
|
|
381
381
|
|
|
382
|
-
def
|
|
382
|
+
def get_current_user_remote_user(repo_loc: str) -> str:
|
|
383
383
|
"""Retrieves the URI identifier for the current user
|
|
384
384
|
|
|
385
385
|
Returns
|
|
386
386
|
-------
|
|
387
387
|
str
|
|
388
|
-
|
|
388
|
+
remote username
|
|
389
389
|
"""
|
|
390
390
|
_local_conf = read_local_fdpconfig(repo_loc)
|
|
391
391
|
try:
|
|
392
|
-
|
|
392
|
+
_remote_user = _local_conf["user"]["remote_user"]
|
|
393
393
|
except KeyError:
|
|
394
|
-
|
|
395
|
-
if not
|
|
396
|
-
raise fdp_exc.CLIConfigurationError("No user
|
|
397
|
-
return
|
|
394
|
+
_remote_user = None
|
|
395
|
+
if not _remote_user or _remote_user == "None":
|
|
396
|
+
raise fdp_exc.CLIConfigurationError("No remote user defined.")
|
|
397
|
+
return _remote_user
|
|
398
398
|
|
|
399
399
|
def check_registry_exists(registry: str = None) -> typing.Optional[str]:
|
|
400
400
|
"""Checks if fair registry is set up on users machine
|
|
@@ -580,6 +580,18 @@ def _handle_github(user_github: str) -> typing.Tuple[typing.Dict, str]:
|
|
|
580
580
|
click.echo(f"Found entry: {_user_info['github']} ")
|
|
581
581
|
return _user_info, _def_ospace
|
|
582
582
|
|
|
583
|
+
def _handle_gitlab(user_gitlab, gitlab_url = "https://gitlab.com") -> typing.Tuple[typing.Dict, str]:
|
|
584
|
+
_user_info = fdp_id.check_gitlab(user_gitlab.strip(), gitlab_url.strip())
|
|
585
|
+
while not _user_info:
|
|
586
|
+
time.sleep(3)
|
|
587
|
+
click.echo(f"Invalid GitLab Username '{user_gitlab}' given.")
|
|
588
|
+
user_gitlab = click.prompt("GitLab Username")
|
|
589
|
+
_user_info = fdp_id.check_gitlab(user_gitlab.strip())
|
|
590
|
+
|
|
591
|
+
_def_ospace = _user_info["gitlab"].lower()
|
|
592
|
+
|
|
593
|
+
click.echo(f"Found entry: {_user_info['gitlab']} ")
|
|
594
|
+
return _user_info, _def_ospace
|
|
583
595
|
|
|
584
596
|
|
|
585
597
|
def _handle_uuid() -> typing.Tuple[typing.Dict, str]:
|
|
@@ -608,14 +620,19 @@ def _handle_uuid() -> typing.Tuple[typing.Dict, str]:
|
|
|
608
620
|
return _user_info, _def_ospace
|
|
609
621
|
|
|
610
622
|
|
|
611
|
-
def _get_user_info_and_namespaces(local: bool = False) -> typing.Dict[str, typing.Dict]:
|
|
623
|
+
def _get_user_info_and_namespaces(local: bool = False, remote_url: str = None) -> typing.Dict[str, typing.Dict]:
|
|
612
624
|
_user_email = click.prompt("Email (optional)", default = "")
|
|
613
625
|
|
|
614
626
|
_invalid_input = True
|
|
627
|
+
if not local:
|
|
628
|
+
_remote_auth_provider = fdp_req.get_auth_provider(remote_url)
|
|
629
|
+
_remote_auth_url = fdp_req.get_auth_url(remote_url)
|
|
630
|
+
else:
|
|
631
|
+
_remote_auth_url = None
|
|
615
632
|
|
|
616
633
|
while _invalid_input:
|
|
617
634
|
_id_type = click.prompt(
|
|
618
|
-
"User ID system (GITHUB/ORCID/ROR/GRID/None)", default="GITHUB"
|
|
635
|
+
"User ID system (GITHUB/GITLAB/ORCID/ROR/GRID/None)", default="GITHUB"
|
|
619
636
|
)
|
|
620
637
|
if _id_type.upper() == "GITHUB":
|
|
621
638
|
_user_github = click.prompt("GitHub Username")
|
|
@@ -625,6 +642,18 @@ def _get_user_info_and_namespaces(local: bool = False) -> typing.Dict[str, typin
|
|
|
625
642
|
_user_info["family_name"] = click.prompt("Family Name")
|
|
626
643
|
_user_info["name"] = " ".join([_user_info["given_names"], _user_info["family_name"]])
|
|
627
644
|
_invalid_input = False
|
|
645
|
+
elif _id_type.upper() == "GITLAB":
|
|
646
|
+
if not _remote_auth_url:
|
|
647
|
+
_remote_auth_url = click.prompt(
|
|
648
|
+
"GitLab URL", default="https://gitlab.com/"
|
|
649
|
+
)
|
|
650
|
+
_user_gitlab = click.prompt("GitLab Username")
|
|
651
|
+
_user_info, _def_ospace = _handle_gitlab(_user_gitlab, _remote_auth_url)
|
|
652
|
+
if not _user_info["name"]:
|
|
653
|
+
_user_info["given_names"] = click.prompt("Given Names")
|
|
654
|
+
_user_info["family_name"] = click.prompt("Family Name")
|
|
655
|
+
_user_info["name"] = " ".join([_user_info["given_names"], _user_info["family_name"]])
|
|
656
|
+
_invalid_input = False
|
|
628
657
|
elif _id_type.upper() == "ORCID":
|
|
629
658
|
_user_orcid = click.prompt("ORCID")
|
|
630
659
|
_user_info, _def_ospace = _handle_orcid(_user_orcid)
|
|
@@ -651,18 +680,34 @@ def _get_user_info_and_namespaces(local: bool = False) -> typing.Dict[str, typin
|
|
|
651
680
|
|
|
652
681
|
_namespaces = {"input": _def_ispace, "output": _def_ospace}
|
|
653
682
|
|
|
654
|
-
if not
|
|
655
|
-
if
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
683
|
+
if not local:
|
|
684
|
+
if _remote_auth_provider == "GitHub":
|
|
685
|
+
if not "github" in _user_info:
|
|
686
|
+
_user_github = click.prompt("GitHub Username")
|
|
687
|
+
_user_github_info = _handle_github(_user_github)[0]
|
|
688
|
+
_user_info["remote_user"] = _user_github_info["github"]
|
|
689
|
+
else:
|
|
690
|
+
_user_info["remote_user"] = _user_info["github"]
|
|
691
|
+
elif _remote_auth_provider == "GitLab":
|
|
692
|
+
if not "gitlab" in _user_info:
|
|
693
|
+
_user_gitlab = click.prompt("GitLab Username")
|
|
694
|
+
_user_gitlab_info = _handle_gitlab(_user_gitlab, _remote_auth_url)[0]
|
|
695
|
+
_user_info["remote_user"] = _user_gitlab_info["gitlab"]
|
|
696
|
+
else:
|
|
697
|
+
_user_info["remote_user"] = _user_info["gitlab"]
|
|
698
|
+
else:
|
|
699
|
+
_user_info["remote_user"] = "FAIRDataPipeline"
|
|
700
|
+
|
|
701
|
+
# Unset unused variables
|
|
702
|
+
_user_info.pop('github', None)
|
|
703
|
+
_user_info.pop('gitlab', None)
|
|
661
704
|
|
|
662
705
|
if not _user_email:
|
|
663
|
-
_user_info["email"] = f'{_user_info["
|
|
706
|
+
_user_info["email"] = f'{_user_info["remote_user"]}@users.noreply.github.com'
|
|
664
707
|
else:
|
|
665
708
|
_user_info["email"] = _user_email
|
|
709
|
+
|
|
710
|
+
logger.debug(f'{_user_info}')
|
|
666
711
|
|
|
667
712
|
return {"user": _user_info, "namespaces": _namespaces}
|
|
668
713
|
|
|
@@ -741,7 +786,7 @@ def global_config_query(
|
|
|
741
786
|
|
|
742
787
|
if not fdp_serv.check_server_running():
|
|
743
788
|
if _ := click.confirm(
|
|
744
|
-
"Local registry is offline, would you like to start it?",
|
|
789
|
+
"Local registry is offline, would you like to start it and for it to remain running?",
|
|
745
790
|
default=False,
|
|
746
791
|
):
|
|
747
792
|
fdp_serv.launch_server(registry_dir=registry)
|
|
@@ -768,7 +813,7 @@ def global_config_query(
|
|
|
768
813
|
if _loc_data_store[-1] != os.path.sep:
|
|
769
814
|
_loc_data_store += os.path.sep
|
|
770
815
|
|
|
771
|
-
_glob_conf_dict = _get_user_info_and_namespaces(local)
|
|
816
|
+
_glob_conf_dict = _get_user_info_and_namespaces(local, _remote_url)
|
|
772
817
|
_glob_conf_dict["registries"] = {
|
|
773
818
|
"local": {
|
|
774
819
|
"uri": _local_uri,
|
|
@@ -108,8 +108,8 @@ class User(pydantic.BaseModel):
|
|
|
108
108
|
name: typing.Optional[str] = pydantic.Field(
|
|
109
109
|
None, title="Full Name", description="Full name for the user"
|
|
110
110
|
)
|
|
111
|
-
|
|
112
|
-
None, title="
|
|
111
|
+
remote_user: typing.Optional[str] = pydantic.Field(
|
|
112
|
+
None, title="Remote Username", description="Remote Username for the user"
|
|
113
113
|
)
|
|
114
114
|
|
|
115
115
|
class Config:
|
|
@@ -82,7 +82,7 @@ def check_orcid(orcid: str) -> typing.Dict:
|
|
|
82
82
|
return _result_dict
|
|
83
83
|
|
|
84
84
|
def check_github(github: str) -> typing.Dict:
|
|
85
|
-
"""Checks if valid
|
|
85
|
+
"""Checks if valid GitHub Username using GitHub Username public api
|
|
86
86
|
|
|
87
87
|
Parameters
|
|
88
88
|
----------
|
|
@@ -122,6 +122,43 @@ def check_github(github: str) -> typing.Dict:
|
|
|
122
122
|
|
|
123
123
|
return _result_dict
|
|
124
124
|
|
|
125
|
+
def check_gitlab(gitlab: str, gitlab_url:str = "https://gitlab.com/") -> typing.Dict:
|
|
126
|
+
"""Checks if valid GitLab Username using Gitlab profile address
|
|
127
|
+
|
|
128
|
+
Parameters
|
|
129
|
+
----------
|
|
130
|
+
gitlab : str
|
|
131
|
+
gitlab username to be checked
|
|
132
|
+
gitlab_url: str
|
|
133
|
+
URL of the gitlab instance, defaults to "https://gitlab.com/"
|
|
134
|
+
|
|
135
|
+
Returns
|
|
136
|
+
-------
|
|
137
|
+
typing.Dict
|
|
138
|
+
metadata from the given ID
|
|
139
|
+
"""
|
|
140
|
+
_url = urllib.parse.urljoin(gitlab_url, gitlab)
|
|
141
|
+
requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
|
|
142
|
+
_response = requests.get(_url, verify = False, allow_redirects = True)
|
|
143
|
+
|
|
144
|
+
_result_dict: typing.Dict[str, typing.Any] = {}
|
|
145
|
+
|
|
146
|
+
if _response.status_code == 403:
|
|
147
|
+
time.sleep(3)
|
|
148
|
+
_header = {"Accept": "application/json", 'User-Agent':str(UserAgent().chrome)}
|
|
149
|
+
_response = requests.get(_url, headers=_header, verify = False, allow_redirects = True)
|
|
150
|
+
|
|
151
|
+
if _response.status_code != 200:
|
|
152
|
+
logger.debug(f"{_url} Responded with {_response.status_code}")
|
|
153
|
+
return _result_dict
|
|
154
|
+
|
|
155
|
+
_username = _response.url.rsplit('/', 1)[-1]
|
|
156
|
+
|
|
157
|
+
_result_dict["name"] = None
|
|
158
|
+
_result_dict["gitlab"] = _username
|
|
159
|
+
_result_dict["uri"] = _response.url
|
|
160
|
+
|
|
161
|
+
return _result_dict
|
|
125
162
|
|
|
126
163
|
def check_ror(ror: str) -> typing.Dict:
|
|
127
164
|
"""Checks if valid ROR using ROR public api
|
|
@@ -629,7 +629,25 @@ def get_obj_type_from_url(request_url: str, token: str) -> str:
|
|
|
629
629
|
return ""
|
|
630
630
|
|
|
631
631
|
|
|
632
|
-
def get_author_exists(registry_uri, token = None, name = None, identifier = None):
|
|
632
|
+
def get_author_exists(registry_uri, token = None, name = None, identifier = None) -> str:
|
|
633
|
+
"""Get an author given an name and or identifier
|
|
634
|
+
|
|
635
|
+
Parameters
|
|
636
|
+
----------
|
|
637
|
+
registry_url : str
|
|
638
|
+
Url of the registry to check for the author
|
|
639
|
+
token : str (optional)
|
|
640
|
+
token for the given registry
|
|
641
|
+
name : str (optional)
|
|
642
|
+
name to be checked
|
|
643
|
+
identifier : str (optional)
|
|
644
|
+
identifier to be checker
|
|
645
|
+
|
|
646
|
+
Returns
|
|
647
|
+
-------
|
|
648
|
+
str
|
|
649
|
+
The url of the author if they exist else None
|
|
650
|
+
"""
|
|
633
651
|
if not name and not identifier:
|
|
634
652
|
return {}
|
|
635
653
|
if not token:
|
|
@@ -647,4 +665,50 @@ def get_author_exists(registry_uri, token = None, name = None, identifier = None
|
|
|
647
665
|
)
|
|
648
666
|
if _author_exists:
|
|
649
667
|
return _author_exists[0]["url"]
|
|
650
|
-
return {}
|
|
668
|
+
return {}
|
|
669
|
+
|
|
670
|
+
def get_auth_provider(registry_uri: str, token: str = None) -> str:
|
|
671
|
+
"""Retrived the auth provider from the remote registry
|
|
672
|
+
|
|
673
|
+
Parameters
|
|
674
|
+
----------
|
|
675
|
+
registry_url : str
|
|
676
|
+
url of the remote registry
|
|
677
|
+
token : str (optional)
|
|
678
|
+
remote registry token
|
|
679
|
+
|
|
680
|
+
Returns
|
|
681
|
+
-------
|
|
682
|
+
str
|
|
683
|
+
The remote authentication provider
|
|
684
|
+
"""
|
|
685
|
+
_url = urllib.parse.urljoin(registry_uri, "auth-provider")
|
|
686
|
+
_response = url_get(_url, token)
|
|
687
|
+
if not _response["auth_provider"]:
|
|
688
|
+
raise fdp_exc.RegistryError(f'The remote registry {registry_uri} \
|
|
689
|
+
did not provide an authentication provider \
|
|
690
|
+
is the remote registry correcly configured?')
|
|
691
|
+
return _response["auth_provider"]
|
|
692
|
+
|
|
693
|
+
def get_auth_url(registry_uri: str, token: str = None) -> str:
|
|
694
|
+
"""Retrived the auth url from the remote registry
|
|
695
|
+
|
|
696
|
+
Parameters
|
|
697
|
+
----------
|
|
698
|
+
registry_url : str
|
|
699
|
+
url of the remote registry
|
|
700
|
+
token : str (optional)
|
|
701
|
+
remote registry token
|
|
702
|
+
|
|
703
|
+
Returns
|
|
704
|
+
-------
|
|
705
|
+
str
|
|
706
|
+
The remote authentication url
|
|
707
|
+
"""
|
|
708
|
+
_url = urllib.parse.urljoin(registry_uri, "auth-url")
|
|
709
|
+
_response = url_get(_url, token)
|
|
710
|
+
if not _response["auth_url"]:
|
|
711
|
+
raise fdp_exc.RegistryError(f'The remote registry {registry_uri} \
|
|
712
|
+
did not provide an authentication provider \
|
|
713
|
+
is the remote registry correcly configured?')
|
|
714
|
+
return _response["auth_url"]
|
|
@@ -387,13 +387,13 @@ def install_registry(
|
|
|
387
387
|
stdout=subprocess.DEVNULL if silent else None,
|
|
388
388
|
)
|
|
389
389
|
|
|
390
|
-
subprocess.check_call(
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
)
|
|
390
|
+
# subprocess.check_call(
|
|
391
|
+
# [_venv_python, "-m", "pip", "install", "whitenoise"],
|
|
392
|
+
# shell=False,
|
|
393
|
+
# stdout=subprocess.DEVNULL if silent else None,
|
|
394
|
+
# )
|
|
395
395
|
|
|
396
|
-
_requirements = os.path.join(install_dir, "
|
|
396
|
+
_requirements = os.path.join(install_dir, "requirements.txt")
|
|
397
397
|
|
|
398
398
|
if not os.path.exists(_requirements):
|
|
399
399
|
raise FileNotFoundError(f"Expected file '{_requirements}'")
|
|
@@ -146,8 +146,7 @@ def sync_dependency_chain(
|
|
|
146
146
|
dest_token: str,
|
|
147
147
|
origin_token: str,
|
|
148
148
|
local_data_store: str = None,
|
|
149
|
-
public: bool = False
|
|
150
|
-
remote_author_url = None
|
|
149
|
+
public: bool = False
|
|
151
150
|
) -> typing.Dict[str, str]:
|
|
152
151
|
"""Push an object and all of its dependencies to the remote registry
|
|
153
152
|
|
|
@@ -187,7 +186,7 @@ def sync_dependency_chain(
|
|
|
187
186
|
)
|
|
188
187
|
|
|
189
188
|
_new_urls: typing.Dict[str, str] = {k: "" for k in _dependency_chain}
|
|
190
|
-
_writable_fields: typing.Dict[str, str] = {}
|
|
189
|
+
_writable_fields: typing.Dict[str, str] = {}
|
|
191
190
|
|
|
192
191
|
# For every object (and the order) in the dependency chain
|
|
193
192
|
# post the object then store the URL so it can be used to assemble those
|
|
@@ -224,9 +223,9 @@ def sync_dependency_chain(
|
|
|
224
223
|
object_url=object_url,
|
|
225
224
|
new_urls=_new_urls,
|
|
226
225
|
writable_data=_writable_data,
|
|
226
|
+
object_data = _obj_data,
|
|
227
227
|
local_data_store = local_data_store,
|
|
228
|
-
public = public
|
|
229
|
-
remote_author_url = remote_author_url
|
|
228
|
+
public = public
|
|
230
229
|
)
|
|
231
230
|
|
|
232
231
|
if not fdp_util.is_api_url(dest_uri, _new_url):
|
|
@@ -248,9 +247,9 @@ def _get_new_url(
|
|
|
248
247
|
object_url: str,
|
|
249
248
|
new_urls: typing.Dict,
|
|
250
249
|
writable_data: typing.Dict,
|
|
250
|
+
object_data = {},
|
|
251
251
|
local_data_store = None,
|
|
252
|
-
public = False
|
|
253
|
-
remote_author_url = None
|
|
252
|
+
public = False
|
|
254
253
|
) -> typing.Tuple[typing.Dict, typing.List]:
|
|
255
254
|
"""Internal Function to return a resgistry entry from the remote registry given an origin entry URL
|
|
256
255
|
If the entry does not exist it will be created
|
|
@@ -303,6 +302,7 @@ def _get_new_url(
|
|
|
303
302
|
# (as remote URL will never match local)
|
|
304
303
|
|
|
305
304
|
_obj_type = fdp_req.get_obj_type_from_url(object_url, token=origin_token)
|
|
305
|
+
_obj_id = fdp_req.get_obj_id_from_url(object_url)
|
|
306
306
|
|
|
307
307
|
_filters = {
|
|
308
308
|
k: v
|
|
@@ -317,21 +317,28 @@ def _get_new_url(
|
|
|
317
317
|
if dest_uri == origin_uri:
|
|
318
318
|
raise fdp_exc.InternalError("Cannot push object to its source address")
|
|
319
319
|
|
|
320
|
-
|
|
320
|
+
# If Public is true then any files will be uploaded to the remote registry
|
|
321
|
+
# If local_data_store is set we're pulling to a local registry
|
|
321
322
|
if public and not local_data_store:
|
|
323
|
+
# The remote data_store storage_root URL should always be the 1st storage_root
|
|
322
324
|
_remote_storage_root_url = urllib.parse.urljoin(dest_uri, "storage_root/1/")
|
|
323
|
-
|
|
325
|
+
# If the current objects a storage_root and the root is the first (data_store)
|
|
326
|
+
# then simply return the remote
|
|
327
|
+
if _obj_type == "storage_root" and _obj_id == '1':
|
|
324
328
|
return _remote_storage_root_url
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
329
|
+
# If the current object is a storage_location
|
|
330
|
+
elif _obj_type == "storage_location":
|
|
331
|
+
# Again if the storage_root is the data_store (storage_root 1)
|
|
332
|
+
if fdp_req.get_obj_id_from_url(object_data.get("storage_root", "/2")) == '1':
|
|
333
|
+
# Update the new object path and filter path
|
|
334
|
+
_new_obj_data["path"] = _filters["path"] = _new_obj_data["hash"]
|
|
335
|
+
# Update the new object storage_root
|
|
336
|
+
_new_obj_data["storage_root"] = _remote_storage_root_url
|
|
337
|
+
# Update the filters for storage_root to the ID
|
|
338
|
+
_filters["storage_root"] = fdp_req.get_obj_id_from_url(_remote_storage_root_url)
|
|
333
339
|
|
|
334
340
|
if _obj_type == "object":
|
|
341
|
+
_new_obj_data["authors"] = []
|
|
335
342
|
_new_author_urls = []
|
|
336
343
|
_filters["authors"] = []
|
|
337
344
|
_tmp_obj = fdp_req.url_get(object_url, origin_token)
|
|
@@ -402,16 +409,16 @@ def sync_user_author(
|
|
|
402
409
|
dest_token: str,
|
|
403
410
|
origin_token: str,
|
|
404
411
|
author_url: str,
|
|
405
|
-
|
|
412
|
+
remote_user: str
|
|
406
413
|
) -> None:
|
|
407
414
|
current_user = fdp_req.get(
|
|
408
415
|
dest_uri,
|
|
409
416
|
"users",
|
|
410
417
|
dest_token,
|
|
411
|
-
params= {"username":
|
|
418
|
+
params= {"username": remote_user}
|
|
412
419
|
)
|
|
413
420
|
if not current_user:
|
|
414
|
-
raise fdp_exc.RegistryError(f"No user matching {
|
|
421
|
+
raise fdp_exc.RegistryError(f"No user matching {remote_user}, on remote registry", "Does your Remote username match the remote registry user?")
|
|
415
422
|
current_user_url = current_user[0]['url']
|
|
416
423
|
fdp_store.store_user_author(dest_uri, dest_token, current_user_url, author_url)
|
|
417
424
|
|
|
@@ -432,13 +439,13 @@ def get_new_author_url(
|
|
|
432
439
|
if k in _writable_fields
|
|
433
440
|
}
|
|
434
441
|
return _get_new_url(
|
|
435
|
-
origin_uri,
|
|
436
|
-
origin_token,
|
|
437
|
-
dest_uri,
|
|
438
|
-
dest_token,
|
|
439
|
-
author_url,
|
|
440
|
-
{},
|
|
441
|
-
_writable_data
|
|
442
|
+
origin_uri= origin_uri,
|
|
443
|
+
origin_token= origin_token,
|
|
444
|
+
dest_uri= dest_uri,
|
|
445
|
+
dest_token= dest_token,
|
|
446
|
+
object_url= author_url,
|
|
447
|
+
new_urls= {},
|
|
448
|
+
writable_data= _writable_data
|
|
442
449
|
)
|
|
443
450
|
|
|
444
451
|
def sync_data_products(
|
|
@@ -449,8 +456,7 @@ def sync_data_products(
|
|
|
449
456
|
remote_label: str,
|
|
450
457
|
data_products: typing.List[str],
|
|
451
458
|
local_data_store: str = None,
|
|
452
|
-
force: bool = False
|
|
453
|
-
remote_author_url = None
|
|
459
|
+
force: bool = False
|
|
454
460
|
) -> None:
|
|
455
461
|
"""Transfer data products from one registry to another
|
|
456
462
|
|
|
@@ -544,8 +550,7 @@ def sync_data_products(
|
|
|
544
550
|
dest_token=dest_token,
|
|
545
551
|
origin_token=origin_token,
|
|
546
552
|
local_data_store=local_data_store,
|
|
547
|
-
public=_is_public
|
|
548
|
-
remote_author_url = remote_author_url
|
|
553
|
+
public=_is_public
|
|
549
554
|
)
|
|
550
555
|
# If local_data_store assume we're syncing from remote to local
|
|
551
556
|
if local_data_store:
|
|
@@ -567,8 +572,8 @@ def sync_data_products(
|
|
|
567
572
|
for origin_input_code_run in origin_input_code_runs:
|
|
568
573
|
dest_component_url = get_dest_component_url(origin_component_url, dest_uri, dest_token, origin_token)
|
|
569
574
|
dest_inputs = [dest_component_url]
|
|
570
|
-
dest_inputs += get_dest_inputs(origin_input_code_run["inputs"], origin_uri, dest_uri, dest_token, origin_token, remote_label
|
|
571
|
-
sync_code_run(origin_uri, dest_uri, dest_token, origin_token, origin_input_code_run["uuid"], inputs= dest_inputs
|
|
575
|
+
dest_inputs += get_dest_inputs(origin_input_code_run["inputs"], origin_uri, dest_uri, dest_token, origin_token, remote_label)
|
|
576
|
+
sync_code_run(origin_uri, dest_uri, dest_token, origin_token, origin_input_code_run["uuid"], inputs= dest_inputs)
|
|
572
577
|
|
|
573
578
|
origin_output_code_runs = fdp_req.get(
|
|
574
579
|
origin_uri,
|
|
@@ -578,9 +583,10 @@ def sync_data_products(
|
|
|
578
583
|
)
|
|
579
584
|
for origin_output_code_run in origin_output_code_runs:
|
|
580
585
|
dest_component_url = get_dest_component_url(origin_component_url, dest_uri, dest_token, origin_token)
|
|
581
|
-
dest_inputs = get_dest_inputs(origin_output_code_run["inputs"], origin_uri, dest_uri, dest_token, origin_token, remote_label, force
|
|
582
|
-
sync_code_run(origin_uri, dest_uri, dest_token, origin_token, origin_output_code_run["uuid"], inputs= dest_inputs, outputs= [dest_component_url]
|
|
586
|
+
dest_inputs = get_dest_inputs(origin_output_code_run["inputs"], origin_uri, dest_uri, dest_token, origin_token, remote_label, force)
|
|
587
|
+
sync_code_run(origin_uri, dest_uri, dest_token, origin_token, origin_output_code_run["uuid"], inputs= dest_inputs, outputs= [dest_component_url])
|
|
583
588
|
logger.info(f"Synced Data Product: {data_product}")
|
|
589
|
+
|
|
584
590
|
def get_dest_component_url(
|
|
585
591
|
origin_component_url: str,
|
|
586
592
|
dest_uri: str,
|
|
@@ -625,8 +631,7 @@ def sync_code_runs(
|
|
|
625
631
|
remote_label: str,
|
|
626
632
|
code_runs: typing.List[str],
|
|
627
633
|
local_data_store: str = None,
|
|
628
|
-
force: bool = False
|
|
629
|
-
remote_author_url = None
|
|
634
|
+
force: bool = False
|
|
630
635
|
) -> None:
|
|
631
636
|
"""Transfer data code_run(s) from one registry to another
|
|
632
637
|
|
|
@@ -695,8 +700,7 @@ def sync_code_runs(
|
|
|
695
700
|
remote_label,
|
|
696
701
|
_origin_data_products_formatted,
|
|
697
702
|
local_data_store,
|
|
698
|
-
force
|
|
699
|
-
remote_author_url)
|
|
703
|
+
force)
|
|
700
704
|
|
|
701
705
|
# Iterate through formatted objects and get their new values from the remote registry
|
|
702
706
|
for _origin_data_product_formatted in _origin_data_products_formatted:
|
|
@@ -724,7 +728,7 @@ def sync_code_runs(
|
|
|
724
728
|
if _origin_data_product_formatted in _outputs_data_products:
|
|
725
729
|
_dest_outputs += _dest_object["components"]
|
|
726
730
|
logger.debug(f'attempting to sync coderun {code_run_uuid} with inputs {_dest_inputs} and outputs {_dest_outputs}')
|
|
727
|
-
sync_code_run(origin_uri, dest_uri, dest_token, origin_token, code_run_uuid, _dest_inputs, _dest_outputs
|
|
731
|
+
sync_code_run(origin_uri, dest_uri, dest_token, origin_token, code_run_uuid, _dest_inputs, _dest_outputs)
|
|
728
732
|
logger.info(f"Synced Code Run: {code_run_uuid}")
|
|
729
733
|
|
|
730
734
|
# Internal function to return the (remote) object associated with a code_run field containing and object url
|
|
@@ -771,8 +775,7 @@ def sync_code_run(
|
|
|
771
775
|
origin_token: str,
|
|
772
776
|
code_run_uuid: str,
|
|
773
777
|
inputs = [],
|
|
774
|
-
outputs = []
|
|
775
|
-
remote_author_url = None) -> typing.Dict:
|
|
778
|
+
outputs = []) -> typing.Dict:
|
|
776
779
|
"""_summary_
|
|
777
780
|
|
|
778
781
|
Args:
|
|
@@ -819,8 +822,7 @@ def sync_code_run(
|
|
|
819
822
|
origin_uri=origin_uri,
|
|
820
823
|
dest_token=dest_token,
|
|
821
824
|
origin_token=origin_token,
|
|
822
|
-
public= True
|
|
823
|
-
remote_author_url = remote_author_url
|
|
825
|
+
public= True
|
|
824
826
|
)
|
|
825
827
|
_dest_code_run_model_config = get_dest_object_url(code_run["model_config"], dest_uri, dest_token, origin_token)
|
|
826
828
|
upload_object(origin_uri, dest_uri, dest_token, origin_token, code_run["model_config"])
|
|
@@ -833,8 +835,7 @@ def sync_code_run(
|
|
|
833
835
|
origin_uri=origin_uri,
|
|
834
836
|
dest_token=dest_token,
|
|
835
837
|
origin_token=origin_token,
|
|
836
|
-
public= True
|
|
837
|
-
remote_author_url = remote_author_url
|
|
838
|
+
public= True
|
|
838
839
|
)
|
|
839
840
|
_dest_code_run_code_repo = get_dest_object_url(code_run["code_repo"], dest_uri, dest_token, origin_token)
|
|
840
841
|
# Sync Submision Script
|
|
@@ -844,8 +845,7 @@ def sync_code_run(
|
|
|
844
845
|
origin_uri=origin_uri,
|
|
845
846
|
dest_token=dest_token,
|
|
846
847
|
origin_token=origin_token,
|
|
847
|
-
public= True
|
|
848
|
-
remote_author_url = remote_author_url
|
|
848
|
+
public= True
|
|
849
849
|
)
|
|
850
850
|
_dest_code_run_submission_script = get_dest_object_url(code_run["submission_script"], dest_uri, dest_token, origin_token)
|
|
851
851
|
upload_object(origin_uri, dest_uri, dest_token, origin_token, code_run["submission_script"])
|
|
@@ -940,8 +940,7 @@ def get_dest_inputs(origin_inputs: typing.List,
|
|
|
940
940
|
dest_token: str,
|
|
941
941
|
origin_token: str,
|
|
942
942
|
remote_label: str,
|
|
943
|
-
force: bool = False
|
|
944
|
-
remote_author_url = None) -> list:
|
|
943
|
+
force: bool = False) -> list:
|
|
945
944
|
"""Returns a list of input component urls on the destination registry
|
|
946
945
|
from a given list of input component urls from the origin registry
|
|
947
946
|
assumes the destination componets already exists in the remote registry
|
|
@@ -962,7 +961,7 @@ def get_dest_inputs(origin_inputs: typing.List,
|
|
|
962
961
|
for origin_input in origin_inputs:
|
|
963
962
|
component_data_products = get_data_products_from_component(origin_input, origin_token)
|
|
964
963
|
if component_data_products:
|
|
965
|
-
sync_data_products(origin_uri, dest_uri, dest_token, origin_token, remote_label, format_data_product_list(component_data_products, origin_token), force= force
|
|
964
|
+
sync_data_products(origin_uri, dest_uri, dest_token, origin_token, remote_label, format_data_product_list(component_data_products, origin_token), force= force)
|
|
966
965
|
for data_product_url in component_data_products:
|
|
967
966
|
origin_data_product_object_url = fdp_req.url_get(data_product_url, origin_token)["object"]
|
|
968
967
|
dest_object_url = get_dest_object_url(origin_data_product_object_url, dest_uri, dest_token, origin_token)
|
|
@@ -975,8 +974,7 @@ def get_dest_inputs(origin_inputs: typing.List,
|
|
|
975
974
|
origin_uri=origin_uri,
|
|
976
975
|
dest_token=dest_token,
|
|
977
976
|
origin_token=origin_token,
|
|
978
|
-
public= is_component_public(origin_input, origin_token)
|
|
979
|
-
remote_author_url = remote_author_url
|
|
977
|
+
public= is_component_public(origin_input, origin_token)
|
|
980
978
|
)
|
|
981
979
|
dest_inputs.append(get_dest_component_url(origin_input, dest_uri, dest_token, origin_token))
|
|
982
980
|
return dest_inputs
|
|
@@ -400,7 +400,7 @@ class FAIR:
|
|
|
400
400
|
),
|
|
401
401
|
origin_token=fdp_req.local_token(),
|
|
402
402
|
author_url = remote_author_url,
|
|
403
|
-
|
|
403
|
+
remote_user = fdp_conf.get_current_user_remote_user(self._session_loc)
|
|
404
404
|
)
|
|
405
405
|
|
|
406
406
|
fdp_sync.sync_code_runs(
|
|
@@ -411,8 +411,7 @@ class FAIR:
|
|
|
411
411
|
),
|
|
412
412
|
origin_token=fdp_req.local_token(),
|
|
413
413
|
remote_label=remote,
|
|
414
|
-
code_runs=_staged_code_runs
|
|
415
|
-
remote_author_url = remote_author_url
|
|
414
|
+
code_runs=_staged_code_runs
|
|
416
415
|
)
|
|
417
416
|
|
|
418
417
|
fdp_sync.sync_data_products(
|
|
@@ -423,8 +422,7 @@ class FAIR:
|
|
|
423
422
|
),
|
|
424
423
|
origin_token=fdp_req.local_token(),
|
|
425
424
|
remote_label=remote,
|
|
426
|
-
data_products=_staged_data_products
|
|
427
|
-
remote_author_url = remote_author_url
|
|
425
|
+
data_products=_staged_data_products
|
|
428
426
|
)
|
|
429
427
|
|
|
430
428
|
self._session_config.write_log_lines(
|
|
@@ -495,7 +493,7 @@ class FAIR:
|
|
|
495
493
|
|
|
496
494
|
self._logger.debug("Pulling data products locally")
|
|
497
495
|
|
|
498
|
-
# Only
|
|
496
|
+
# Only pull data products if there are any to do so, this covers the
|
|
499
497
|
# case whereby no remote has been setup and we just want to register
|
|
500
498
|
# items on the local registry
|
|
501
499
|
if _readables:
|
|
@@ -1329,7 +1327,7 @@ class FAIR:
|
|
|
1329
1327
|
"Expected key 'directory' for local registry in CLI configuration"
|
|
1330
1328
|
)
|
|
1331
1329
|
|
|
1332
|
-
_user_keys = ["email", "family_name", "given_names", "uuid", "
|
|
1330
|
+
_user_keys = ["email", "family_name", "given_names", "uuid", "remote_user"]
|
|
1333
1331
|
|
|
1334
1332
|
for key in _user_keys:
|
|
1335
1333
|
if key not in cli_config["user"]:
|
|
@@ -1338,7 +1336,7 @@ class FAIR:
|
|
|
1338
1336
|
f"Expected key 'user:{key}' in CLI configuration file"
|
|
1339
1337
|
)
|
|
1340
1338
|
|
|
1341
|
-
if not cli_config["user"]["
|
|
1339
|
+
if not cli_config["user"]["remote_user"] and not cli_config["user"]["uuid"]:
|
|
1342
1340
|
raise fdp_exc.CLIConfigurationError(
|
|
1343
1341
|
"At least one of 'user:github' and 'user:uuid' must be provided "
|
|
1344
1342
|
" in CLI configuration"
|
|
@@ -82,7 +82,7 @@ def create_configurations(
|
|
|
82
82
|
"orcid": "000-0000-0000-0000",
|
|
83
83
|
"uri": f'{fdp_id.ID_URIS["github"]}FAIRDataPipeline',
|
|
84
84
|
"uuid": "2ddb2358-84bf-43ff-b2aa-3ac7dc3b49f1",
|
|
85
|
-
"
|
|
85
|
+
"remote_user": "FAIRDataPipeline"
|
|
86
86
|
},
|
|
87
87
|
"git": {
|
|
88
88
|
"local_repo": local_git_dir,
|
|
@@ -10,7 +10,6 @@ authors = [
|
|
|
10
10
|
classifiers = [
|
|
11
11
|
"License :: OSI Approved :: BSD License",
|
|
12
12
|
"Intended Audience :: Science/Research",
|
|
13
|
-
"Programming Language :: Python :: 3.8",
|
|
14
13
|
"Programming Language :: Python :: 3.9",
|
|
15
14
|
"Programming Language :: Python :: 3.10",
|
|
16
15
|
"Programming Language :: Python :: 3.11",
|
|
@@ -25,7 +24,7 @@ classifiers = [
|
|
|
25
24
|
]
|
|
26
25
|
description = "Synchronization interface for the SCRC FAIR Data Pipeline registry"
|
|
27
26
|
name = "fair-cli"
|
|
28
|
-
version = "0.9.
|
|
27
|
+
version = "0.9.6"
|
|
29
28
|
|
|
30
29
|
homepage = "https://www.fairdatapipeline.org/"
|
|
31
30
|
|
|
@@ -55,7 +54,7 @@ Jinja2 = "^3.0.1"
|
|
|
55
54
|
PyYAML = ">=5.4.1,<7.0.0"
|
|
56
55
|
click = "^8.0.0"
|
|
57
56
|
email-validator = "^1.1.3"
|
|
58
|
-
pre-commit = "^
|
|
57
|
+
pre-commit = "^3"
|
|
59
58
|
pydantic = {extras = ["email"], version = "^1.9.0"}
|
|
60
59
|
python = "^3.8.0,<4.0"
|
|
61
60
|
requests = "^2.23.0"
|
|
@@ -69,26 +68,26 @@ fake-useragent = "^1"
|
|
|
69
68
|
[tool.poetry.dev-dependencies]
|
|
70
69
|
bandit = "*"
|
|
71
70
|
black = "*"
|
|
72
|
-
coverage = "
|
|
73
|
-
deepdiff = "^
|
|
71
|
+
coverage = "*"
|
|
72
|
+
deepdiff = "^7"
|
|
74
73
|
flake8 = "*"
|
|
75
74
|
isort = "^5.10.1"
|
|
76
75
|
mypy = "^0.931"
|
|
77
76
|
poetry = "^1.1.12"
|
|
78
|
-
pre-commit = "^
|
|
77
|
+
pre-commit = "^3"
|
|
79
78
|
pycodestyle = "^2.7.0"
|
|
80
79
|
pydocstyle = "^6.0.0"
|
|
81
|
-
pylama = "^
|
|
80
|
+
pylama = "^8"
|
|
82
81
|
pytest = "*"
|
|
82
|
+
pytest-shutil = "*"
|
|
83
83
|
pytest-cov = "*"
|
|
84
84
|
pytest-dependency = "*"
|
|
85
85
|
pytest-mock = "*"
|
|
86
|
-
pytest-virtualenv = "*"
|
|
87
86
|
requests-mock = "*"
|
|
88
87
|
boto3 = "^1.24"
|
|
89
|
-
moto = "^
|
|
90
|
-
Flask = "^
|
|
91
|
-
Flask-Cors = "^
|
|
88
|
+
moto = "^5"
|
|
89
|
+
Flask = "^3"
|
|
90
|
+
Flask-Cors = "^4"
|
|
92
91
|
|
|
93
92
|
[tool.poetry.scripts]
|
|
94
93
|
fair = 'fair.cli:cli'
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|