fair-cli 0.9.4__tar.gz → 0.9.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. {fair_cli-0.9.4 → fair_cli-0.9.6}/CITATION.cff +2 -2
  2. {fair_cli-0.9.4 → fair_cli-0.9.6}/PKG-INFO +2 -2
  3. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/configuration/__init__.py +64 -19
  4. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/configuration/validation.py +2 -2
  5. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/identifiers.py +38 -1
  6. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/registry/requests.py +66 -2
  7. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/registry/server.py +6 -6
  8. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/registry/sync.py +51 -53
  9. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/session.py +6 -8
  10. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/testing.py +1 -1
  11. {fair_cli-0.9.4 → fair_cli-0.9.6}/pyproject.toml +10 -11
  12. {fair_cli-0.9.4 → fair_cli-0.9.6}/CHANGELOG.md +0 -0
  13. {fair_cli-0.9.4 → fair_cli-0.9.6}/LICENSE +0 -0
  14. {fair_cli-0.9.4 → fair_cli-0.9.6}/README.md +0 -0
  15. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/__init__.py +0 -0
  16. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/cli.py +0 -0
  17. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/common.py +0 -0
  18. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/exceptions.py +0 -0
  19. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/files.txt +0 -0
  20. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/history.py +0 -0
  21. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/logging.py +0 -0
  22. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/register.py +0 -0
  23. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/registry/__init__.py +0 -0
  24. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/registry/file_formats.json +0 -0
  25. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/registry/file_types.py +0 -0
  26. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/registry/storage.py +0 -0
  27. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/registry/versioning.py +0 -0
  28. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/run.py +0 -0
  29. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/staging.py +0 -0
  30. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/templates/__init__.py +0 -0
  31. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/templates/config.jinja +0 -0
  32. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/templates/hist.jinja +0 -0
  33. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/user_config/__init__.py +0 -0
  34. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/user_config/globbing.py +0 -0
  35. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/user_config/validation.py +0 -0
  36. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/utilities.py +0 -0
  37. {fair_cli-0.9.4 → fair_cli-0.9.6}/fair/virtualenv.py +0 -0
@@ -26,7 +26,7 @@ authors:
26
26
  affiliation: University of Glasgow
27
27
  orcid: https://orcid.org/0000-0002-4424-9890
28
28
 
29
- date-released: 2023-09-07
29
+ date-released: 2024-08-04
30
30
  keywords:
31
31
  - FAIR Data Pipeline
32
32
  - FAIR
@@ -36,4 +36,4 @@ license: BSD-2-Clause
36
36
  message: If you use this software, please cite it using these metadata.
37
37
  repository-code: https://github.com/FAIRDataPipeline/FAIR-CLI/
38
38
  title: "The FAIR Data Pipeline command line tool"
39
- version: 0.9.4
39
+ version: 0.9.6
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: fair-cli
3
- Version: 0.9.4
3
+ Version: 0.9.6
4
4
  Summary: Synchronization interface for the SCRC FAIR Data Pipeline registry
5
5
  Home-page: https://www.fairdatapipeline.org/
6
6
  License: BSD-2-Clause
@@ -30,7 +30,7 @@ Requires-Dist: PyYAML (>=5.4.1,<7.0.0)
30
30
  Requires-Dist: click (>=8.0.0,<9.0.0)
31
31
  Requires-Dist: email-validator (>=1.1.3,<2.0.0)
32
32
  Requires-Dist: fake-useragent (>=1,<2)
33
- Requires-Dist: pre-commit (>=2.15.0,<3.0.0)
33
+ Requires-Dist: pre-commit (>=3,<4)
34
34
  Requires-Dist: pydantic[email] (>=1.9.0,<2.0.0)
35
35
  Requires-Dist: requests (>=2.23.0,<3.0.0)
36
36
  Requires-Dist: rich (>=10.2.3,<12.0.0)
@@ -379,22 +379,22 @@ def get_current_user_uri(repo_loc: str) -> str:
379
379
  raise fdp_exc.CLIConfigurationError("No user URI identifier defined.")
380
380
  return _uri
381
381
 
382
- def get_current_user_github(repo_loc: str) -> str:
382
+ def get_current_user_remote_user(repo_loc: str) -> str:
383
383
  """Retrieves the URI identifier for the current user
384
384
 
385
385
  Returns
386
386
  -------
387
387
  str
388
- github username
388
+ remote username
389
389
  """
390
390
  _local_conf = read_local_fdpconfig(repo_loc)
391
391
  try:
392
- _github = _local_conf["user"]["github"]
392
+ _remote_user = _local_conf["user"]["remote_user"]
393
393
  except KeyError:
394
- _github = None
395
- if not _github or _github == "None":
396
- raise fdp_exc.CLIConfigurationError("No user GitHub username defined.")
397
- return _github
394
+ _remote_user = None
395
+ if not _remote_user or _remote_user == "None":
396
+ raise fdp_exc.CLIConfigurationError("No remote user defined.")
397
+ return _remote_user
398
398
 
399
399
  def check_registry_exists(registry: str = None) -> typing.Optional[str]:
400
400
  """Checks if fair registry is set up on users machine
@@ -580,6 +580,18 @@ def _handle_github(user_github: str) -> typing.Tuple[typing.Dict, str]:
580
580
  click.echo(f"Found entry: {_user_info['github']} ")
581
581
  return _user_info, _def_ospace
582
582
 
583
+ def _handle_gitlab(user_gitlab, gitlab_url = "https://gitlab.com") -> typing.Tuple[typing.Dict, str]:
584
+ _user_info = fdp_id.check_gitlab(user_gitlab.strip(), gitlab_url.strip())
585
+ while not _user_info:
586
+ time.sleep(3)
587
+ click.echo(f"Invalid GitLab Username '{user_gitlab}' given.")
588
+ user_gitlab = click.prompt("GitLab Username")
589
+ _user_info = fdp_id.check_gitlab(user_gitlab.strip())
590
+
591
+ _def_ospace = _user_info["gitlab"].lower()
592
+
593
+ click.echo(f"Found entry: {_user_info['gitlab']} ")
594
+ return _user_info, _def_ospace
583
595
 
584
596
 
585
597
  def _handle_uuid() -> typing.Tuple[typing.Dict, str]:
@@ -608,14 +620,19 @@ def _handle_uuid() -> typing.Tuple[typing.Dict, str]:
608
620
  return _user_info, _def_ospace
609
621
 
610
622
 
611
- def _get_user_info_and_namespaces(local: bool = False) -> typing.Dict[str, typing.Dict]:
623
+ def _get_user_info_and_namespaces(local: bool = False, remote_url: str = None) -> typing.Dict[str, typing.Dict]:
612
624
  _user_email = click.prompt("Email (optional)", default = "")
613
625
 
614
626
  _invalid_input = True
627
+ if not local:
628
+ _remote_auth_provider = fdp_req.get_auth_provider(remote_url)
629
+ _remote_auth_url = fdp_req.get_auth_url(remote_url)
630
+ else:
631
+ _remote_auth_url = None
615
632
 
616
633
  while _invalid_input:
617
634
  _id_type = click.prompt(
618
- "User ID system (GITHUB/ORCID/ROR/GRID/None)", default="GITHUB"
635
+ "User ID system (GITHUB/GITLAB/ORCID/ROR/GRID/None)", default="GITHUB"
619
636
  )
620
637
  if _id_type.upper() == "GITHUB":
621
638
  _user_github = click.prompt("GitHub Username")
@@ -625,6 +642,18 @@ def _get_user_info_and_namespaces(local: bool = False) -> typing.Dict[str, typin
625
642
  _user_info["family_name"] = click.prompt("Family Name")
626
643
  _user_info["name"] = " ".join([_user_info["given_names"], _user_info["family_name"]])
627
644
  _invalid_input = False
645
+ elif _id_type.upper() == "GITLAB":
646
+ if not _remote_auth_url:
647
+ _remote_auth_url = click.prompt(
648
+ "GitLab URL", default="https://gitlab.com/"
649
+ )
650
+ _user_gitlab = click.prompt("GitLab Username")
651
+ _user_info, _def_ospace = _handle_gitlab(_user_gitlab, _remote_auth_url)
652
+ if not _user_info["name"]:
653
+ _user_info["given_names"] = click.prompt("Given Names")
654
+ _user_info["family_name"] = click.prompt("Family Name")
655
+ _user_info["name"] = " ".join([_user_info["given_names"], _user_info["family_name"]])
656
+ _invalid_input = False
628
657
  elif _id_type.upper() == "ORCID":
629
658
  _user_orcid = click.prompt("ORCID")
630
659
  _user_info, _def_ospace = _handle_orcid(_user_orcid)
@@ -651,18 +680,34 @@ def _get_user_info_and_namespaces(local: bool = False) -> typing.Dict[str, typin
651
680
 
652
681
  _namespaces = {"input": _def_ispace, "output": _def_ospace}
653
682
 
654
- if not "github" in _user_info:
655
- if local:
656
- _user_info["github"] = "FAIRDataPipeline"
657
- else:
658
- _user_github = click.prompt("GitHub Username")
659
- _user_github_info = _handle_github(_user_github)[0]
660
- _user_info["github"] = _user_github_info["github"]
683
+ if not local:
684
+ if _remote_auth_provider == "GitHub":
685
+ if not "github" in _user_info:
686
+ _user_github = click.prompt("GitHub Username")
687
+ _user_github_info = _handle_github(_user_github)[0]
688
+ _user_info["remote_user"] = _user_github_info["github"]
689
+ else:
690
+ _user_info["remote_user"] = _user_info["github"]
691
+ elif _remote_auth_provider == "GitLab":
692
+ if not "gitlab" in _user_info:
693
+ _user_gitlab = click.prompt("GitLab Username")
694
+ _user_gitlab_info = _handle_gitlab(_user_gitlab, _remote_auth_url)[0]
695
+ _user_info["remote_user"] = _user_gitlab_info["gitlab"]
696
+ else:
697
+ _user_info["remote_user"] = _user_info["gitlab"]
698
+ else:
699
+ _user_info["remote_user"] = "FAIRDataPipeline"
700
+
701
+ # Unset unused variables
702
+ _user_info.pop('github', None)
703
+ _user_info.pop('gitlab', None)
661
704
 
662
705
  if not _user_email:
663
- _user_info["email"] = f'{_user_info["github"]}@users.noreply.github.com'
706
+ _user_info["email"] = f'{_user_info["remote_user"]}@users.noreply.github.com'
664
707
  else:
665
708
  _user_info["email"] = _user_email
709
+
710
+ logger.debug(f'{_user_info}')
666
711
 
667
712
  return {"user": _user_info, "namespaces": _namespaces}
668
713
 
@@ -741,7 +786,7 @@ def global_config_query(
741
786
 
742
787
  if not fdp_serv.check_server_running():
743
788
  if _ := click.confirm(
744
- "Local registry is offline, would you like to start it?",
789
+ "Local registry is offline, would you like to start it and for it to remain running?",
745
790
  default=False,
746
791
  ):
747
792
  fdp_serv.launch_server(registry_dir=registry)
@@ -768,7 +813,7 @@ def global_config_query(
768
813
  if _loc_data_store[-1] != os.path.sep:
769
814
  _loc_data_store += os.path.sep
770
815
 
771
- _glob_conf_dict = _get_user_info_and_namespaces(local)
816
+ _glob_conf_dict = _get_user_info_and_namespaces(local, _remote_url)
772
817
  _glob_conf_dict["registries"] = {
773
818
  "local": {
774
819
  "uri": _local_uri,
@@ -108,8 +108,8 @@ class User(pydantic.BaseModel):
108
108
  name: typing.Optional[str] = pydantic.Field(
109
109
  None, title="Full Name", description="Full name for the user"
110
110
  )
111
- github: typing.Optional[str] = pydantic.Field(
112
- None, title="GitHub Username", description="GitHub Username for the user"
111
+ remote_user: typing.Optional[str] = pydantic.Field(
112
+ None, title="Remote Username", description="Remote Username for the user"
113
113
  )
114
114
 
115
115
  class Config:
@@ -82,7 +82,7 @@ def check_orcid(orcid: str) -> typing.Dict:
82
82
  return _result_dict
83
83
 
84
84
  def check_github(github: str) -> typing.Dict:
85
- """Checks if valid ORCID using ORCID public api
85
+ """Checks if valid GitHub Username using GitHub Username public api
86
86
 
87
87
  Parameters
88
88
  ----------
@@ -122,6 +122,43 @@ def check_github(github: str) -> typing.Dict:
122
122
 
123
123
  return _result_dict
124
124
 
125
+ def check_gitlab(gitlab: str, gitlab_url:str = "https://gitlab.com/") -> typing.Dict:
126
+ """Checks if valid GitLab Username using Gitlab profile address
127
+
128
+ Parameters
129
+ ----------
130
+ gitlab : str
131
+ gitlab username to be checked
132
+ gitlab_url: str
133
+ URL of the gitlab instance, defaults to "https://gitlab.com/"
134
+
135
+ Returns
136
+ -------
137
+ typing.Dict
138
+ metadata from the given ID
139
+ """
140
+ _url = urllib.parse.urljoin(gitlab_url, gitlab)
141
+ requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
142
+ _response = requests.get(_url, verify = False, allow_redirects = True)
143
+
144
+ _result_dict: typing.Dict[str, typing.Any] = {}
145
+
146
+ if _response.status_code == 403:
147
+ time.sleep(3)
148
+ _header = {"Accept": "application/json", 'User-Agent':str(UserAgent().chrome)}
149
+ _response = requests.get(_url, headers=_header, verify = False, allow_redirects = True)
150
+
151
+ if _response.status_code != 200:
152
+ logger.debug(f"{_url} Responded with {_response.status_code}")
153
+ return _result_dict
154
+
155
+ _username = _response.url.rsplit('/', 1)[-1]
156
+
157
+ _result_dict["name"] = None
158
+ _result_dict["gitlab"] = _username
159
+ _result_dict["uri"] = _response.url
160
+
161
+ return _result_dict
125
162
 
126
163
  def check_ror(ror: str) -> typing.Dict:
127
164
  """Checks if valid ROR using ROR public api
@@ -629,7 +629,25 @@ def get_obj_type_from_url(request_url: str, token: str) -> str:
629
629
  return ""
630
630
 
631
631
 
632
- def get_author_exists(registry_uri, token = None, name = None, identifier = None):
632
+ def get_author_exists(registry_uri, token = None, name = None, identifier = None) -> str:
633
+ """Get an author given an name and or identifier
634
+
635
+ Parameters
636
+ ----------
637
+ registry_url : str
638
+ Url of the registry to check for the author
639
+ token : str (optional)
640
+ token for the given registry
641
+ name : str (optional)
642
+ name to be checked
643
+ identifier : str (optional)
644
+ identifier to be checker
645
+
646
+ Returns
647
+ -------
648
+ str
649
+ The url of the author if they exist else None
650
+ """
633
651
  if not name and not identifier:
634
652
  return {}
635
653
  if not token:
@@ -647,4 +665,50 @@ def get_author_exists(registry_uri, token = None, name = None, identifier = None
647
665
  )
648
666
  if _author_exists:
649
667
  return _author_exists[0]["url"]
650
- return {}
668
+ return {}
669
+
670
+ def get_auth_provider(registry_uri: str, token: str = None) -> str:
671
+ """Retrived the auth provider from the remote registry
672
+
673
+ Parameters
674
+ ----------
675
+ registry_url : str
676
+ url of the remote registry
677
+ token : str (optional)
678
+ remote registry token
679
+
680
+ Returns
681
+ -------
682
+ str
683
+ The remote authentication provider
684
+ """
685
+ _url = urllib.parse.urljoin(registry_uri, "auth-provider")
686
+ _response = url_get(_url, token)
687
+ if not _response["auth_provider"]:
688
+ raise fdp_exc.RegistryError(f'The remote registry {registry_uri} \
689
+ did not provide an authentication provider \
690
+ is the remote registry correcly configured?')
691
+ return _response["auth_provider"]
692
+
693
+ def get_auth_url(registry_uri: str, token: str = None) -> str:
694
+ """Retrived the auth url from the remote registry
695
+
696
+ Parameters
697
+ ----------
698
+ registry_url : str
699
+ url of the remote registry
700
+ token : str (optional)
701
+ remote registry token
702
+
703
+ Returns
704
+ -------
705
+ str
706
+ The remote authentication url
707
+ """
708
+ _url = urllib.parse.urljoin(registry_uri, "auth-url")
709
+ _response = url_get(_url, token)
710
+ if not _response["auth_url"]:
711
+ raise fdp_exc.RegistryError(f'The remote registry {registry_uri} \
712
+ did not provide an authentication provider \
713
+ is the remote registry correcly configured?')
714
+ return _response["auth_url"]
@@ -387,13 +387,13 @@ def install_registry(
387
387
  stdout=subprocess.DEVNULL if silent else None,
388
388
  )
389
389
 
390
- subprocess.check_call(
391
- [_venv_python, "-m", "pip", "install", "whitenoise"],
392
- shell=False,
393
- stdout=subprocess.DEVNULL if silent else None,
394
- )
390
+ # subprocess.check_call(
391
+ # [_venv_python, "-m", "pip", "install", "whitenoise"],
392
+ # shell=False,
393
+ # stdout=subprocess.DEVNULL if silent else None,
394
+ # )
395
395
 
396
- _requirements = os.path.join(install_dir, "local-requirements.txt")
396
+ _requirements = os.path.join(install_dir, "requirements.txt")
397
397
 
398
398
  if not os.path.exists(_requirements):
399
399
  raise FileNotFoundError(f"Expected file '{_requirements}'")
@@ -146,8 +146,7 @@ def sync_dependency_chain(
146
146
  dest_token: str,
147
147
  origin_token: str,
148
148
  local_data_store: str = None,
149
- public: bool = False,
150
- remote_author_url = None
149
+ public: bool = False
151
150
  ) -> typing.Dict[str, str]:
152
151
  """Push an object and all of its dependencies to the remote registry
153
152
 
@@ -187,7 +186,7 @@ def sync_dependency_chain(
187
186
  )
188
187
 
189
188
  _new_urls: typing.Dict[str, str] = {k: "" for k in _dependency_chain}
190
- _writable_fields: typing.Dict[str, str] = {}
189
+ _writable_fields: typing.Dict[str, str] = {}
191
190
 
192
191
  # For every object (and the order) in the dependency chain
193
192
  # post the object then store the URL so it can be used to assemble those
@@ -224,9 +223,9 @@ def sync_dependency_chain(
224
223
  object_url=object_url,
225
224
  new_urls=_new_urls,
226
225
  writable_data=_writable_data,
226
+ object_data = _obj_data,
227
227
  local_data_store = local_data_store,
228
- public = public,
229
- remote_author_url = remote_author_url
228
+ public = public
230
229
  )
231
230
 
232
231
  if not fdp_util.is_api_url(dest_uri, _new_url):
@@ -248,9 +247,9 @@ def _get_new_url(
248
247
  object_url: str,
249
248
  new_urls: typing.Dict,
250
249
  writable_data: typing.Dict,
250
+ object_data = {},
251
251
  local_data_store = None,
252
- public = False,
253
- remote_author_url = None
252
+ public = False
254
253
  ) -> typing.Tuple[typing.Dict, typing.List]:
255
254
  """Internal Function to return a resgistry entry from the remote registry given an origin entry URL
256
255
  If the entry does not exist it will be created
@@ -303,6 +302,7 @@ def _get_new_url(
303
302
  # (as remote URL will never match local)
304
303
 
305
304
  _obj_type = fdp_req.get_obj_type_from_url(object_url, token=origin_token)
305
+ _obj_id = fdp_req.get_obj_id_from_url(object_url)
306
306
 
307
307
  _filters = {
308
308
  k: v
@@ -317,21 +317,28 @@ def _get_new_url(
317
317
  if dest_uri == origin_uri:
318
318
  raise fdp_exc.InternalError("Cannot push object to its source address")
319
319
 
320
- # If public replace storage root, path
320
+ # If Public is true then any files will be uploaded to the remote registry
321
+ # If local_data_store is set we're pulling to a local registry
321
322
  if public and not local_data_store:
323
+ # The remote data_store storage_root URL should always be the 1st storage_root
322
324
  _remote_storage_root_url = urllib.parse.urljoin(dest_uri, "storage_root/1/")
323
- if _obj_type == "storage_root":
325
+ # If the current objects a storage_root and the root is the first (data_store)
326
+ # then simply return the remote
327
+ if _obj_type == "storage_root" and _obj_id == '1':
324
328
  return _remote_storage_root_url
325
- if "path" in _new_obj_data:
326
- _new_obj_data["path"] = _filters["path"] = _new_obj_data["hash"]
327
- if "storage_root" in _new_obj_data:
328
- _new_obj_data["storage_root"] = _remote_storage_root_url
329
- _filters["storage_root"] = fdp_req.get_obj_id_from_url(_remote_storage_root_url)
330
-
331
- # if remote_author_url and _obj_type == "object":
332
- # _new_obj_data["authors"] = _filters["authors"] = [remote_author_url]
329
+ # If the current object is a storage_location
330
+ elif _obj_type == "storage_location":
331
+ # Again if the storage_root is the data_store (storage_root 1)
332
+ if fdp_req.get_obj_id_from_url(object_data.get("storage_root", "/2")) == '1':
333
+ # Update the new object path and filter path
334
+ _new_obj_data["path"] = _filters["path"] = _new_obj_data["hash"]
335
+ # Update the new object storage_root
336
+ _new_obj_data["storage_root"] = _remote_storage_root_url
337
+ # Update the filters for storage_root to the ID
338
+ _filters["storage_root"] = fdp_req.get_obj_id_from_url(_remote_storage_root_url)
333
339
 
334
340
  if _obj_type == "object":
341
+ _new_obj_data["authors"] = []
335
342
  _new_author_urls = []
336
343
  _filters["authors"] = []
337
344
  _tmp_obj = fdp_req.url_get(object_url, origin_token)
@@ -402,16 +409,16 @@ def sync_user_author(
402
409
  dest_token: str,
403
410
  origin_token: str,
404
411
  author_url: str,
405
- github: str
412
+ remote_user: str
406
413
  ) -> None:
407
414
  current_user = fdp_req.get(
408
415
  dest_uri,
409
416
  "users",
410
417
  dest_token,
411
- params= {"username": github}
418
+ params= {"username": remote_user}
412
419
  )
413
420
  if not current_user:
414
- raise fdp_exc.RegistryError(f"No user matching {github}, on remote registry", "Does your GitHub username match the remote registry GitHub user?")
421
+ raise fdp_exc.RegistryError(f"No user matching {remote_user}, on remote registry", "Does your Remote username match the remote registry user?")
415
422
  current_user_url = current_user[0]['url']
416
423
  fdp_store.store_user_author(dest_uri, dest_token, current_user_url, author_url)
417
424
 
@@ -432,13 +439,13 @@ def get_new_author_url(
432
439
  if k in _writable_fields
433
440
  }
434
441
  return _get_new_url(
435
- origin_uri,
436
- origin_token,
437
- dest_uri,
438
- dest_token,
439
- author_url,
440
- {},
441
- _writable_data
442
+ origin_uri= origin_uri,
443
+ origin_token= origin_token,
444
+ dest_uri= dest_uri,
445
+ dest_token= dest_token,
446
+ object_url= author_url,
447
+ new_urls= {},
448
+ writable_data= _writable_data
442
449
  )
443
450
 
444
451
  def sync_data_products(
@@ -449,8 +456,7 @@ def sync_data_products(
449
456
  remote_label: str,
450
457
  data_products: typing.List[str],
451
458
  local_data_store: str = None,
452
- force: bool = False,
453
- remote_author_url = None
459
+ force: bool = False
454
460
  ) -> None:
455
461
  """Transfer data products from one registry to another
456
462
 
@@ -544,8 +550,7 @@ def sync_data_products(
544
550
  dest_token=dest_token,
545
551
  origin_token=origin_token,
546
552
  local_data_store=local_data_store,
547
- public=_is_public,
548
- remote_author_url = remote_author_url
553
+ public=_is_public
549
554
  )
550
555
  # If local_data_store assume we're syncing from remote to local
551
556
  if local_data_store:
@@ -567,8 +572,8 @@ def sync_data_products(
567
572
  for origin_input_code_run in origin_input_code_runs:
568
573
  dest_component_url = get_dest_component_url(origin_component_url, dest_uri, dest_token, origin_token)
569
574
  dest_inputs = [dest_component_url]
570
- dest_inputs += get_dest_inputs(origin_input_code_run["inputs"], origin_uri, dest_uri, dest_token, origin_token, remote_label, remote_author_url)
571
- sync_code_run(origin_uri, dest_uri, dest_token, origin_token, origin_input_code_run["uuid"], inputs= dest_inputs, remote_author_url = remote_author_url)
575
+ dest_inputs += get_dest_inputs(origin_input_code_run["inputs"], origin_uri, dest_uri, dest_token, origin_token, remote_label)
576
+ sync_code_run(origin_uri, dest_uri, dest_token, origin_token, origin_input_code_run["uuid"], inputs= dest_inputs)
572
577
 
573
578
  origin_output_code_runs = fdp_req.get(
574
579
  origin_uri,
@@ -578,9 +583,10 @@ def sync_data_products(
578
583
  )
579
584
  for origin_output_code_run in origin_output_code_runs:
580
585
  dest_component_url = get_dest_component_url(origin_component_url, dest_uri, dest_token, origin_token)
581
- dest_inputs = get_dest_inputs(origin_output_code_run["inputs"], origin_uri, dest_uri, dest_token, origin_token, remote_label, force, remote_author_url)
582
- sync_code_run(origin_uri, dest_uri, dest_token, origin_token, origin_output_code_run["uuid"], inputs= dest_inputs, outputs= [dest_component_url], remote_author_url = remote_author_url)
586
+ dest_inputs = get_dest_inputs(origin_output_code_run["inputs"], origin_uri, dest_uri, dest_token, origin_token, remote_label, force)
587
+ sync_code_run(origin_uri, dest_uri, dest_token, origin_token, origin_output_code_run["uuid"], inputs= dest_inputs, outputs= [dest_component_url])
583
588
  logger.info(f"Synced Data Product: {data_product}")
589
+
584
590
  def get_dest_component_url(
585
591
  origin_component_url: str,
586
592
  dest_uri: str,
@@ -625,8 +631,7 @@ def sync_code_runs(
625
631
  remote_label: str,
626
632
  code_runs: typing.List[str],
627
633
  local_data_store: str = None,
628
- force: bool = False,
629
- remote_author_url = None
634
+ force: bool = False
630
635
  ) -> None:
631
636
  """Transfer data code_run(s) from one registry to another
632
637
 
@@ -695,8 +700,7 @@ def sync_code_runs(
695
700
  remote_label,
696
701
  _origin_data_products_formatted,
697
702
  local_data_store,
698
- force,
699
- remote_author_url)
703
+ force)
700
704
 
701
705
  # Iterate through formatted objects and get their new values from the remote registry
702
706
  for _origin_data_product_formatted in _origin_data_products_formatted:
@@ -724,7 +728,7 @@ def sync_code_runs(
724
728
  if _origin_data_product_formatted in _outputs_data_products:
725
729
  _dest_outputs += _dest_object["components"]
726
730
  logger.debug(f'attempting to sync coderun {code_run_uuid} with inputs {_dest_inputs} and outputs {_dest_outputs}')
727
- sync_code_run(origin_uri, dest_uri, dest_token, origin_token, code_run_uuid, _dest_inputs, _dest_outputs, remote_author_url = remote_author_url)
731
+ sync_code_run(origin_uri, dest_uri, dest_token, origin_token, code_run_uuid, _dest_inputs, _dest_outputs)
728
732
  logger.info(f"Synced Code Run: {code_run_uuid}")
729
733
 
730
734
  # Internal function to return the (remote) object associated with a code_run field containing and object url
@@ -771,8 +775,7 @@ def sync_code_run(
771
775
  origin_token: str,
772
776
  code_run_uuid: str,
773
777
  inputs = [],
774
- outputs = [],
775
- remote_author_url = None) -> typing.Dict:
778
+ outputs = []) -> typing.Dict:
776
779
  """_summary_
777
780
 
778
781
  Args:
@@ -819,8 +822,7 @@ def sync_code_run(
819
822
  origin_uri=origin_uri,
820
823
  dest_token=dest_token,
821
824
  origin_token=origin_token,
822
- public= True,
823
- remote_author_url = remote_author_url
825
+ public= True
824
826
  )
825
827
  _dest_code_run_model_config = get_dest_object_url(code_run["model_config"], dest_uri, dest_token, origin_token)
826
828
  upload_object(origin_uri, dest_uri, dest_token, origin_token, code_run["model_config"])
@@ -833,8 +835,7 @@ def sync_code_run(
833
835
  origin_uri=origin_uri,
834
836
  dest_token=dest_token,
835
837
  origin_token=origin_token,
836
- public= True,
837
- remote_author_url = remote_author_url
838
+ public= True
838
839
  )
839
840
  _dest_code_run_code_repo = get_dest_object_url(code_run["code_repo"], dest_uri, dest_token, origin_token)
840
841
  # Sync Submision Script
@@ -844,8 +845,7 @@ def sync_code_run(
844
845
  origin_uri=origin_uri,
845
846
  dest_token=dest_token,
846
847
  origin_token=origin_token,
847
- public= True,
848
- remote_author_url = remote_author_url
848
+ public= True
849
849
  )
850
850
  _dest_code_run_submission_script = get_dest_object_url(code_run["submission_script"], dest_uri, dest_token, origin_token)
851
851
  upload_object(origin_uri, dest_uri, dest_token, origin_token, code_run["submission_script"])
@@ -940,8 +940,7 @@ def get_dest_inputs(origin_inputs: typing.List,
940
940
  dest_token: str,
941
941
  origin_token: str,
942
942
  remote_label: str,
943
- force: bool = False,
944
- remote_author_url = None) -> list:
943
+ force: bool = False) -> list:
945
944
  """Returns a list of input component urls on the destination registry
946
945
  from a given list of input component urls from the origin registry
947
946
  assumes the destination componets already exists in the remote registry
@@ -962,7 +961,7 @@ def get_dest_inputs(origin_inputs: typing.List,
962
961
  for origin_input in origin_inputs:
963
962
  component_data_products = get_data_products_from_component(origin_input, origin_token)
964
963
  if component_data_products:
965
- sync_data_products(origin_uri, dest_uri, dest_token, origin_token, remote_label, format_data_product_list(component_data_products, origin_token), force= force, remote_author_url = remote_author_url)
964
+ sync_data_products(origin_uri, dest_uri, dest_token, origin_token, remote_label, format_data_product_list(component_data_products, origin_token), force= force)
966
965
  for data_product_url in component_data_products:
967
966
  origin_data_product_object_url = fdp_req.url_get(data_product_url, origin_token)["object"]
968
967
  dest_object_url = get_dest_object_url(origin_data_product_object_url, dest_uri, dest_token, origin_token)
@@ -975,8 +974,7 @@ def get_dest_inputs(origin_inputs: typing.List,
975
974
  origin_uri=origin_uri,
976
975
  dest_token=dest_token,
977
976
  origin_token=origin_token,
978
- public= is_component_public(origin_input, origin_token),
979
- remote_author_url = remote_author_url
977
+ public= is_component_public(origin_input, origin_token)
980
978
  )
981
979
  dest_inputs.append(get_dest_component_url(origin_input, dest_uri, dest_token, origin_token))
982
980
  return dest_inputs
@@ -400,7 +400,7 @@ class FAIR:
400
400
  ),
401
401
  origin_token=fdp_req.local_token(),
402
402
  author_url = remote_author_url,
403
- github = fdp_conf.get_current_user_github(self._session_loc)
403
+ remote_user = fdp_conf.get_current_user_remote_user(self._session_loc)
404
404
  )
405
405
 
406
406
  fdp_sync.sync_code_runs(
@@ -411,8 +411,7 @@ class FAIR:
411
411
  ),
412
412
  origin_token=fdp_req.local_token(),
413
413
  remote_label=remote,
414
- code_runs=_staged_code_runs,
415
- remote_author_url = remote_author_url
414
+ code_runs=_staged_code_runs
416
415
  )
417
416
 
418
417
  fdp_sync.sync_data_products(
@@ -423,8 +422,7 @@ class FAIR:
423
422
  ),
424
423
  origin_token=fdp_req.local_token(),
425
424
  remote_label=remote,
426
- data_products=_staged_data_products,
427
- remote_author_url = remote_author_url
425
+ data_products=_staged_data_products
428
426
  )
429
427
 
430
428
  self._session_config.write_log_lines(
@@ -495,7 +493,7 @@ class FAIR:
495
493
 
496
494
  self._logger.debug("Pulling data products locally")
497
495
 
498
- # Only push data products if there are any to do so, this covers the
496
+ # Only pull data products if there are any to do so, this covers the
499
497
  # case whereby no remote has been setup and we just want to register
500
498
  # items on the local registry
501
499
  if _readables:
@@ -1329,7 +1327,7 @@ class FAIR:
1329
1327
  "Expected key 'directory' for local registry in CLI configuration"
1330
1328
  )
1331
1329
 
1332
- _user_keys = ["email", "family_name", "given_names", "uuid", "github"]
1330
+ _user_keys = ["email", "family_name", "given_names", "uuid", "remote_user"]
1333
1331
 
1334
1332
  for key in _user_keys:
1335
1333
  if key not in cli_config["user"]:
@@ -1338,7 +1336,7 @@ class FAIR:
1338
1336
  f"Expected key 'user:{key}' in CLI configuration file"
1339
1337
  )
1340
1338
 
1341
- if not cli_config["user"]["github"] and not cli_config["user"]["uuid"]:
1339
+ if not cli_config["user"]["remote_user"] and not cli_config["user"]["uuid"]:
1342
1340
  raise fdp_exc.CLIConfigurationError(
1343
1341
  "At least one of 'user:github' and 'user:uuid' must be provided "
1344
1342
  " in CLI configuration"
@@ -82,7 +82,7 @@ def create_configurations(
82
82
  "orcid": "000-0000-0000-0000",
83
83
  "uri": f'{fdp_id.ID_URIS["github"]}FAIRDataPipeline',
84
84
  "uuid": "2ddb2358-84bf-43ff-b2aa-3ac7dc3b49f1",
85
- "github": "FAIRDataPipeline"
85
+ "remote_user": "FAIRDataPipeline"
86
86
  },
87
87
  "git": {
88
88
  "local_repo": local_git_dir,
@@ -10,7 +10,6 @@ authors = [
10
10
  classifiers = [
11
11
  "License :: OSI Approved :: BSD License",
12
12
  "Intended Audience :: Science/Research",
13
- "Programming Language :: Python :: 3.8",
14
13
  "Programming Language :: Python :: 3.9",
15
14
  "Programming Language :: Python :: 3.10",
16
15
  "Programming Language :: Python :: 3.11",
@@ -25,7 +24,7 @@ classifiers = [
25
24
  ]
26
25
  description = "Synchronization interface for the SCRC FAIR Data Pipeline registry"
27
26
  name = "fair-cli"
28
- version = "0.9.4"
27
+ version = "0.9.6"
29
28
 
30
29
  homepage = "https://www.fairdatapipeline.org/"
31
30
 
@@ -55,7 +54,7 @@ Jinja2 = "^3.0.1"
55
54
  PyYAML = ">=5.4.1,<7.0.0"
56
55
  click = "^8.0.0"
57
56
  email-validator = "^1.1.3"
58
- pre-commit = "^2.15.0"
57
+ pre-commit = "^3"
59
58
  pydantic = {extras = ["email"], version = "^1.9.0"}
60
59
  python = "^3.8.0,<4.0"
61
60
  requests = "^2.23.0"
@@ -69,26 +68,26 @@ fake-useragent = "^1"
69
68
  [tool.poetry.dev-dependencies]
70
69
  bandit = "*"
71
70
  black = "*"
72
- coverage = "7.2.*"
73
- deepdiff = "^5.5.0"
71
+ coverage = "*"
72
+ deepdiff = "^7"
74
73
  flake8 = "*"
75
74
  isort = "^5.10.1"
76
75
  mypy = "^0.931"
77
76
  poetry = "^1.1.12"
78
- pre-commit = "^2.17.0"
77
+ pre-commit = "^3"
79
78
  pycodestyle = "^2.7.0"
80
79
  pydocstyle = "^6.0.0"
81
- pylama = "^7.7.1"
80
+ pylama = "^8"
82
81
  pytest = "*"
82
+ pytest-shutil = "*"
83
83
  pytest-cov = "*"
84
84
  pytest-dependency = "*"
85
85
  pytest-mock = "*"
86
- pytest-virtualenv = "*"
87
86
  requests-mock = "*"
88
87
  boto3 = "^1.24"
89
- moto = "^4.0.3"
90
- Flask = "^2.2.2"
91
- Flask-Cors = "^3.0.10"
88
+ moto = "^5"
89
+ Flask = "^3"
90
+ Flask-Cors = "^4"
92
91
 
93
92
  [tool.poetry.scripts]
94
93
  fair = 'fair.cli:cli'
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes