fair-cli 0.7.2__tar.gz → 0.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fair_cli-0.7.2 → fair_cli-0.8.0}/CITATION.cff +1 -1
- {fair_cli-0.7.2 → fair_cli-0.8.0}/PKG-INFO +2 -2
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/cli.py +52 -26
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/common.py +8 -1
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/configuration/__init__.py +82 -30
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/configuration/validation.py +3 -0
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/history.py +8 -2
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/identifiers.py +58 -7
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/register.py +33 -12
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/registry/requests.py +12 -10
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/registry/server.py +14 -14
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/registry/storage.py +7 -2
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/registry/sync.py +83 -20
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/session.py +57 -24
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/testing.py +3 -2
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/user_config/__init__.py +8 -0
- {fair_cli-0.7.2 → fair_cli-0.8.0}/pyproject.toml +14 -15
- {fair_cli-0.7.2 → fair_cli-0.8.0}/CHANGELOG.md +0 -0
- {fair_cli-0.7.2 → fair_cli-0.8.0}/LICENSE +0 -0
- {fair_cli-0.7.2 → fair_cli-0.8.0}/README.md +0 -0
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/__init__.py +0 -0
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/exceptions.py +0 -0
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/files.txt +0 -0
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/logging.py +0 -0
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/registry/__init__.py +0 -0
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/registry/file_formats.json +0 -0
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/registry/file_types.py +0 -0
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/registry/versioning.py +0 -0
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/run.py +0 -0
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/staging.py +0 -0
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/templates/__init__.py +0 -0
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/templates/config.jinja +0 -0
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/templates/hist.jinja +0 -0
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/user_config/globbing.py +0 -0
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/user_config/validation.py +0 -0
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/utilities.py +0 -0
- {fair_cli-0.7.2 → fair_cli-0.8.0}/fair/virtualenv.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: fair-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.8.0
|
|
4
4
|
Summary: Synchronization interface for the SCRC FAIR Data Pipeline registry
|
|
5
5
|
Home-page: https://www.fairdatapipeline.org/
|
|
6
6
|
License: BSD-2-Clause
|
|
@@ -29,7 +29,7 @@ Requires-Dist: Jinja2 (>=3.0.1,<4.0.0)
|
|
|
29
29
|
Requires-Dist: PyYAML (>=5.4.1,<7.0.0)
|
|
30
30
|
Requires-Dist: click (>=8.0.0,<9.0.0)
|
|
31
31
|
Requires-Dist: email-validator (>=1.1.3,<2.0.0)
|
|
32
|
-
Requires-Dist:
|
|
32
|
+
Requires-Dist: fake-useragent (>=1,<2)
|
|
33
33
|
Requires-Dist: pre-commit (>=2.15.0,<3.0.0)
|
|
34
34
|
Requires-Dist: pydantic[email] (>=1.9.0,<2.0.0)
|
|
35
35
|
Requires-Dist: requests (>=2.23.0,<3.0.0)
|
|
@@ -104,7 +104,8 @@ def status(verbose, debug) -> None:
|
|
|
104
104
|
fair_session.status_code_runs()
|
|
105
105
|
except fdp_exc.FAIRCLIException as e:
|
|
106
106
|
e.err_print()
|
|
107
|
-
|
|
107
|
+
if e.level.lower() == "error":
|
|
108
|
+
sys.exit(e.exit_code)
|
|
108
109
|
|
|
109
110
|
@cli.group(invoke_without_command=True)
|
|
110
111
|
@click.option("--debug/--no-debug", help="Run in debug mode", default=False)
|
|
@@ -116,8 +117,10 @@ def list(ctx, debug, remote) -> None:
|
|
|
116
117
|
ctx.obj = {}
|
|
117
118
|
ctx.obj['DEBUG'] = debug
|
|
118
119
|
ctx.obj['REMOTE'] = remote
|
|
119
|
-
|
|
120
|
-
|
|
120
|
+
_current_args = " ".join(sys.argv)
|
|
121
|
+
if not ("data-products" in _current_args or "code-runs" in _current_args):
|
|
122
|
+
ctx.invoke(data_products)
|
|
123
|
+
ctx.invoke(code_runs)
|
|
121
124
|
|
|
122
125
|
@list.command()
|
|
123
126
|
@click.pass_context
|
|
@@ -131,7 +134,8 @@ def data_products(ctx) -> None:
|
|
|
131
134
|
fair_session.show_all_data_products(remote = remote)
|
|
132
135
|
except fdp_exc.FAIRCLIException as e:
|
|
133
136
|
e.err_print()
|
|
134
|
-
|
|
137
|
+
if e.level.lower() == "error":
|
|
138
|
+
sys.exit(e.exit_code)
|
|
135
139
|
|
|
136
140
|
@list.command()
|
|
137
141
|
@click.pass_context
|
|
@@ -145,7 +149,8 @@ def code_runs(ctx) -> None:
|
|
|
145
149
|
fair_session.show_all_code_runs(remote = remote)
|
|
146
150
|
except fdp_exc.FAIRCLIException as e:
|
|
147
151
|
e.err_print()
|
|
148
|
-
|
|
152
|
+
if e.level.lower() == "error":
|
|
153
|
+
sys.exit(e.exit_code)
|
|
149
154
|
|
|
150
155
|
@cli.command()
|
|
151
156
|
@click.option("--debug/--no-debug", help="Run in debug mode", default=False)
|
|
@@ -157,7 +162,7 @@ def create(debug, output: str) -> None:
|
|
|
157
162
|
if output
|
|
158
163
|
else os.path.join(os.getcwd(), fdp_com.USER_CONFIG_FILE)
|
|
159
164
|
)
|
|
160
|
-
click.echo(f"Generating new user configuration file
|
|
165
|
+
click.echo(f"Generating new user configuration file '{output}'")
|
|
161
166
|
with fdp_session.FAIR(os.getcwd(), debug=debug) as fair_session:
|
|
162
167
|
fair_session.make_starter_config(output)
|
|
163
168
|
|
|
@@ -173,7 +178,8 @@ def reset(debug: bool) -> None:
|
|
|
173
178
|
fair_session.reset_staging()
|
|
174
179
|
except fdp_exc.FAIRCLIException as e:
|
|
175
180
|
e.err_print()
|
|
176
|
-
|
|
181
|
+
if e.level.lower() == "error":
|
|
182
|
+
sys.exit(e.exit_code)
|
|
177
183
|
|
|
178
184
|
|
|
179
185
|
@cli.command()
|
|
@@ -241,8 +247,8 @@ def init(
|
|
|
241
247
|
fair_session.make_starter_config(config)
|
|
242
248
|
except fdp_exc.FAIRCLIException as e:
|
|
243
249
|
e.err_print()
|
|
244
|
-
|
|
245
|
-
|
|
250
|
+
if e.level.lower() == "error":
|
|
251
|
+
sys.exit(e.exit_code)
|
|
246
252
|
|
|
247
253
|
@cli.command()
|
|
248
254
|
@click.option(
|
|
@@ -295,7 +301,8 @@ def purge(glob: bool, debug: bool, yes: bool, data: bool, all: bool) -> None:
|
|
|
295
301
|
fair_session.purge(global_cfg=glob, clear_data=data, clear_all=all)
|
|
296
302
|
except fdp_exc.FAIRCLIException as e:
|
|
297
303
|
e.err_print()
|
|
298
|
-
|
|
304
|
+
if e.level.lower() == "error":
|
|
305
|
+
sys.exit(e.exit_code)
|
|
299
306
|
|
|
300
307
|
|
|
301
308
|
@cli.group()
|
|
@@ -320,7 +327,8 @@ def uninstall(debug: bool):
|
|
|
320
327
|
fdp_svr.uninstall_registry()
|
|
321
328
|
except fdp_exc.FAIRCLIException as e:
|
|
322
329
|
e.err_print()
|
|
323
|
-
|
|
330
|
+
if e.level.lower() == "error":
|
|
331
|
+
sys.exit(e.exit_code)
|
|
324
332
|
|
|
325
333
|
|
|
326
334
|
@registry.command()
|
|
@@ -343,7 +351,8 @@ def install(debug: bool, force: bool, directory: str, version: str):
|
|
|
343
351
|
click.echo(f"Installed registry version '{_version}'")
|
|
344
352
|
except fdp_exc.FAIRCLIException as e:
|
|
345
353
|
e.err_print()
|
|
346
|
-
|
|
354
|
+
if e.level.lower() == "error":
|
|
355
|
+
sys.exit(e.exit_code)
|
|
347
356
|
|
|
348
357
|
|
|
349
358
|
@registry.command()
|
|
@@ -362,7 +371,8 @@ def start(debug: bool, port: int, address: str) -> None:
|
|
|
362
371
|
)
|
|
363
372
|
except fdp_exc.FAIRCLIException as e:
|
|
364
373
|
e.err_print()
|
|
365
|
-
|
|
374
|
+
if e.level.lower() == "error":
|
|
375
|
+
sys.exit(e.exit_code)
|
|
366
376
|
|
|
367
377
|
|
|
368
378
|
@registry.command()
|
|
@@ -379,7 +389,8 @@ def stop(force: bool, debug: bool) -> None:
|
|
|
379
389
|
fdp_session.FAIR(os.getcwd(), server_mode=_mode, debug=debug)
|
|
380
390
|
except fdp_exc.FAIRCLIException as e:
|
|
381
391
|
e.err_print()
|
|
382
|
-
|
|
392
|
+
if e.level.lower() == "error":
|
|
393
|
+
sys.exit(e.exit_code)
|
|
383
394
|
|
|
384
395
|
|
|
385
396
|
@registry.command()
|
|
@@ -392,7 +403,8 @@ def status(debug) -> None:
|
|
|
392
403
|
fair_session.registry_status()
|
|
393
404
|
except fdp_exc.FAIRCLIException as e:
|
|
394
405
|
e.err_print()
|
|
395
|
-
|
|
406
|
+
if e.level.lower() == "error":
|
|
407
|
+
sys.exit(e.exit_code)
|
|
396
408
|
|
|
397
409
|
|
|
398
410
|
@cli.command()
|
|
@@ -403,7 +415,8 @@ def log(debug: bool) -> None:
|
|
|
403
415
|
fdp_hist.show_history(os.getcwd())
|
|
404
416
|
except fdp_exc.FAIRCLIException as e:
|
|
405
417
|
e.err_print()
|
|
406
|
-
|
|
418
|
+
if e.level.lower() == "error":
|
|
419
|
+
sys.exit(e.exit_code)
|
|
407
420
|
|
|
408
421
|
|
|
409
422
|
@cli.command()
|
|
@@ -415,7 +428,8 @@ def view(job_id: str, debug: bool) -> None:
|
|
|
415
428
|
fdp_hist.show_job_log(os.getcwd(), job_id)
|
|
416
429
|
except fdp_exc.FAIRCLIException as e:
|
|
417
430
|
e.err_print()
|
|
418
|
-
|
|
431
|
+
if e.level.lower() == "error":
|
|
432
|
+
sys.exit(e.exit_code)
|
|
419
433
|
|
|
420
434
|
|
|
421
435
|
@cli.command()
|
|
@@ -436,7 +450,8 @@ def unstage(identifier: str, debug: bool, job: bool) -> None:
|
|
|
436
450
|
)
|
|
437
451
|
except fdp_exc.FAIRCLIException as e:
|
|
438
452
|
e.err_print()
|
|
439
|
-
|
|
453
|
+
if e.level.lower() == "error":
|
|
454
|
+
sys.exit(e.exit_code)
|
|
440
455
|
|
|
441
456
|
|
|
442
457
|
@cli.command()
|
|
@@ -466,7 +481,8 @@ def add(identifier: str, debug: bool) -> None:
|
|
|
466
481
|
)
|
|
467
482
|
except fdp_exc.FAIRCLIException as e:
|
|
468
483
|
e.err_print()
|
|
469
|
-
|
|
484
|
+
if e.level.lower() == "error":
|
|
485
|
+
sys.exit(e.exit_code)
|
|
470
486
|
|
|
471
487
|
|
|
472
488
|
@cli.command()
|
|
@@ -512,6 +528,7 @@ def run(
|
|
|
512
528
|
):
|
|
513
529
|
"""Initialises a job with the option to specify a bash command"""
|
|
514
530
|
# Allow no config to be specified, if that is the case use default local
|
|
531
|
+
click.echo("Running run please wait")
|
|
515
532
|
config = config[0] if config else fdp_com.local_user_config(os.getcwd())
|
|
516
533
|
try:
|
|
517
534
|
with fdp_session.FAIR(
|
|
@@ -529,7 +546,8 @@ def run(
|
|
|
529
546
|
click.echo(fdp_run.get_job_dir(_hash))
|
|
530
547
|
except fdp_exc.FAIRCLIException as e:
|
|
531
548
|
e.err_print()
|
|
532
|
-
|
|
549
|
+
if e.level.lower() == "error":
|
|
550
|
+
sys.exit(e.exit_code)
|
|
533
551
|
|
|
534
552
|
|
|
535
553
|
@cli.group(invoke_without_command=True)
|
|
@@ -544,7 +562,8 @@ def remote(ctx, verbose: bool = False, debug: bool = False):
|
|
|
544
562
|
fair_session.list_remotes(verbose)
|
|
545
563
|
except fdp_exc.FAIRCLIException as e:
|
|
546
564
|
e.err_print()
|
|
547
|
-
|
|
565
|
+
if e.level.lower() == "error":
|
|
566
|
+
sys.exit(e.exit_code)
|
|
548
567
|
|
|
549
568
|
|
|
550
569
|
@remote.command()
|
|
@@ -569,7 +588,8 @@ def add(options: typing.List[str], debug: bool) -> None:
|
|
|
569
588
|
fair_session.add_remote(_url, _label)
|
|
570
589
|
except fdp_exc.FAIRCLIException as e:
|
|
571
590
|
e.err_print()
|
|
572
|
-
|
|
591
|
+
if e.level.lower() == "error":
|
|
592
|
+
sys.exit(e.exit_code)
|
|
573
593
|
|
|
574
594
|
|
|
575
595
|
@remote.command()
|
|
@@ -588,7 +608,8 @@ def remove(label: str, debug: bool) -> None:
|
|
|
588
608
|
fair_session.remove_remove(label)
|
|
589
609
|
except fdp_exc.FAIRCLIException as e:
|
|
590
610
|
e.err_print()
|
|
591
|
-
|
|
611
|
+
if e.level.lower() == "error":
|
|
612
|
+
sys.exit(e.exit_code)
|
|
592
613
|
|
|
593
614
|
|
|
594
615
|
@remote.command()
|
|
@@ -603,7 +624,8 @@ def modify(ctx, label: str, url: str, debug: bool) -> None:
|
|
|
603
624
|
fair_session.modify_remote(label, url)
|
|
604
625
|
except fdp_exc.FAIRCLIException as e:
|
|
605
626
|
e.err_print()
|
|
606
|
-
|
|
627
|
+
if e.level.lower() == "error":
|
|
628
|
+
sys.exit(e.exit_code)
|
|
607
629
|
|
|
608
630
|
|
|
609
631
|
@cli.command()
|
|
@@ -616,6 +638,7 @@ def modify(ctx, label: str, url: str, debug: bool) -> None:
|
|
|
616
638
|
)
|
|
617
639
|
def push(remote: str, debug: bool, dirty: bool):
|
|
618
640
|
"""Push data between the local and remote registry"""
|
|
641
|
+
click.echo("Running push please wait")
|
|
619
642
|
remote = remote[0] if remote else "origin"
|
|
620
643
|
try:
|
|
621
644
|
with fdp_session.FAIR(
|
|
@@ -627,7 +650,8 @@ def push(remote: str, debug: bool, dirty: bool):
|
|
|
627
650
|
fair_session.push(remote)
|
|
628
651
|
except fdp_exc.FAIRCLIException as e:
|
|
629
652
|
e.err_print()
|
|
630
|
-
|
|
653
|
+
if e.level.lower() == "error":
|
|
654
|
+
sys.exit(e.exit_code)
|
|
631
655
|
|
|
632
656
|
|
|
633
657
|
@cli.group()
|
|
@@ -658,6 +682,7 @@ def config_email(user_email: str) -> None:
|
|
|
658
682
|
)
|
|
659
683
|
def pull(config: str, debug: bool, local: bool):
|
|
660
684
|
"""Update local registry from remotes and sources"""
|
|
685
|
+
click.echo("Running pull please wait")
|
|
661
686
|
config = config[0] if config else fdp_com.local_user_config(os.getcwd())
|
|
662
687
|
try:
|
|
663
688
|
with fdp_session.FAIR(
|
|
@@ -671,7 +696,8 @@ def pull(config: str, debug: bool, local: bool):
|
|
|
671
696
|
fair.pull()
|
|
672
697
|
except fdp_exc.FAIRCLIException as e:
|
|
673
698
|
e.err_print()
|
|
674
|
-
|
|
699
|
+
if e.level.lower() == "error":
|
|
700
|
+
sys.exit(e.exit_code)
|
|
675
701
|
|
|
676
702
|
|
|
677
703
|
if __name__ in "__main__":
|
|
@@ -296,4 +296,11 @@ def set_file_permissions(path: str):
|
|
|
296
296
|
for dir in [os.path.join(root,d) for d in dirs]:
|
|
297
297
|
os.chmod(dir, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
|
|
298
298
|
for file in [os.path.join(root, f) for f in files]:
|
|
299
|
-
os.chmod(file, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
|
|
299
|
+
os.chmod(file, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
|
|
300
|
+
|
|
301
|
+
def remove_readonly(fn, path, excinfo):
|
|
302
|
+
try:
|
|
303
|
+
os.chmod(path, stat.S_IWRITE)
|
|
304
|
+
fn(path)
|
|
305
|
+
except Exception as exc:
|
|
306
|
+
print("Skipped:", path, "because:\n", exc)
|
|
@@ -262,7 +262,7 @@ def remote_git_repo(fair_repo_loc: str) -> str:
|
|
|
262
262
|
|
|
263
263
|
def get_remote_token(
|
|
264
264
|
repo_dir: str, remote: str = "origin", local: bool = False
|
|
265
|
-
) -> str:
|
|
265
|
+
) -> typing.Optional[str]:
|
|
266
266
|
_local_config = read_local_fdpconfig(repo_dir)
|
|
267
267
|
if remote not in _local_config["registries"]:
|
|
268
268
|
raise fdp_exc.CLIConfigurationError(
|
|
@@ -378,6 +378,22 @@ def get_current_user_uri(repo_loc: str) -> str:
|
|
|
378
378
|
raise fdp_exc.CLIConfigurationError("No user URI identifier defined.")
|
|
379
379
|
return _uri
|
|
380
380
|
|
|
381
|
+
def get_current_user_github(repo_loc: str) -> str:
|
|
382
|
+
"""Retrieves the URI identifier for the current user
|
|
383
|
+
|
|
384
|
+
Returns
|
|
385
|
+
-------
|
|
386
|
+
str
|
|
387
|
+
github username
|
|
388
|
+
"""
|
|
389
|
+
_local_conf = read_local_fdpconfig(repo_loc)
|
|
390
|
+
try:
|
|
391
|
+
_github = _local_conf["user"]["github"]
|
|
392
|
+
except KeyError:
|
|
393
|
+
_github = None
|
|
394
|
+
if not _github or _github == "None":
|
|
395
|
+
raise fdp_exc.CLIConfigurationError("No user GitHub username defined.")
|
|
396
|
+
return _github
|
|
381
397
|
|
|
382
398
|
def check_registry_exists(registry: str = None) -> typing.Optional[str]:
|
|
383
399
|
"""Checks if fair registry is set up on users machine
|
|
@@ -436,10 +452,10 @@ def get_local_port(local_uri: str = None) -> int:
|
|
|
436
452
|
return _port
|
|
437
453
|
|
|
438
454
|
|
|
439
|
-
def update_local_port() -> str:
|
|
455
|
+
def update_local_port(registry_dir: str = None) -> str:
|
|
440
456
|
"""Updates the local port in the global configuration from the session port file"""
|
|
441
|
-
_current_port = fdp_com.registry_session_port()
|
|
442
|
-
_current_address = fdp_com.registry_session_address()
|
|
457
|
+
_current_port = fdp_com.registry_session_port(registry_dir)
|
|
458
|
+
_current_address = fdp_com.registry_session_address(registry_dir)
|
|
443
459
|
|
|
444
460
|
_new_url = f'http://{_current_address}:{_current_port}/api/'
|
|
445
461
|
|
|
@@ -474,7 +490,7 @@ def _handle_orcid(user_orcid: str) -> typing.Tuple[typing.Dict, str]:
|
|
|
474
490
|
user_orcid = click.prompt("ORCID")
|
|
475
491
|
_user_info = fdp_id.check_orcid(user_orcid.strip())
|
|
476
492
|
|
|
477
|
-
_user_info["orcid"] = user_orcid
|
|
493
|
+
_user_info["orcid"] = user_orcid.strip()
|
|
478
494
|
|
|
479
495
|
click.echo(
|
|
480
496
|
f"Found entry: {_user_info['given_names']} {_user_info['family_name']}"
|
|
@@ -509,7 +525,7 @@ def _handle_ror(user_ror: str) -> typing.Tuple[typing.Dict, str]:
|
|
|
509
525
|
user_ror = click.prompt("ROR ID")
|
|
510
526
|
_user_info = fdp_id.check_ror(user_ror.strip())
|
|
511
527
|
|
|
512
|
-
_user_info["ror"] = user_ror
|
|
528
|
+
_user_info["ror"] = user_ror.strip()
|
|
513
529
|
|
|
514
530
|
click.echo(f"Found entry: {_user_info['family_name']} ")
|
|
515
531
|
|
|
@@ -540,7 +556,7 @@ def _handle_grid(user_grid: str) -> typing.Tuple[typing.Dict, str]:
|
|
|
540
556
|
user_grid = click.prompt("GRID ID")
|
|
541
557
|
_user_info = fdp_id.check_grid(user_grid.strip())
|
|
542
558
|
|
|
543
|
-
_user_info["grid"] = user_grid
|
|
559
|
+
_user_info["grid"] = user_grid.strip()
|
|
544
560
|
|
|
545
561
|
click.echo(f"Found entry: {_user_info['family_name']} ")
|
|
546
562
|
|
|
@@ -548,6 +564,22 @@ def _handle_grid(user_grid: str) -> typing.Tuple[typing.Dict, str]:
|
|
|
548
564
|
|
|
549
565
|
return _user_info, _def_ospace
|
|
550
566
|
|
|
567
|
+
def _handle_github(user_github: str) -> typing.Tuple[typing.Dict, str]:
|
|
568
|
+
|
|
569
|
+
_user_info = fdp_id.check_github(user_github.strip())
|
|
570
|
+
|
|
571
|
+
while not _user_info:
|
|
572
|
+
time.sleep(3)
|
|
573
|
+
click.echo(f"Invalid GitHub Username '{user_github}' given.")
|
|
574
|
+
user_github = click.prompt("GitHub Username")
|
|
575
|
+
_user_info = fdp_id.check_github(user_github.strip())
|
|
576
|
+
|
|
577
|
+
_def_ospace = _user_info["github"].lower()
|
|
578
|
+
|
|
579
|
+
click.echo(f"Found entry: {_user_info['github']} ")
|
|
580
|
+
return _user_info, _def_ospace
|
|
581
|
+
|
|
582
|
+
|
|
551
583
|
|
|
552
584
|
def _handle_uuid() -> typing.Tuple[typing.Dict, str]:
|
|
553
585
|
"""Obtain metadata for user where no ID provided
|
|
@@ -575,17 +607,24 @@ def _handle_uuid() -> typing.Tuple[typing.Dict, str]:
|
|
|
575
607
|
return _user_info, _def_ospace
|
|
576
608
|
|
|
577
609
|
|
|
578
|
-
def _get_user_info_and_namespaces() -> typing.Dict[str, typing.Dict]:
|
|
579
|
-
_user_email = click.prompt("Email")
|
|
610
|
+
def _get_user_info_and_namespaces(local: bool = False) -> typing.Dict[str, typing.Dict]:
|
|
611
|
+
_user_email = click.prompt("Email (optional)", default = "")
|
|
580
612
|
|
|
581
613
|
_invalid_input = True
|
|
582
614
|
|
|
583
615
|
while _invalid_input:
|
|
584
616
|
_id_type = click.prompt(
|
|
585
|
-
"User ID system (ORCID/ROR/GRID/None)", default="
|
|
617
|
+
"User ID system (GITHUB/ORCID/ROR/GRID/None)", default="GITHUB"
|
|
586
618
|
)
|
|
587
|
-
|
|
588
|
-
|
|
619
|
+
if _id_type.upper() == "GITHUB":
|
|
620
|
+
_user_github = click.prompt("GitHub Username")
|
|
621
|
+
_user_info, _def_ospace = _handle_github(_user_github)
|
|
622
|
+
if not _user_info["name"]:
|
|
623
|
+
_user_info["given_names"] = click.prompt("Given Names")
|
|
624
|
+
_user_info["family_name"] = click.prompt("Family Name")
|
|
625
|
+
_user_info["name"] = " ".join([_user_info["given_names"], _user_info["family_name"]])
|
|
626
|
+
_invalid_input = False
|
|
627
|
+
elif _id_type.upper() == "ORCID":
|
|
589
628
|
_user_orcid = click.prompt("ORCID")
|
|
590
629
|
_user_info, _def_ospace = _handle_orcid(_user_orcid)
|
|
591
630
|
_invalid_input = False
|
|
@@ -603,14 +642,25 @@ def _get_user_info_and_namespaces() -> typing.Dict[str, typing.Dict]:
|
|
|
603
642
|
_user_info["uuid"] = _user_uuid
|
|
604
643
|
_invalid_input = False
|
|
605
644
|
|
|
606
|
-
_user_info["email"] = _user_email
|
|
607
|
-
|
|
608
645
|
_def_ospace = _def_ospace.lower().replace(" ", "").strip()
|
|
609
646
|
_def_ospace = click.prompt("Default output namespace", default=_def_ospace)
|
|
610
647
|
_def_ispace = click.prompt("Default input namespace", default=_def_ospace)
|
|
611
648
|
|
|
612
649
|
_namespaces = {"input": _def_ispace, "output": _def_ospace}
|
|
613
650
|
|
|
651
|
+
if not "github" in _user_info:
|
|
652
|
+
if local:
|
|
653
|
+
_user_info["github"] = "FAIRDataPipeline"
|
|
654
|
+
else:
|
|
655
|
+
_user_github = click.prompt("GitHub Username")
|
|
656
|
+
_user_github_info = _handle_github(_user_github)[0]
|
|
657
|
+
_user_info["github"] = _user_github_info["github"]
|
|
658
|
+
|
|
659
|
+
if not _user_email:
|
|
660
|
+
_user_info["email"] = f'{_user_info["github"]}@users.noreply.github.com'
|
|
661
|
+
else:
|
|
662
|
+
_user_info["email"] = _user_email
|
|
663
|
+
|
|
614
664
|
return {"user": _user_info, "namespaces": _namespaces}
|
|
615
665
|
|
|
616
666
|
|
|
@@ -668,21 +718,23 @@ def global_config_query(
|
|
|
668
718
|
default=_remote_url.replace("/api/", "/data/"),
|
|
669
719
|
)
|
|
670
720
|
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
while (
|
|
677
|
-
not os.path.exists(_rem_key_file)
|
|
678
|
-
or not open(_rem_key_file).read().strip()
|
|
679
|
-
):
|
|
680
|
-
click.echo(
|
|
681
|
-
f"Token file '{_rem_key_file}' does not exist or is empty, "
|
|
682
|
-
"please provide a valid token file."
|
|
721
|
+
_rem_key_valid = False
|
|
722
|
+
while not _rem_key_valid:
|
|
723
|
+
_rem_key = click.prompt(
|
|
724
|
+
f"Remote API Token",
|
|
683
725
|
)
|
|
684
|
-
|
|
685
|
-
|
|
726
|
+
if len(_rem_key) == 40:
|
|
727
|
+
_rem_key_valid = True
|
|
728
|
+
else:
|
|
729
|
+
click.echo("Remote token should be 40 characters long")
|
|
730
|
+
|
|
731
|
+
_rem_key_file = os.path.join(fdp_com.global_config_dir(), "remotetoken.txt")
|
|
732
|
+
|
|
733
|
+
with open(_rem_key_file, 'w') as f:
|
|
734
|
+
f.write(_rem_key)
|
|
735
|
+
|
|
736
|
+
if not os.path.exists(_rem_key_file):
|
|
737
|
+
raise fdp_exc.CLIConfigurationError(f'Token could not be written to {_rem_key_file}')
|
|
686
738
|
|
|
687
739
|
if not fdp_serv.check_server_running():
|
|
688
740
|
if _ := click.confirm(
|
|
@@ -696,7 +748,7 @@ def global_config_query(
|
|
|
696
748
|
pathlib.Path(_cache_addr).touch()
|
|
697
749
|
|
|
698
750
|
else:
|
|
699
|
-
click.echo("Temporarily launching
|
|
751
|
+
click.echo("Temporarily launching local registry to retrieve API token.")
|
|
700
752
|
fdp_serv.launch_server(registry_dir=registry)
|
|
701
753
|
fdp_serv.stop_server(registry_dir=registry, local_uri=_local_uri)
|
|
702
754
|
try:
|
|
@@ -713,7 +765,7 @@ def global_config_query(
|
|
|
713
765
|
if _loc_data_store[-1] != os.path.sep:
|
|
714
766
|
_loc_data_store += os.path.sep
|
|
715
767
|
|
|
716
|
-
_glob_conf_dict = _get_user_info_and_namespaces()
|
|
768
|
+
_glob_conf_dict = _get_user_info_and_namespaces(local)
|
|
717
769
|
_glob_conf_dict["registries"] = {
|
|
718
770
|
"local": {
|
|
719
771
|
"uri": _local_uri,
|
|
@@ -108,6 +108,9 @@ class User(pydantic.BaseModel):
|
|
|
108
108
|
name: typing.Optional[str] = pydantic.Field(
|
|
109
109
|
None, title="Full Name", description="Full name for the user"
|
|
110
110
|
)
|
|
111
|
+
github: typing.Optional[str] = pydantic.Field(
|
|
112
|
+
None, title="GitHub Username", description="GitHub Username for the user"
|
|
113
|
+
)
|
|
111
114
|
|
|
112
115
|
class Config:
|
|
113
116
|
extra = "forbid"
|
|
@@ -22,6 +22,8 @@ import os
|
|
|
22
22
|
import click
|
|
23
23
|
import rich
|
|
24
24
|
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
|
|
25
27
|
import fair.common as fdp_com
|
|
26
28
|
import fair.exceptions as fdp_exc
|
|
27
29
|
import fair.run as fdp_run
|
|
@@ -61,8 +63,10 @@ def show_job_log(repo_loc: str, job_id: str) -> str:
|
|
|
61
63
|
str
|
|
62
64
|
log file location for the given job
|
|
63
65
|
"""
|
|
66
|
+
_job_dir = Path(f"{fdp_com.default_jobs_dir()}")
|
|
67
|
+
|
|
64
68
|
_sorted_time_dirs = sorted(
|
|
65
|
-
glob.glob(os.path.join(
|
|
69
|
+
glob.glob(os.path.join(_job_dir, "*")), reverse=True
|
|
66
70
|
)
|
|
67
71
|
|
|
68
72
|
_log_files = [
|
|
@@ -107,8 +111,10 @@ def show_history(repo_loc: str, length: int = 10) -> None:
|
|
|
107
111
|
max number of entries to display, by default 10
|
|
108
112
|
"""
|
|
109
113
|
|
|
114
|
+
_job_dir = Path(f"{fdp_com.default_jobs_dir()}")
|
|
115
|
+
|
|
110
116
|
_sorted_time_dirs = sorted(
|
|
111
|
-
glob.glob(os.path.join(
|
|
117
|
+
glob.glob(os.path.join(_job_dir, "*")), reverse=True
|
|
112
118
|
)
|
|
113
119
|
|
|
114
120
|
_log_files = [
|
|
@@ -24,15 +24,22 @@ import urllib.parse
|
|
|
24
24
|
|
|
25
25
|
import requests
|
|
26
26
|
import requests.exceptions
|
|
27
|
+
import logging
|
|
28
|
+
from urllib3.exceptions import InsecureRequestWarning
|
|
29
|
+
from fake_useragent import UserAgent
|
|
30
|
+
|
|
31
|
+
logger = logging.getLogger("FAIRDataPipeline.Identifiers")
|
|
27
32
|
|
|
28
33
|
ID_URIS = {
|
|
29
34
|
"orcid": "https://orcid.org/",
|
|
30
35
|
"ror": "https://ror.org/",
|
|
36
|
+
"github" : "https://github.com/"
|
|
31
37
|
}
|
|
32
38
|
|
|
33
39
|
QUERY_URLS = {
|
|
34
40
|
"orcid": "https://pub.orcid.org/v3.0/",
|
|
35
41
|
"ror": "https://api.ror.org/organizations?query=",
|
|
42
|
+
"github": "https://api.github.com/users/"
|
|
36
43
|
}
|
|
37
44
|
|
|
38
45
|
|
|
@@ -49,14 +56,16 @@ def check_orcid(orcid: str) -> typing.Dict:
|
|
|
49
56
|
typing.Dict
|
|
50
57
|
metadata from the given ID
|
|
51
58
|
"""
|
|
52
|
-
|
|
59
|
+
orcid = orcid.replace(ID_URIS["orcid"], "")
|
|
53
60
|
_header = {"Accept": "application/json"}
|
|
54
61
|
_url = urllib.parse.urljoin(QUERY_URLS["orcid"], orcid)
|
|
55
|
-
|
|
62
|
+
requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
|
|
63
|
+
_response = requests.get(_url, headers=_header, verify = False, allow_redirects = True)
|
|
56
64
|
|
|
57
65
|
_result_dict: typing.Dict[str, typing.Any] = {}
|
|
58
66
|
|
|
59
67
|
if _response.status_code != 200:
|
|
68
|
+
logger.debug(f"{_url} Responded with {_response.status_code}")
|
|
60
69
|
return _result_dict
|
|
61
70
|
|
|
62
71
|
_names = _response.json()["person"]["name"]
|
|
@@ -72,6 +81,42 @@ def check_orcid(orcid: str) -> typing.Dict:
|
|
|
72
81
|
|
|
73
82
|
return _result_dict
|
|
74
83
|
|
|
84
|
+
def check_github(github: str) -> typing.Dict:
|
|
85
|
+
"""Checks if valid ORCID using ORCID public api
|
|
86
|
+
|
|
87
|
+
Parameters
|
|
88
|
+
----------
|
|
89
|
+
github : str
|
|
90
|
+
github username to be checked
|
|
91
|
+
|
|
92
|
+
Returns
|
|
93
|
+
-------
|
|
94
|
+
typing.Dict
|
|
95
|
+
metadata from the given ID
|
|
96
|
+
"""
|
|
97
|
+
_header = {"Accept": "application/json"}
|
|
98
|
+
_url = urllib.parse.urljoin(QUERY_URLS["github"], github)
|
|
99
|
+
requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
|
|
100
|
+
_response = requests.get(_url, headers=_header, verify = False, allow_redirects = True)
|
|
101
|
+
|
|
102
|
+
_result_dict: typing.Dict[str, typing.Any] = {}
|
|
103
|
+
|
|
104
|
+
if _response.status_code != 200:
|
|
105
|
+
logger.debug(f"{_url} Responded with {_response.status_code}")
|
|
106
|
+
return _result_dict
|
|
107
|
+
|
|
108
|
+
_login = _response.json()["login"]
|
|
109
|
+
_name = _response.json()["name"]
|
|
110
|
+
if _name:
|
|
111
|
+
_result_dict["family_name"] = _name.split()[-1]
|
|
112
|
+
_result_dict["given_names"] = " ".join(_name.split()[:-1])
|
|
113
|
+
|
|
114
|
+
_result_dict["name"] = _name
|
|
115
|
+
_result_dict["github"] = _login
|
|
116
|
+
_result_dict["uri"] = f'{ID_URIS["github"]}{_login}'
|
|
117
|
+
|
|
118
|
+
return _result_dict
|
|
119
|
+
|
|
75
120
|
|
|
76
121
|
def check_ror(ror: str) -> typing.Dict:
|
|
77
122
|
"""Checks if valid ROR using ROR public api
|
|
@@ -86,7 +131,6 @@ def check_ror(ror: str) -> typing.Dict:
|
|
|
86
131
|
typing.Dict
|
|
87
132
|
metadata from the given ID
|
|
88
133
|
"""
|
|
89
|
-
|
|
90
134
|
_result_dict = _check_generic_ror(ror)
|
|
91
135
|
if _result_dict:
|
|
92
136
|
_result_dict["ror"] = ror
|
|
@@ -124,13 +168,13 @@ def _check_generic_ror(id: str) -> typing.Dict:
|
|
|
124
168
|
typing.Dict
|
|
125
169
|
metadata from the given ID
|
|
126
170
|
"""
|
|
127
|
-
|
|
128
171
|
_url = f"{QUERY_URLS['ror']}{id}"
|
|
129
172
|
_response = requests.get(_url)
|
|
130
173
|
|
|
131
174
|
_result_dict: typing.Dict[str, typing.Any] = {}
|
|
132
175
|
|
|
133
176
|
if _response.status_code != 200:
|
|
177
|
+
logger.debug(f"{_url} Responded with {_response.status_code}")
|
|
134
178
|
return _result_dict
|
|
135
179
|
|
|
136
180
|
if _response.json()["number_of_results"] == 0:
|
|
@@ -164,18 +208,25 @@ def check_id_permitted(identifier: str, retries: int = 5) -> bool:
|
|
|
164
208
|
if valid identifier
|
|
165
209
|
"""
|
|
166
210
|
_n_attempts = 0
|
|
211
|
+
fake_agent = False
|
|
167
212
|
|
|
168
213
|
while _n_attempts < retries:
|
|
169
214
|
try:
|
|
170
|
-
requests.
|
|
215
|
+
requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
|
|
216
|
+
headers = {}
|
|
217
|
+
if fake_agent:
|
|
218
|
+
headers = {'User-Agent':str(UserAgent().chrome)}
|
|
219
|
+
requests.get(identifier, verify = False, allow_redirects = True, headers = headers).raise_for_status()
|
|
171
220
|
return True
|
|
172
221
|
except (
|
|
173
222
|
requests.exceptions.MissingSchema,
|
|
174
223
|
requests.exceptions.HTTPError,
|
|
175
224
|
requests.exceptions.ConnectionError,
|
|
176
|
-
):
|
|
225
|
+
) as e:
|
|
177
226
|
_n_attempts += 1
|
|
178
|
-
time.sleep(
|
|
227
|
+
time.sleep(3)
|
|
228
|
+
fake_agent = True
|
|
229
|
+
logger.warning(f"Error identifier: '{identifier}' caused '{e}'")
|
|
179
230
|
continue
|
|
180
231
|
|
|
181
232
|
return False
|