cloudos-cli 2.17.0__tar.gz → 2.19.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/PKG-INFO +6 -3
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/README.md +2 -1
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/cloudos_cli/__main__.py +59 -15
- cloudos_cli-2.19.0/cloudos_cli/_version.py +1 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/cloudos_cli/clos.py +44 -11
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/cloudos_cli/jobs/job.py +4 -11
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/cloudos_cli.egg-info/PKG-INFO +6 -3
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/cloudos_cli.egg-info/requires.txt +1 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/setup.py +1 -1
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/tests/test_clos/test_get_project_list.py +15 -10
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/tests/test_clos/test_process_project_list.py +3 -3
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/tests/test_jobs/test_project_id.py +3 -3
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/tests/test_jobs/test_send_job.py +4 -2
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/tests/test_jobs/test_workflow_id.py +3 -3
- cloudos_cli-2.17.0/cloudos_cli/_version.py +0 -1
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/LICENSE +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/cloudos_cli/__init__.py +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/cloudos_cli/jobs/__init__.py +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/cloudos_cli/queue/__init__.py +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/cloudos_cli/queue/queue.py +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/cloudos_cli/utils/__init__.py +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/cloudos_cli/utils/errors.py +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/cloudos_cli/utils/requests.py +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/cloudos_cli.egg-info/SOURCES.txt +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/cloudos_cli.egg-info/dependency_links.txt +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/cloudos_cli.egg-info/entry_points.txt +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/cloudos_cli.egg-info/top_level.txt +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/setup.cfg +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/tests/__init__.py +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/tests/functions_for_pytest.py +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/tests/test_clos/__init__.py +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/tests/test_clos/test_create_cromwell_header.py +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/tests/test_clos/test_cromwell_switch.py +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/tests/test_clos/test_detect_workflow.py +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/tests/test_clos/test_get_cromwell_status.py +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/tests/test_clos/test_get_curated_workflow_list.py +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/tests/test_clos/test_get_job_list.py +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/tests/test_clos/test_get_job_status.py +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/tests/test_clos/test_get_user_info.py +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/tests/test_clos/test_get_workflow_list.py +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/tests/test_clos/test_is_module.py +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/tests/test_clos/test_process_job_list.py +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/tests/test_clos/test_process_workflow_list.py +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/tests/test_clos/test_wait_job_completion.py +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/tests/test_clos/test_workflow_import.py +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/tests/test_jobs/__init__.py +0 -0
- {cloudos_cli-2.17.0 → cloudos_cli-2.19.0}/tests/test_jobs/test_convert_nextflow_to_json.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: cloudos_cli
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.19.0
|
|
4
4
|
Summary: Python package for interacting with CloudOS
|
|
5
5
|
Home-page: https://github.com/lifebit-ai/cloudos-cli
|
|
6
6
|
Author: David Piñeyro
|
|
@@ -12,6 +12,7 @@ Requires-Python: >=3.7
|
|
|
12
12
|
Description-Content-Type: text/markdown
|
|
13
13
|
License-File: LICENSE
|
|
14
14
|
Requires-Dist: click>=8.0.1
|
|
15
|
+
Requires-Dist: rich-click>=1.8.2
|
|
15
16
|
Requires-Dist: pandas>=1.3.4
|
|
16
17
|
Requires-Dist: numpy==1.26.4
|
|
17
18
|
Requires-Dist: requests>=2.26.0
|
|
@@ -26,6 +27,7 @@ Dynamic: classifier
|
|
|
26
27
|
Dynamic: description
|
|
27
28
|
Dynamic: description-content-type
|
|
28
29
|
Dynamic: home-page
|
|
30
|
+
Dynamic: license-file
|
|
29
31
|
Dynamic: provides-extra
|
|
30
32
|
Dynamic: requires-dist
|
|
31
33
|
Dynamic: requires-python
|
|
@@ -46,6 +48,7 @@ click>=8.0.1
|
|
|
46
48
|
pandas>=1.3.4
|
|
47
49
|
numpy==1.26.4
|
|
48
50
|
requests>=2.26.0
|
|
51
|
+
rich_click>=1.8.2
|
|
49
52
|
```
|
|
50
53
|
|
|
51
54
|
## Installation
|
|
@@ -231,7 +234,7 @@ CLOUDOS="https://cloudos.lifebit.ai"
|
|
|
231
234
|
WORKSPACE_ID="xxxxx"
|
|
232
235
|
PROJECT_NAME="API jobs"
|
|
233
236
|
WORKFLOW_NAME="rnatoy"
|
|
234
|
-
JOB_PARAMS="
|
|
237
|
+
JOB_PARAMS="cloudos_cli/examples/rnatoy.config"
|
|
235
238
|
```
|
|
236
239
|
|
|
237
240
|
As you can see, a file with the job parameters is used to configure the
|
|
@@ -13,6 +13,7 @@ click>=8.0.1
|
|
|
13
13
|
pandas>=1.3.4
|
|
14
14
|
numpy==1.26.4
|
|
15
15
|
requests>=2.26.0
|
|
16
|
+
rich_click>=1.8.2
|
|
16
17
|
```
|
|
17
18
|
|
|
18
19
|
## Installation
|
|
@@ -198,7 +199,7 @@ CLOUDOS="https://cloudos.lifebit.ai"
|
|
|
198
199
|
WORKSPACE_ID="xxxxx"
|
|
199
200
|
PROJECT_NAME="API jobs"
|
|
200
201
|
WORKFLOW_NAME="rnatoy"
|
|
201
|
-
JOB_PARAMS="
|
|
202
|
+
JOB_PARAMS="cloudos_cli/examples/rnatoy.config"
|
|
202
203
|
```
|
|
203
204
|
|
|
204
205
|
As you can see, a file with the job parameters is used to configure the
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
|
|
3
|
-
import click
|
|
3
|
+
import rich_click as click
|
|
4
4
|
import cloudos_cli.jobs.job as jb
|
|
5
5
|
from cloudos_cli.clos import Cloudos
|
|
6
6
|
from cloudos_cli.queue.queue import Queue
|
|
@@ -18,6 +18,12 @@ JOB_FAILED = 'failed'
|
|
|
18
18
|
JOB_ABORTED = 'aborted'
|
|
19
19
|
JOB_RUNNING = 'running'
|
|
20
20
|
REQUEST_INTERVAL_CROMWELL = 30
|
|
21
|
+
AWS_NEXTFLOW_VERSIONS = ['22.10.8', '24.04.4']
|
|
22
|
+
AZURE_NEXTFLOW_VERSIONS = ['22.11.1-edge']
|
|
23
|
+
HPC_NEXTFLOW_VERSIONS = ['22.10.8']
|
|
24
|
+
AWS_NEXTFLOW_LATEST = '24.04.4'
|
|
25
|
+
AZURE_NEXTFLOW_LATEST = '22.11.1-edge'
|
|
26
|
+
HPC_NEXTFLOW_LATEST = '22.10.8'
|
|
21
27
|
|
|
22
28
|
|
|
23
29
|
def ssl_selector(disable_ssl_verification, ssl_cert):
|
|
@@ -126,7 +132,7 @@ def queue():
|
|
|
126
132
|
help=('Nextflow version to use when executing the workflow in CloudOS. ' +
|
|
127
133
|
'Please, note that versions above 22.10.8 are only DSL2 compatible. ' +
|
|
128
134
|
'Default=22.10.8.'),
|
|
129
|
-
type=click.Choice(['22.10.8', '24.04.4', 'latest']),
|
|
135
|
+
type=click.Choice(['22.10.8', '24.04.4', '22.11.1-edge', 'latest']),
|
|
130
136
|
default='22.10.8')
|
|
131
137
|
@click.option('--git-commit',
|
|
132
138
|
help=('The exact whole 40 character commit hash to run for ' +
|
|
@@ -410,9 +416,26 @@ def run(apikey,
|
|
|
410
416
|
else:
|
|
411
417
|
docker_login = False
|
|
412
418
|
if nextflow_version == 'latest':
|
|
413
|
-
|
|
414
|
-
|
|
419
|
+
if execution_platform == 'aws':
|
|
420
|
+
nextflow_version = AWS_NEXTFLOW_LATEST
|
|
421
|
+
elif execution_platform == 'azure':
|
|
422
|
+
nextflow_version = AZURE_NEXTFLOW_LATEST
|
|
423
|
+
else:
|
|
424
|
+
nextflow_version = HPC_NEXTFLOW_LATEST
|
|
425
|
+
print(f'[Message] You have specified Nextflow version \'latest\' for execution platform \'{execution_platform}\'. The workflow will use the ' +
|
|
415
426
|
f'latest version available on CloudOS: {nextflow_version}.')
|
|
427
|
+
if execution_platform == 'aws':
|
|
428
|
+
if nextflow_version not in AWS_NEXTFLOW_VERSIONS:
|
|
429
|
+
print(f'[Message] For execution platform \'aws\', the workflow will use the default \'22.10.8\' version on CloudOS.')
|
|
430
|
+
nextflow_version = '22.10.8'
|
|
431
|
+
if execution_platform == 'azure':
|
|
432
|
+
if nextflow_version not in AZURE_NEXTFLOW_VERSIONS:
|
|
433
|
+
print(f'[Message] For execution platform \'azure\', the workflow will use the \'22.11.1-edge\' version on CloudOS.')
|
|
434
|
+
nextflow_version = '22.11.1-edge'
|
|
435
|
+
if execution_platform == 'hpc':
|
|
436
|
+
if nextflow_version not in HPC_NEXTFLOW_VERSIONS:
|
|
437
|
+
print(f'[Message] For execution platform \'hpc\', the workflow will use the \'22.10.8\' version on CloudOS.')
|
|
438
|
+
nextflow_version = '22.10.8'
|
|
416
439
|
if nextflow_version != '22.10.8':
|
|
417
440
|
print(f'[Warning] You have specified Nextflow version {nextflow_version}. This version requires the pipeline ' +
|
|
418
441
|
'to be written in DSL2 and does not support DSL1.')
|
|
@@ -807,6 +830,10 @@ def list_jobs(apikey,
|
|
|
807
830
|
print('\t' + str(cl) + '\n')
|
|
808
831
|
print('\tSearching for jobs in the following workspace: ' +
|
|
809
832
|
f'{workspace_id}')
|
|
833
|
+
# Check if the user provided the --page option
|
|
834
|
+
ctx = click.get_current_context()
|
|
835
|
+
if not isinstance(page, int) or page < 1:
|
|
836
|
+
raise ValueError('Please, use a positive integer (>= 1) for the --page parameter')
|
|
810
837
|
if last_n_jobs != 'all':
|
|
811
838
|
try:
|
|
812
839
|
last_n_jobs = int(last_n_jobs)
|
|
@@ -814,7 +841,12 @@ def list_jobs(apikey,
|
|
|
814
841
|
print("[ERROR] last-n-jobs value was not valid. Please use a positive int or 'all'")
|
|
815
842
|
raise
|
|
816
843
|
my_jobs_r = cl.get_job_list(workspace_id, last_n_jobs, page, archived, verify_ssl)
|
|
817
|
-
if
|
|
844
|
+
if len(my_jobs_r) == 0:
|
|
845
|
+
if ctx.get_parameter_source('page') == click.core.ParameterSource.DEFAULT:
|
|
846
|
+
print('\t[Message] A total of 0 jobs collected. This is likely because your workspace has no jobs created yet.')
|
|
847
|
+
else:
|
|
848
|
+
print('\t[Message] A total of 0 jobs collected. This is likely because the --page you requested does not exist. Please, try a smaller number for --page or collect all the jobs by not using --page parameter.')
|
|
849
|
+
elif output_format == 'csv':
|
|
818
850
|
my_jobs = cl.process_job_list(my_jobs_r, all_fields)
|
|
819
851
|
my_jobs.to_csv(outfile, index=False)
|
|
820
852
|
print(f'\tJob list collected with a total of {my_jobs.shape[0]} jobs.')
|
|
@@ -997,6 +1029,10 @@ def import_workflows(apikey,
|
|
|
997
1029
|
'just the preconfigured selected fields. Only applicable ' +
|
|
998
1030
|
'when --output-format=csv'),
|
|
999
1031
|
is_flag=True)
|
|
1032
|
+
@click.option('--page',
|
|
1033
|
+
help=('Response page to retrieve. Default=1.'),
|
|
1034
|
+
type=int,
|
|
1035
|
+
default=1)
|
|
1000
1036
|
@click.option('--verbose',
|
|
1001
1037
|
help='Whether to print information messages or not.',
|
|
1002
1038
|
is_flag=True)
|
|
@@ -1012,6 +1048,7 @@ def list_projects(apikey,
|
|
|
1012
1048
|
output_basename,
|
|
1013
1049
|
output_format,
|
|
1014
1050
|
all_fields,
|
|
1051
|
+
page,
|
|
1015
1052
|
verbose,
|
|
1016
1053
|
disable_ssl_verification,
|
|
1017
1054
|
ssl_cert):
|
|
@@ -1028,21 +1065,28 @@ def list_projects(apikey,
|
|
|
1028
1065
|
print('\t' + str(cl) + '\n')
|
|
1029
1066
|
print('\tSearching for projects in the following workspace: ' +
|
|
1030
1067
|
f'{workspace_id}')
|
|
1031
|
-
|
|
1032
|
-
|
|
1068
|
+
# Check if the user provided the --page option
|
|
1069
|
+
ctx = click.get_current_context()
|
|
1070
|
+
if ctx.get_parameter_source('page') == click.core.ParameterSource.DEFAULT:
|
|
1071
|
+
get_all = True
|
|
1072
|
+
else:
|
|
1073
|
+
get_all = False
|
|
1074
|
+
if not isinstance(page, int) or page < 1:
|
|
1075
|
+
raise ValueError('Please, use a positive integer (>= 1) for the --page parameter')
|
|
1076
|
+
my_projects_r = cl.get_project_list(workspace_id, verify_ssl, page=page, get_all=get_all)
|
|
1077
|
+
if len(my_projects_r) == 0:
|
|
1078
|
+
if ctx.get_parameter_source('page') == click.core.ParameterSource.DEFAULT:
|
|
1079
|
+
print('\t[Message] A total of 0 projects collected. This is likely because your workspace has no projects created yet.')
|
|
1080
|
+
else:
|
|
1081
|
+
print('\t[Message] A total of 0 projects collected. This is likely because the --page you requested does not exist. Please, try a smaller number for --page or collect all the projects by not using --page parameter.')
|
|
1082
|
+
elif output_format == 'csv':
|
|
1033
1083
|
my_projects = cl.process_project_list(my_projects_r, all_fields)
|
|
1034
1084
|
my_projects.to_csv(outfile, index=False)
|
|
1035
1085
|
print(f'\tProject list collected with a total of {my_projects.shape[0]} projects.')
|
|
1036
1086
|
elif output_format == 'json':
|
|
1037
1087
|
with open(outfile, 'w') as o:
|
|
1038
|
-
o.write(my_projects_r
|
|
1039
|
-
|
|
1040
|
-
if 'projects' in content:
|
|
1041
|
-
content_l = len(content['projects'])
|
|
1042
|
-
else:
|
|
1043
|
-
content_l = len(content)
|
|
1044
|
-
print('\tProject list collected with a total of ' +
|
|
1045
|
-
f'{content_l} projects.')
|
|
1088
|
+
o.write(json.dumps(my_projects_r))
|
|
1089
|
+
print(f'\tProject list collected with a total of {len(my_projects_r)} projects.')
|
|
1046
1090
|
else:
|
|
1047
1091
|
raise ValueError('Unrecognised output format. Please use one of [csv|json]')
|
|
1048
1092
|
print(f'\tProject list saved to {outfile}')
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = '2.19.0'
|
|
@@ -313,6 +313,8 @@ class Cloudos:
|
|
|
313
313
|
'project.updatedAt'
|
|
314
314
|
]
|
|
315
315
|
df_full = pd.json_normalize(r)
|
|
316
|
+
if df_full.empty:
|
|
317
|
+
return df_full
|
|
316
318
|
if all_fields:
|
|
317
319
|
df = df_full
|
|
318
320
|
else:
|
|
@@ -548,7 +550,8 @@ class Cloudos:
|
|
|
548
550
|
else:
|
|
549
551
|
return False
|
|
550
552
|
|
|
551
|
-
def get_project_list(self, workspace_id, verify=True
|
|
553
|
+
def get_project_list(self, workspace_id, verify=True, get_all=True,
|
|
554
|
+
page=1, page_size=10, max_page_size=1000):
|
|
552
555
|
"""Get all the project from a CloudOS workspace.
|
|
553
556
|
|
|
554
557
|
Parameters
|
|
@@ -559,6 +562,15 @@ class Cloudos:
|
|
|
559
562
|
Whether to use SSL verification or not. Alternatively, if
|
|
560
563
|
a string is passed, it will be interpreted as the path to
|
|
561
564
|
the SSL certificate file.
|
|
565
|
+
get_all : bool
|
|
566
|
+
Whether to get all available curated workflows or just the
|
|
567
|
+
indicated page.
|
|
568
|
+
page : int
|
|
569
|
+
The page number to retrieve, from the paginated response.
|
|
570
|
+
page_size : int
|
|
571
|
+
The number of workflows by page. From 1 to 1000.
|
|
572
|
+
max_page_size : int
|
|
573
|
+
Max page size defined by the API server. It is currently 1000.
|
|
562
574
|
|
|
563
575
|
Returns
|
|
564
576
|
-------
|
|
@@ -569,11 +581,36 @@ class Cloudos:
|
|
|
569
581
|
"Content-type": "application/json",
|
|
570
582
|
"apikey": self.apikey
|
|
571
583
|
}
|
|
572
|
-
r = retry_requests_get("{}/api/
|
|
584
|
+
r = retry_requests_get("{}/api/v2/projects?teamId={}&pageSize={}&page={}".format(self.cloudos_url, workspace_id, page_size, page),
|
|
573
585
|
headers=headers, verify=verify)
|
|
574
586
|
if r.status_code >= 400:
|
|
575
587
|
raise BadRequestException(r)
|
|
576
|
-
|
|
588
|
+
content = json.loads(r.content)
|
|
589
|
+
if get_all:
|
|
590
|
+
total_projects = content['total']
|
|
591
|
+
if total_projects <= max_page_size:
|
|
592
|
+
r = retry_requests_get("{}/api/v2/projects?teamId={}&pageSize={}&page={}".format(self.cloudos_url, workspace_id, total_projects, 1),
|
|
593
|
+
headers=headers, verify=verify)
|
|
594
|
+
if r.status_code >= 400:
|
|
595
|
+
raise BadRequestException(r)
|
|
596
|
+
return json.loads(r.content)['projects']
|
|
597
|
+
else:
|
|
598
|
+
n_pages = (total_projects // max_page_size) + int((total_projects % max_page_size) > 0)
|
|
599
|
+
for p in range(n_pages):
|
|
600
|
+
p += 1
|
|
601
|
+
r = retry_requests_get(
|
|
602
|
+
"{}/api/v2/projects?teamId={}&pageSize={}&page={}".format(
|
|
603
|
+
self.cloudos_url, workspace_id, max_page_size, p),
|
|
604
|
+
headers=headers, verify=verify)
|
|
605
|
+
if r.status_code >= 400:
|
|
606
|
+
raise BadRequestException(r)
|
|
607
|
+
if p == 1:
|
|
608
|
+
all_content_p = json.loads(r.content)['projects']
|
|
609
|
+
else:
|
|
610
|
+
all_content_p += json.loads(r.content)['projects']
|
|
611
|
+
return all_content_p
|
|
612
|
+
else:
|
|
613
|
+
return content['projects']
|
|
577
614
|
|
|
578
615
|
@staticmethod
|
|
579
616
|
def process_project_list(r, all_fields=False):
|
|
@@ -582,10 +619,7 @@ class Cloudos:
|
|
|
582
619
|
Parameters
|
|
583
620
|
----------
|
|
584
621
|
r : requests.models.Response
|
|
585
|
-
|
|
586
|
-
- A list with 2 elements: 'total' and 'projects', being 'projects' a list of dicts,
|
|
587
|
-
one for each project.
|
|
588
|
-
- A list of dicts, one for each project.
|
|
622
|
+
A list of dicts, each corresponding to a project.
|
|
589
623
|
all_fields : bool. Default=False
|
|
590
624
|
Whether to return a reduced version of the DataFrame containing
|
|
591
625
|
only the selected columns or the full DataFrame.
|
|
@@ -607,10 +641,9 @@ class Cloudos:
|
|
|
607
641
|
'jobCount',
|
|
608
642
|
'notebookSessionCount'
|
|
609
643
|
]
|
|
610
|
-
|
|
611
|
-
if
|
|
612
|
-
|
|
613
|
-
df_full = pd.json_normalize(my_projects)
|
|
644
|
+
df_full = pd.json_normalize(r)
|
|
645
|
+
if df_full.empty:
|
|
646
|
+
return df_full
|
|
614
647
|
if all_fields:
|
|
615
648
|
df = df_full
|
|
616
649
|
else:
|
|
@@ -157,18 +157,11 @@ class Job(Cloudos):
|
|
|
157
157
|
elif "importsFile" in element.keys() and element["importsFile"] == importsfile:
|
|
158
158
|
return element["_id"]
|
|
159
159
|
elif resource == 'projects':
|
|
160
|
-
|
|
161
|
-
content = json.loads(r.content)
|
|
160
|
+
content = self.get_project_list(workspace_id, verify=verify)
|
|
162
161
|
# New API projects endpoint spec
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
return element["_id"]
|
|
167
|
-
# Old API projects endpoint spec added for backwards compatibility
|
|
168
|
-
elif type(content) is list:
|
|
169
|
-
for element in content:
|
|
170
|
-
if element["name"] == name:
|
|
171
|
-
return element["_id"]
|
|
162
|
+
for element in content:
|
|
163
|
+
if element["name"] == name:
|
|
164
|
+
return element["_id"]
|
|
172
165
|
if mainfile is not None:
|
|
173
166
|
raise ValueError(f'[ERROR] A workflow named \'{name}\' with a mainFile \'{mainfile}\'' +
|
|
174
167
|
f' and an importsFile \'{importsfile}\' was not found')
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: cloudos_cli
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.19.0
|
|
4
4
|
Summary: Python package for interacting with CloudOS
|
|
5
5
|
Home-page: https://github.com/lifebit-ai/cloudos-cli
|
|
6
6
|
Author: David Piñeyro
|
|
@@ -12,6 +12,7 @@ Requires-Python: >=3.7
|
|
|
12
12
|
Description-Content-Type: text/markdown
|
|
13
13
|
License-File: LICENSE
|
|
14
14
|
Requires-Dist: click>=8.0.1
|
|
15
|
+
Requires-Dist: rich-click>=1.8.2
|
|
15
16
|
Requires-Dist: pandas>=1.3.4
|
|
16
17
|
Requires-Dist: numpy==1.26.4
|
|
17
18
|
Requires-Dist: requests>=2.26.0
|
|
@@ -26,6 +27,7 @@ Dynamic: classifier
|
|
|
26
27
|
Dynamic: description
|
|
27
28
|
Dynamic: description-content-type
|
|
28
29
|
Dynamic: home-page
|
|
30
|
+
Dynamic: license-file
|
|
29
31
|
Dynamic: provides-extra
|
|
30
32
|
Dynamic: requires-dist
|
|
31
33
|
Dynamic: requires-python
|
|
@@ -46,6 +48,7 @@ click>=8.0.1
|
|
|
46
48
|
pandas>=1.3.4
|
|
47
49
|
numpy==1.26.4
|
|
48
50
|
requests>=2.26.0
|
|
51
|
+
rich_click>=1.8.2
|
|
49
52
|
```
|
|
50
53
|
|
|
51
54
|
## Installation
|
|
@@ -231,7 +234,7 @@ CLOUDOS="https://cloudos.lifebit.ai"
|
|
|
231
234
|
WORKSPACE_ID="xxxxx"
|
|
232
235
|
PROJECT_NAME="API jobs"
|
|
233
236
|
WORKFLOW_NAME="rnatoy"
|
|
234
|
-
JOB_PARAMS="
|
|
237
|
+
JOB_PARAMS="cloudos_cli/examples/rnatoy.config"
|
|
235
238
|
```
|
|
236
239
|
|
|
237
240
|
As you can see, a file with the job parameters is used to configure the
|
|
@@ -23,7 +23,7 @@ setuptools.setup(
|
|
|
23
23
|
entry_points={"console_scripts": [
|
|
24
24
|
"cloudos=cloudos_cli.__main__:run_cloudos_cli"
|
|
25
25
|
]},
|
|
26
|
-
install_requires=["click>=8.0.1", "pandas>=1.3.4", "numpy==1.26.4", "requests>=2.26.0"],
|
|
26
|
+
install_requires=["click>=8.0.1", "rich-click>=1.8.2", "pandas>=1.3.4", "numpy==1.26.4", "requests>=2.26.0"],
|
|
27
27
|
extras_require={
|
|
28
28
|
"test": ["pytest", "mock", "responses", "requests_mock"]
|
|
29
29
|
},
|
|
@@ -6,11 +6,14 @@ import responses
|
|
|
6
6
|
from responses import matchers
|
|
7
7
|
from cloudos_cli.clos import Cloudos
|
|
8
8
|
from cloudos_cli.utils.errors import BadRequestException
|
|
9
|
+
from tests.functions_for_pytest import load_json_file
|
|
9
10
|
|
|
11
|
+
INPUT = "tests/test_data/projects.json"
|
|
10
12
|
APIKEY = 'vnoiweur89u2ongs'
|
|
11
13
|
CLOUDOS_URL = 'http://cloudos.lifebit.ai'
|
|
12
14
|
WORKSPACE_ID = 'lv89ufc838sdig'
|
|
13
|
-
|
|
15
|
+
PAGE_SIZE = 10
|
|
16
|
+
PAGE = 1
|
|
14
17
|
|
|
15
18
|
@mock.patch('cloudos_cli.clos', mock.MagicMock())
|
|
16
19
|
@responses.activate
|
|
@@ -18,27 +21,29 @@ def test_get_project_list_correct_response():
|
|
|
18
21
|
"""
|
|
19
22
|
Test 'get_project_list' to work as intended
|
|
20
23
|
"""
|
|
21
|
-
|
|
24
|
+
create_json = load_json_file(INPUT)
|
|
25
|
+
params = {"teamId": WORKSPACE_ID, "pageSize": PAGE_SIZE, "page": PAGE}
|
|
22
26
|
header = {
|
|
23
27
|
"Accept": "application/json, text/plain, */*",
|
|
24
28
|
"Content-Type": "application/json;charset=UTF-8",
|
|
25
29
|
"apikey": APIKEY
|
|
26
30
|
}
|
|
27
|
-
search_str = f"teamId={WORKSPACE_ID}"
|
|
31
|
+
search_str = f"teamId={WORKSPACE_ID}&pageSize={PAGE_SIZE}&page={PAGE}"
|
|
28
32
|
# mock GET method with the .json
|
|
29
33
|
responses.add(
|
|
30
34
|
responses.GET,
|
|
31
|
-
url=f"{CLOUDOS_URL}/api/
|
|
35
|
+
url=f"{CLOUDOS_URL}/api/v2/projects?{search_str}",
|
|
36
|
+
body=create_json,
|
|
32
37
|
headers=header,
|
|
33
38
|
match=[matchers.query_param_matcher(params)],
|
|
34
39
|
status=200)
|
|
35
40
|
# start cloudOS service
|
|
36
41
|
clos = Cloudos(apikey=APIKEY, cromwell_token=None, cloudos_url=CLOUDOS_URL)
|
|
37
42
|
# get mock response
|
|
38
|
-
response = clos.get_project_list(WORKSPACE_ID)
|
|
43
|
+
response = clos.get_project_list(WORKSPACE_ID, page_size=PAGE_SIZE, page=PAGE)
|
|
39
44
|
# check the response
|
|
40
|
-
assert response
|
|
41
|
-
assert
|
|
45
|
+
assert isinstance(response, list)
|
|
46
|
+
assert len(response) == 1
|
|
42
47
|
|
|
43
48
|
|
|
44
49
|
@mock.patch('cloudos_cli.clos', mock.MagicMock())
|
|
@@ -51,17 +56,17 @@ def test_get_project_list_incorrect_response():
|
|
|
51
56
|
error_message = {"statusCode": 400, "code": "BadRequest",
|
|
52
57
|
"message": "Bad Request.", "time": "2022-11-23_17:31:07"}
|
|
53
58
|
error_json = json.dumps(error_message)
|
|
54
|
-
params = {"teamId": WORKSPACE_ID}
|
|
59
|
+
params = {"teamId": WORKSPACE_ID, "pageSize": PAGE_SIZE, "page": PAGE}
|
|
55
60
|
header = {
|
|
56
61
|
"Accept": "application/json, text/plain, */*",
|
|
57
62
|
"Content-Type": "application/json;charset=UTF-8",
|
|
58
63
|
"apikey": APIKEY
|
|
59
64
|
}
|
|
60
|
-
search_str = f"teamId={WORKSPACE_ID}"
|
|
65
|
+
search_str = f"teamId={WORKSPACE_ID}&pageSize={PAGE_SIZE}&page={PAGE}"
|
|
61
66
|
# mock GET method with the .json
|
|
62
67
|
responses.add(
|
|
63
68
|
responses.GET,
|
|
64
|
-
url=f"{CLOUDOS_URL}/api/
|
|
69
|
+
url=f"{CLOUDOS_URL}/api/v2/projects?{search_str}",
|
|
65
70
|
body=error_json,
|
|
66
71
|
headers=header,
|
|
67
72
|
match=[matchers.query_param_matcher(params)],
|
|
@@ -18,10 +18,10 @@ def fixture_mocked_requests_get():
|
|
|
18
18
|
with open(INPUT_JSON, encoding="utf-8") as json_data:
|
|
19
19
|
data_d = json.load(json_data)
|
|
20
20
|
with requests_mock.Mocker() as mock:
|
|
21
|
-
mock.get(f"http://test_cloud_os/api/
|
|
21
|
+
mock.get(f"http://test_cloud_os/api/v2/projects?teamId={test_workspace_id}",
|
|
22
22
|
json=data_d)
|
|
23
|
-
r_get = requests.get(f"http://test_cloud_os/api/
|
|
24
|
-
return r_get
|
|
23
|
+
r_get = requests.get(f"http://test_cloud_os/api/v2/projects?teamId={test_workspace_id}")
|
|
24
|
+
return json.loads(r_get.content)['projects']
|
|
25
25
|
|
|
26
26
|
|
|
27
27
|
def test_process_project_list_all_fields_false(mocked_requests_get):
|
|
@@ -26,7 +26,7 @@ def test_project_id():
|
|
|
26
26
|
"""
|
|
27
27
|
create_json_project = load_json_file(INPUT_PROJECT)
|
|
28
28
|
create_json_workflow = load_json_file(INPUT_WORKFLOW)
|
|
29
|
-
params_projects = {"teamId": WORKSPACE_ID}
|
|
29
|
+
params_projects = {"teamId": WORKSPACE_ID, "pageSize": PAGE_SIZE, "page": PAGE}
|
|
30
30
|
params_workflows = {
|
|
31
31
|
"teamId": WORKSPACE_ID,
|
|
32
32
|
"pageSize": PAGE_SIZE,
|
|
@@ -37,12 +37,12 @@ def test_project_id():
|
|
|
37
37
|
"Content-Type": "application/json;charset=UTF-8",
|
|
38
38
|
"apikey": APIKEY
|
|
39
39
|
}
|
|
40
|
-
search_str_projects = f"teamId={WORKSPACE_ID}"
|
|
40
|
+
search_str_projects = f"teamId={WORKSPACE_ID}&pageSize={PAGE_SIZE}&page={PAGE}"
|
|
41
41
|
search_str_workflows = f"teamId={WORKSPACE_ID}&pageSize={PAGE_SIZE}&page={PAGE}&archived.status={ARCHIVED_STATUS}"
|
|
42
42
|
# mock GET method with the .json
|
|
43
43
|
responses.add(
|
|
44
44
|
responses.GET,
|
|
45
|
-
url=f"{CLOUDOS_URL}/api/
|
|
45
|
+
url=f"{CLOUDOS_URL}/api/v2/projects?{search_str_projects}",
|
|
46
46
|
body=create_json_project,
|
|
47
47
|
headers=header,
|
|
48
48
|
match=[matchers.query_param_matcher(params_projects)],
|
|
@@ -35,6 +35,7 @@ def test_send_job():
|
|
|
35
35
|
create_json_workflow = load_json_file(INPUT_WORKFLOW)
|
|
36
36
|
create_json = load_json_file(INPUT)
|
|
37
37
|
params_job = {"teamId": WORKSPACE_ID}
|
|
38
|
+
params_projects = {"teamId": WORKSPACE_ID, "pageSize": PAGE_SIZE, "page": PAGE}
|
|
38
39
|
params_workflows = {
|
|
39
40
|
"teamId": WORKSPACE_ID,
|
|
40
41
|
"pageSize": PAGE_SIZE,
|
|
@@ -45,6 +46,7 @@ def test_send_job():
|
|
|
45
46
|
"apikey": APIKEY
|
|
46
47
|
}
|
|
47
48
|
search_str = f"teamId={WORKSPACE_ID}"
|
|
49
|
+
search_str_projects = f"teamId={WORKSPACE_ID}&pageSize={PAGE_SIZE}&page={PAGE}"
|
|
48
50
|
search_str_workflows = f"teamId={WORKSPACE_ID}&pageSize={PAGE_SIZE}&page={PAGE}&archived.status={ARCHIVED_STATUS}"
|
|
49
51
|
# mock GET method with the .json
|
|
50
52
|
responses.add(
|
|
@@ -56,10 +58,10 @@ def test_send_job():
|
|
|
56
58
|
status=200)
|
|
57
59
|
responses.add(
|
|
58
60
|
responses.GET,
|
|
59
|
-
url=f"{CLOUDOS_URL}/api/
|
|
61
|
+
url=f"{CLOUDOS_URL}/api/v2/projects?{search_str_projects}",
|
|
60
62
|
body=create_json_project,
|
|
61
63
|
headers=header,
|
|
62
|
-
match=[matchers.query_param_matcher(
|
|
64
|
+
match=[matchers.query_param_matcher(params_projects)],
|
|
63
65
|
status=200)
|
|
64
66
|
responses.add(
|
|
65
67
|
responses.GET,
|
|
@@ -26,7 +26,7 @@ def test_workflow_id():
|
|
|
26
26
|
"""
|
|
27
27
|
create_json_project = load_json_file(INPUT_PROJECT)
|
|
28
28
|
create_json_workflow = load_json_file(INPUT_WORKFLOW)
|
|
29
|
-
params = {"teamId": WORKSPACE_ID}
|
|
29
|
+
params = {"teamId": WORKSPACE_ID, "pageSize": PAGE_SIZE, "page": PAGE}
|
|
30
30
|
params_workflows = {
|
|
31
31
|
"teamId": WORKSPACE_ID,
|
|
32
32
|
"pageSize": PAGE_SIZE,
|
|
@@ -37,12 +37,12 @@ def test_workflow_id():
|
|
|
37
37
|
"Content-Type": "application/json;charset=UTF-8",
|
|
38
38
|
"apikey": APIKEY
|
|
39
39
|
}
|
|
40
|
-
search_str = f"teamId={WORKSPACE_ID}"
|
|
40
|
+
search_str = f"teamId={WORKSPACE_ID}&pageSize={PAGE_SIZE}&page={PAGE}"
|
|
41
41
|
search_str_workflows = f"teamId={WORKSPACE_ID}&pageSize={PAGE_SIZE}&page={PAGE}&archived.status={ARCHIVED_STATUS}"
|
|
42
42
|
# mock GET method with the .json
|
|
43
43
|
responses.add(
|
|
44
44
|
responses.GET,
|
|
45
|
-
url=f"{CLOUDOS_URL}/api/
|
|
45
|
+
url=f"{CLOUDOS_URL}/api/v2/projects?{search_str}",
|
|
46
46
|
body=create_json_project,
|
|
47
47
|
headers=header,
|
|
48
48
|
match=[matchers.query_param_matcher(params)],
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = '2.17.0'
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|