cloudos-cli 2.47.1__tar.gz → 2.49.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/PKG-INFO +26 -8
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/README.md +25 -7
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli/__main__.py +100 -11
- cloudos_cli-2.49.0/cloudos_cli/_version.py +1 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli/clos.py +214 -17
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli.egg-info/PKG-INFO +26 -8
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli.egg-info/SOURCES.txt +2 -1
- cloudos_cli-2.49.0/tests/test_cli_project_create.py +53 -0
- cloudos_cli-2.47.1/cloudos_cli/_version.py +0 -1
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/LICENSE +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli/__init__.py +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli/configure/__init__.py +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli/configure/configure.py +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli/datasets/__init__.py +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli/datasets/datasets.py +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli/import_wf/__init__.py +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli/import_wf/import_wf.py +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli/jobs/__init__.py +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli/jobs/job.py +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli/link/__init__.py +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli/link/link.py +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli/procurement/__init__.py +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli/procurement/images.py +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli/queue/__init__.py +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli/queue/queue.py +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli/utils/__init__.py +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli/utils/array_job.py +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli/utils/cloud.py +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli/utils/details.py +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli/utils/errors.py +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli/utils/last_wf.py +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli/utils/requests.py +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli/utils/resources.py +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli.egg-info/dependency_links.txt +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli.egg-info/entry_points.txt +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli.egg-info/requires.txt +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/cloudos_cli.egg-info/top_level.txt +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/setup.cfg +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/setup.py +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/tests/__init__.py +0 -0
- {cloudos_cli-2.47.1 → cloudos_cli-2.49.0}/tests/functions_for_pytest.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cloudos_cli
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.49.0
|
|
4
4
|
Summary: Python package for interacting with CloudOS
|
|
5
5
|
Home-page: https://github.com/lifebit-ai/cloudos-cli
|
|
6
6
|
Author: David Piñeyro
|
|
@@ -180,7 +180,7 @@ cloudos --help
|
|
|
180
180
|
│ configure CloudOS configuration. │
|
|
181
181
|
│ cromwell Cromwell server functionality: check status, start and stop. │
|
|
182
182
|
│ job CloudOS job functionality: run, check and abort jobs in CloudOS. │
|
|
183
|
-
│ project CloudOS project functionality: list projects in CloudOS.
|
|
183
|
+
│ project CloudOS project functionality: list and create projects in CloudOS. │
|
|
184
184
|
│ queue CloudOS job queue functionality. │
|
|
185
185
|
│ workflow CloudOS workflow functionality: list and import workflows. │
|
|
186
186
|
╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
@@ -775,16 +775,15 @@ This file can later be used when running a job with `cloudos job run --job-confi
|
|
|
775
775
|
> [!NOTE]
|
|
776
776
|
> Job details can only be retrieved for a single user, cannot see other user's job details.
|
|
777
777
|
|
|
778
|
-
#### Get a list of
|
|
778
|
+
#### Get a list of workspace jobs from a CloudOS
|
|
779
779
|
|
|
780
|
-
You can get a summary of
|
|
781
|
-
parameter) in two different formats:
|
|
780
|
+
You can get a summary of the workspace's last 30 submitted jobs (or a selected number of last jobs using `--last-n-jobs n` parameter) in two different formats:
|
|
782
781
|
|
|
783
782
|
- CSV: this is a table with a minimum predefined set of columns by default, or all the
|
|
784
783
|
available columns using the `--all-fields` argument.
|
|
785
|
-
- JSON: all the available information from
|
|
784
|
+
- JSON: all the available information from the workspace jobs, in JSON format (`--all-fields` is always enabled for this format).
|
|
786
785
|
|
|
787
|
-
To get a list with
|
|
786
|
+
To get a list with the workspace's last 30 submitted jobs, in CSV format, use
|
|
788
787
|
the following command:
|
|
789
788
|
|
|
790
789
|
```bash
|
|
@@ -806,7 +805,7 @@ Executing list...
|
|
|
806
805
|
|
|
807
806
|
In addition, a file named `joblist.csv` is created.
|
|
808
807
|
|
|
809
|
-
To get the same information, but for all
|
|
808
|
+
To get the same information, but for all the workspace's jobs and in JSON format, use the following command:
|
|
810
809
|
|
|
811
810
|
```bash
|
|
812
811
|
cloudos job list \
|
|
@@ -947,6 +946,25 @@ Executing list...
|
|
|
947
946
|
Workflow list saved to project_list.csv
|
|
948
947
|
```
|
|
949
948
|
|
|
949
|
+
#### Create a new project in CloudOS
|
|
950
|
+
|
|
951
|
+
You can create a new project in your CloudOS workspace using the `project create` command.
|
|
952
|
+
This command requires the name of the new project and will return the project ID upon successful creation.
|
|
953
|
+
|
|
954
|
+
```bash
|
|
955
|
+
cloudos project create \
|
|
956
|
+
--cloudos-url $CLOUDOS \
|
|
957
|
+
--apikey $MY_API_KEY \
|
|
958
|
+
--workspace-id $WORKSPACE_ID \
|
|
959
|
+
--new-project "My New Project"
|
|
960
|
+
```
|
|
961
|
+
|
|
962
|
+
The expected output is something similar to:
|
|
963
|
+
|
|
964
|
+
```console
|
|
965
|
+
Project "My New Project" created successfully with ID: 64f1a23b8e4c9d001234abcd
|
|
966
|
+
```
|
|
967
|
+
|
|
950
968
|
#### Get a list of the available job queues
|
|
951
969
|
|
|
952
970
|
Job queues are required for running jobs using AWS batch executor. The available job queues in your CloudOS workspace are
|
|
@@ -145,7 +145,7 @@ cloudos --help
|
|
|
145
145
|
│ configure CloudOS configuration. │
|
|
146
146
|
│ cromwell Cromwell server functionality: check status, start and stop. │
|
|
147
147
|
│ job CloudOS job functionality: run, check and abort jobs in CloudOS. │
|
|
148
|
-
│ project CloudOS project functionality: list projects in CloudOS.
|
|
148
|
+
│ project CloudOS project functionality: list and create projects in CloudOS. │
|
|
149
149
|
│ queue CloudOS job queue functionality. │
|
|
150
150
|
│ workflow CloudOS workflow functionality: list and import workflows. │
|
|
151
151
|
╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
@@ -740,16 +740,15 @@ This file can later be used when running a job with `cloudos job run --job-confi
|
|
|
740
740
|
> [!NOTE]
|
|
741
741
|
> Job details can only be retrieved for a single user, cannot see other user's job details.
|
|
742
742
|
|
|
743
|
-
#### Get a list of
|
|
743
|
+
#### Get a list of workspace jobs from a CloudOS
|
|
744
744
|
|
|
745
|
-
You can get a summary of
|
|
746
|
-
parameter) in two different formats:
|
|
745
|
+
You can get a summary of the workspace's last 30 submitted jobs (or a selected number of last jobs using `--last-n-jobs n` parameter) in two different formats:
|
|
747
746
|
|
|
748
747
|
- CSV: this is a table with a minimum predefined set of columns by default, or all the
|
|
749
748
|
available columns using the `--all-fields` argument.
|
|
750
|
-
- JSON: all the available information from
|
|
749
|
+
- JSON: all the available information from the workspace jobs, in JSON format (`--all-fields` is always enabled for this format).
|
|
751
750
|
|
|
752
|
-
To get a list with
|
|
751
|
+
To get a list with the workspace's last 30 submitted jobs, in CSV format, use
|
|
753
752
|
the following command:
|
|
754
753
|
|
|
755
754
|
```bash
|
|
@@ -771,7 +770,7 @@ Executing list...
|
|
|
771
770
|
|
|
772
771
|
In addition, a file named `joblist.csv` is created.
|
|
773
772
|
|
|
774
|
-
To get the same information, but for all
|
|
773
|
+
To get the same information, but for all the workspace's jobs and in JSON format, use the following command:
|
|
775
774
|
|
|
776
775
|
```bash
|
|
777
776
|
cloudos job list \
|
|
@@ -912,6 +911,25 @@ Executing list...
|
|
|
912
911
|
Workflow list saved to project_list.csv
|
|
913
912
|
```
|
|
914
913
|
|
|
914
|
+
#### Create a new project in CloudOS
|
|
915
|
+
|
|
916
|
+
You can create a new project in your CloudOS workspace using the `project create` command.
|
|
917
|
+
This command requires the name of the new project and will return the project ID upon successful creation.
|
|
918
|
+
|
|
919
|
+
```bash
|
|
920
|
+
cloudos project create \
|
|
921
|
+
--cloudos-url $CLOUDOS \
|
|
922
|
+
--apikey $MY_API_KEY \
|
|
923
|
+
--workspace-id $WORKSPACE_ID \
|
|
924
|
+
--new-project "My New Project"
|
|
925
|
+
```
|
|
926
|
+
|
|
927
|
+
The expected output is something similar to:
|
|
928
|
+
|
|
929
|
+
```console
|
|
930
|
+
Project "My New Project" created successfully with ID: 64f1a23b8e4c9d001234abcd
|
|
931
|
+
```
|
|
932
|
+
|
|
915
933
|
#### Get a list of the available job queues
|
|
916
934
|
|
|
917
935
|
Job queues are required for running jobs using AWS batch executor. The available job queues in your CloudOS workspace are
|
|
@@ -79,7 +79,8 @@ def run_cloudos_cli(ctx):
|
|
|
79
79
|
'import': shared_config
|
|
80
80
|
},
|
|
81
81
|
'project': {
|
|
82
|
-
'list': shared_config
|
|
82
|
+
'list': shared_config,
|
|
83
|
+
'create': shared_config
|
|
83
84
|
},
|
|
84
85
|
'cromwell': {
|
|
85
86
|
'status': shared_config,
|
|
@@ -139,7 +140,8 @@ def run_cloudos_cli(ctx):
|
|
|
139
140
|
'import': shared_config
|
|
140
141
|
},
|
|
141
142
|
'project': {
|
|
142
|
-
'list': shared_config
|
|
143
|
+
'list': shared_config,
|
|
144
|
+
'create': shared_config
|
|
143
145
|
},
|
|
144
146
|
'cromwell': {
|
|
145
147
|
'status': shared_config,
|
|
@@ -186,7 +188,7 @@ def workflow():
|
|
|
186
188
|
|
|
187
189
|
@run_cloudos_cli.group()
|
|
188
190
|
def project():
|
|
189
|
-
"""CloudOS project functionality: list projects in CloudOS."""
|
|
191
|
+
"""CloudOS project functionality: list and create projects in CloudOS."""
|
|
190
192
|
print(project.__doc__ + '\n')
|
|
191
193
|
|
|
192
194
|
|
|
@@ -1174,17 +1176,17 @@ def job_details(ctx,
|
|
|
1174
1176
|
default='joblist',
|
|
1175
1177
|
required=False)
|
|
1176
1178
|
@click.option('--output-format',
|
|
1177
|
-
help='The desired file format (file extension) for the output. Default=csv.',
|
|
1179
|
+
help='The desired file format (file extension) for the output. For json option --all-fields will be automatically set to True. Default=csv.',
|
|
1178
1180
|
type=click.Choice(['csv', 'json'], case_sensitive=False),
|
|
1179
1181
|
default='csv')
|
|
1180
1182
|
@click.option('--all-fields',
|
|
1181
1183
|
help=('Whether to collect all available fields from jobs or ' +
|
|
1182
1184
|
'just the preconfigured selected fields. Only applicable ' +
|
|
1183
|
-
'when --output-format=csv'),
|
|
1185
|
+
'when --output-format=csv. Automatically enabled for json output.'),
|
|
1184
1186
|
is_flag=True)
|
|
1185
1187
|
@click.option('--last-n-jobs',
|
|
1186
|
-
help=("The number of last
|
|
1187
|
-
"retrieve all
|
|
1188
|
+
help=("The number of last workspace jobs to retrieve. You can use 'all' to " +
|
|
1189
|
+
"retrieve all workspace jobs. Default=30."),
|
|
1188
1190
|
default='30')
|
|
1189
1191
|
@click.option('--page',
|
|
1190
1192
|
help=('Response page to retrieve. If --last-n-jobs is set, then --page ' +
|
|
@@ -1219,7 +1221,7 @@ def list_jobs(ctx,
|
|
|
1219
1221
|
disable_ssl_verification,
|
|
1220
1222
|
ssl_cert,
|
|
1221
1223
|
profile):
|
|
1222
|
-
"""Collect
|
|
1224
|
+
"""Collect workspace jobs from a CloudOS workspace in CSV or JSON format."""
|
|
1223
1225
|
profile = profile or ctx.default_map['job']['list']['profile']
|
|
1224
1226
|
# Create a dictionary with required and non-required params
|
|
1225
1227
|
required_dict = {
|
|
@@ -1268,6 +1270,7 @@ def list_jobs(ctx,
|
|
|
1268
1270
|
except ValueError:
|
|
1269
1271
|
print("[ERROR] last-n-jobs value was not valid. Please use a positive int or 'all'")
|
|
1270
1272
|
raise
|
|
1273
|
+
|
|
1271
1274
|
my_jobs_r = cl.get_job_list(workspace_id, last_n_jobs, page, archived, verify_ssl)
|
|
1272
1275
|
if len(my_jobs_r) == 0:
|
|
1273
1276
|
if ctx.get_parameter_source('page') == click.core.ParameterSource.DEFAULT:
|
|
@@ -1279,15 +1282,14 @@ def list_jobs(ctx,
|
|
|
1279
1282
|
'using --page parameter.')
|
|
1280
1283
|
elif output_format == 'csv':
|
|
1281
1284
|
my_jobs = cl.process_job_list(my_jobs_r, all_fields)
|
|
1282
|
-
|
|
1283
|
-
print(f'\tJob list collected with a total of {my_jobs.shape[0]} jobs.')
|
|
1285
|
+
cl.save_job_list_to_csv(my_jobs, outfile)
|
|
1284
1286
|
elif output_format == 'json':
|
|
1285
1287
|
with open(outfile, 'w') as o:
|
|
1286
1288
|
o.write(json.dumps(my_jobs_r))
|
|
1287
1289
|
print(f'\tJob list collected with a total of {len(my_jobs_r)} jobs.')
|
|
1290
|
+
print(f'\tJob list saved to {outfile}')
|
|
1288
1291
|
else:
|
|
1289
1292
|
raise ValueError('Unrecognised output format. Please use one of [csv|json]')
|
|
1290
|
-
print(f'\tJob list saved to {outfile}')
|
|
1291
1293
|
|
|
1292
1294
|
|
|
1293
1295
|
@job.command('abort')
|
|
@@ -1701,6 +1703,93 @@ def list_projects(ctx,
|
|
|
1701
1703
|
print(f'\tProject list saved to {outfile}')
|
|
1702
1704
|
|
|
1703
1705
|
|
|
1706
|
+
@project.command('create')
|
|
1707
|
+
@click.option('-k',
|
|
1708
|
+
'--apikey',
|
|
1709
|
+
help='Your CloudOS API key',
|
|
1710
|
+
required=True)
|
|
1711
|
+
@click.option('-c',
|
|
1712
|
+
'--cloudos-url',
|
|
1713
|
+
help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'),
|
|
1714
|
+
default=CLOUDOS_URL,
|
|
1715
|
+
required=True)
|
|
1716
|
+
@click.option('--workspace-id',
|
|
1717
|
+
help='The specific CloudOS workspace id.',
|
|
1718
|
+
required=True)
|
|
1719
|
+
@click.option('--new-project',
|
|
1720
|
+
help='The name for the new project.',
|
|
1721
|
+
required=True)
|
|
1722
|
+
@click.option('--verbose',
|
|
1723
|
+
help='Whether to print information messages or not.',
|
|
1724
|
+
is_flag=True)
|
|
1725
|
+
@click.option('--disable-ssl-verification',
|
|
1726
|
+
help=('Disable SSL certificate verification. Please, remember that this option is ' +
|
|
1727
|
+
'not generally recommended for security reasons.'),
|
|
1728
|
+
is_flag=True)
|
|
1729
|
+
@click.option('--ssl-cert',
|
|
1730
|
+
help='Path to your SSL certificate file.')
|
|
1731
|
+
@click.option('--profile', help='Profile to use from the config file', default=None)
|
|
1732
|
+
@click.pass_context
|
|
1733
|
+
def create_project(ctx,
|
|
1734
|
+
apikey,
|
|
1735
|
+
cloudos_url,
|
|
1736
|
+
workspace_id,
|
|
1737
|
+
new_project,
|
|
1738
|
+
verbose,
|
|
1739
|
+
disable_ssl_verification,
|
|
1740
|
+
ssl_cert,
|
|
1741
|
+
profile):
|
|
1742
|
+
"""Create a new project in CloudOS."""
|
|
1743
|
+
profile = profile or ctx.default_map['project']['create']['profile']
|
|
1744
|
+
# Create a dictionary with required and non-required params
|
|
1745
|
+
required_dict = {
|
|
1746
|
+
'apikey': True,
|
|
1747
|
+
'workspace_id': True,
|
|
1748
|
+
'workflow_name': False,
|
|
1749
|
+
'project_name': False,
|
|
1750
|
+
'procurement_id': False
|
|
1751
|
+
}
|
|
1752
|
+
# determine if the user provided all required parameters
|
|
1753
|
+
config_manager = ConfigurationProfile()
|
|
1754
|
+
user_options = (
|
|
1755
|
+
config_manager.load_profile_and_validate_data(
|
|
1756
|
+
ctx,
|
|
1757
|
+
INIT_PROFILE,
|
|
1758
|
+
CLOUDOS_URL,
|
|
1759
|
+
profile=profile,
|
|
1760
|
+
required_dict=required_dict,
|
|
1761
|
+
apikey=apikey,
|
|
1762
|
+
cloudos_url=cloudos_url,
|
|
1763
|
+
workspace_id=workspace_id,
|
|
1764
|
+
project_name=new_project
|
|
1765
|
+
)
|
|
1766
|
+
)
|
|
1767
|
+
# replace the profile parameters with arguments given by the user
|
|
1768
|
+
apikey = user_options['apikey']
|
|
1769
|
+
cloudos_url = user_options['cloudos_url']
|
|
1770
|
+
workspace_id = user_options['workspace_id']
|
|
1771
|
+
|
|
1772
|
+
# verify ssl configuration
|
|
1773
|
+
verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert)
|
|
1774
|
+
|
|
1775
|
+
# Print basic output
|
|
1776
|
+
if verbose:
|
|
1777
|
+
print(f'\tUsing CloudOS URL: {cloudos_url}')
|
|
1778
|
+
print(f'\tUsing workspace: {workspace_id}')
|
|
1779
|
+
print(f'\tProject name: {new_project}')
|
|
1780
|
+
|
|
1781
|
+
cl = Cloudos(cloudos_url=cloudos_url, apikey=apikey, cromwell_token=None)
|
|
1782
|
+
|
|
1783
|
+
try:
|
|
1784
|
+
project_id = cl.create_project(workspace_id, new_project, verify_ssl)
|
|
1785
|
+
print(f'\tProject "{new_project}" created successfully with ID: {project_id}')
|
|
1786
|
+
if verbose:
|
|
1787
|
+
print(f'\tProject URL: {cloudos_url}/app/projects/{project_id}')
|
|
1788
|
+
except Exception as e:
|
|
1789
|
+
print(f'\tError creating project: {str(e)}')
|
|
1790
|
+
sys.exit(1)
|
|
1791
|
+
|
|
1792
|
+
|
|
1704
1793
|
@cromwell.command('status')
|
|
1705
1794
|
@click.version_option()
|
|
1706
1795
|
@click.option('-k',
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = '2.49.0'
|
|
@@ -11,6 +11,8 @@ from cloudos_cli.utils.errors import BadRequestException, JoBNotCompletedExcepti
|
|
|
11
11
|
from cloudos_cli.utils.requests import retry_requests_get, retry_requests_post, retry_requests_put
|
|
12
12
|
import pandas as pd
|
|
13
13
|
from cloudos_cli.utils.last_wf import youngest_workflow_id_by_name
|
|
14
|
+
from datetime import datetime
|
|
15
|
+
|
|
14
16
|
|
|
15
17
|
# GLOBAL VARS
|
|
16
18
|
JOB_COMPLETED = 'completed'
|
|
@@ -430,28 +432,27 @@ class Cloudos:
|
|
|
430
432
|
df : pandas.DataFrame
|
|
431
433
|
A DataFrame with the requested columns from the jobs.
|
|
432
434
|
"""
|
|
433
|
-
COLUMNS = ['
|
|
434
|
-
'team',
|
|
435
|
+
COLUMNS = ['status',
|
|
435
436
|
'name',
|
|
436
|
-
'
|
|
437
|
-
'
|
|
437
|
+
'project.name',
|
|
438
|
+
'user.name',
|
|
439
|
+
'user.surname',
|
|
440
|
+
'workflow.name',
|
|
441
|
+
'_id',
|
|
438
442
|
'startTime',
|
|
439
443
|
'endTime',
|
|
440
444
|
'createdAt',
|
|
441
445
|
'updatedAt',
|
|
442
|
-
'
|
|
443
|
-
'
|
|
444
|
-
'
|
|
445
|
-
'
|
|
446
|
-
'workflow.
|
|
447
|
-
'
|
|
448
|
-
'
|
|
449
|
-
'
|
|
450
|
-
'
|
|
451
|
-
'
|
|
452
|
-
'project.name',
|
|
453
|
-
'project.createdAt',
|
|
454
|
-
'project.updatedAt'
|
|
446
|
+
'revision.commit',
|
|
447
|
+
'realInstancesExecutionCost',
|
|
448
|
+
'masterInstance.usedInstance.type',
|
|
449
|
+
'storageMode',
|
|
450
|
+
'workflow.repository.url',
|
|
451
|
+
'nextflowVersion',
|
|
452
|
+
'batch.enabled',
|
|
453
|
+
'storageSizeInGb',
|
|
454
|
+
'batch.jobQueue.id',
|
|
455
|
+
'usesFusionFileSystem'
|
|
455
456
|
]
|
|
456
457
|
df_full = pd.json_normalize(r)
|
|
457
458
|
if df_full.empty:
|
|
@@ -462,6 +463,156 @@ class Cloudos:
|
|
|
462
463
|
df = df_full.loc[:, COLUMNS]
|
|
463
464
|
return df
|
|
464
465
|
|
|
466
|
+
def reorder_job_list(self, my_jobs_df, filename='my_jobs.csv'):
|
|
467
|
+
"""Save a job list DataFrame to a CSV file with renamed and ordered columns.
|
|
468
|
+
|
|
469
|
+
Parameters
|
|
470
|
+
----------
|
|
471
|
+
my_jobs_df : pandas.DataFrame
|
|
472
|
+
A DataFrame containing job information from process_job_list.
|
|
473
|
+
filename : str
|
|
474
|
+
The name of the file to save the DataFrame to. Default is 'my_jobs.csv'.
|
|
475
|
+
|
|
476
|
+
Returns
|
|
477
|
+
-------
|
|
478
|
+
None
|
|
479
|
+
Saves the DataFrame to a CSV file with renamed and ordered columns.
|
|
480
|
+
"""
|
|
481
|
+
# Handle empty DataFrame
|
|
482
|
+
if my_jobs_df.empty:
|
|
483
|
+
print("Warning: DataFrame is empty. Creating empty CSV file.")
|
|
484
|
+
empty_df = pd.DataFrame()
|
|
485
|
+
empty_df.to_csv(filename, index=False)
|
|
486
|
+
return
|
|
487
|
+
|
|
488
|
+
# Create a copy to avoid modifying the original DataFrame
|
|
489
|
+
jobs_df = my_jobs_df.copy()
|
|
490
|
+
|
|
491
|
+
# 1. Fusion user.name and user.surname into user
|
|
492
|
+
if 'user.name' in jobs_df.columns and 'user.surname' in jobs_df.columns:
|
|
493
|
+
jobs_df['user'] = jobs_df.apply(
|
|
494
|
+
lambda row: f"{row.get('user.name', '')} {row.get('user.surname', '')}".strip()
|
|
495
|
+
if pd.notna(row.get('user.name')) or pd.notna(row.get('user.surname'))
|
|
496
|
+
else None, axis=1
|
|
497
|
+
)
|
|
498
|
+
# Remove original columns
|
|
499
|
+
jobs_df = jobs_df.drop(columns=['user.name', 'user.surname'], errors='ignore')
|
|
500
|
+
|
|
501
|
+
# 2. Convert time fields to human-readable format
|
|
502
|
+
time_columns = ['startTime', 'endTime', 'createdAt', 'updatedAt']
|
|
503
|
+
for col in time_columns:
|
|
504
|
+
if col in jobs_df.columns:
|
|
505
|
+
def format_time(x):
|
|
506
|
+
if pd.notna(x) and isinstance(x, str) and x:
|
|
507
|
+
try:
|
|
508
|
+
return datetime.fromisoformat(x.replace('Z', '+00:00')).strftime('%Y-%m-%d %H:%M:%S UTC')
|
|
509
|
+
except (ValueError, TypeError):
|
|
510
|
+
return x # Return original value if parsing fails
|
|
511
|
+
return None
|
|
512
|
+
jobs_df[col] = jobs_df[col].apply(format_time)
|
|
513
|
+
|
|
514
|
+
# 3. Format realInstancesExecutionCost (divide by 100, show 4 decimals)
|
|
515
|
+
if 'realInstancesExecutionCost' in jobs_df.columns:
|
|
516
|
+
def format_cost(x):
|
|
517
|
+
if pd.notna(x) and x != '' and x is not None:
|
|
518
|
+
try:
|
|
519
|
+
return f"{float(x) / 100:.4f}"
|
|
520
|
+
except (ValueError, TypeError):
|
|
521
|
+
return x # Return original value if conversion fails
|
|
522
|
+
return None
|
|
523
|
+
jobs_df['realInstancesExecutionCost'] = jobs_df['realInstancesExecutionCost'].apply(format_cost)
|
|
524
|
+
|
|
525
|
+
# 4. Calculate Run time (endTime - startTime)
|
|
526
|
+
if 'startTime' in jobs_df.columns and 'endTime' in jobs_df.columns:
|
|
527
|
+
def calculate_runtime(row):
|
|
528
|
+
start_time = row.get('startTime')
|
|
529
|
+
end_time = row.get('endTime')
|
|
530
|
+
if pd.notna(start_time) and pd.notna(end_time) and start_time and end_time:
|
|
531
|
+
# Use original times from the original DataFrame for calculation
|
|
532
|
+
original_start = my_jobs_df.iloc[row.name].get('startTime') if row.name < len(my_jobs_df) else start_time
|
|
533
|
+
original_end = my_jobs_df.iloc[row.name].get('endTime') if row.name < len(my_jobs_df) else end_time
|
|
534
|
+
if pd.notna(original_start) and pd.notna(original_end) and original_start and original_end:
|
|
535
|
+
try:
|
|
536
|
+
start_dt = datetime.fromisoformat(str(original_start).replace('Z', '+00:00'))
|
|
537
|
+
end_dt = datetime.fromisoformat(str(original_end).replace('Z', '+00:00'))
|
|
538
|
+
duration = end_dt - start_dt
|
|
539
|
+
# Format duration as hours:minutes:seconds
|
|
540
|
+
total_seconds = int(duration.total_seconds())
|
|
541
|
+
hours = total_seconds // 3600
|
|
542
|
+
minutes = (total_seconds % 3600) // 60
|
|
543
|
+
seconds = total_seconds % 60
|
|
544
|
+
if hours > 0:
|
|
545
|
+
return f"{hours}h {minutes}m {seconds}s"
|
|
546
|
+
elif minutes > 0:
|
|
547
|
+
return f"{minutes}m {seconds}s"
|
|
548
|
+
else:
|
|
549
|
+
return f"{seconds}s"
|
|
550
|
+
except (ValueError, TypeError):
|
|
551
|
+
return None
|
|
552
|
+
return None
|
|
553
|
+
|
|
554
|
+
jobs_df['Run time'] = jobs_df.apply(calculate_runtime, axis=1)
|
|
555
|
+
|
|
556
|
+
# 5. Format batch.enabled (True -> "Batch", else "N/A")
|
|
557
|
+
if 'batch.enabled' in jobs_df.columns:
|
|
558
|
+
jobs_df['batch.enabled'] = jobs_df['batch.enabled'].apply(
|
|
559
|
+
lambda x: "Batch" if x is True else "N/A"
|
|
560
|
+
)
|
|
561
|
+
|
|
562
|
+
# 6. Rename columns using the provided dictionary
|
|
563
|
+
column_name_mapping = {
|
|
564
|
+
"status": "Status",
|
|
565
|
+
"name": "Name",
|
|
566
|
+
"project.name": "Project",
|
|
567
|
+
"user": "Owner",
|
|
568
|
+
"workflow.name": "Pipeline",
|
|
569
|
+
"_id": "ID",
|
|
570
|
+
"createdAt": "Submit time",
|
|
571
|
+
"updatedAt": "End time",
|
|
572
|
+
"revision.commit": "Commit",
|
|
573
|
+
"realInstancesExecutionCost": "Cost",
|
|
574
|
+
"masterInstance.usedInstance.type": "Resources",
|
|
575
|
+
"storageMode": "Storage type",
|
|
576
|
+
"workflow.repository.url": "Pipeline url",
|
|
577
|
+
"nextflowVersion": "Nextflow version",
|
|
578
|
+
"batch.enabled": "Executor",
|
|
579
|
+
"storageSizeInGb": "Storage size",
|
|
580
|
+
"batch.jobQueue.id": "Job queue ID",
|
|
581
|
+
"usesFusionFileSystem": "Accelerated file staging"
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
# Rename columns that exist in the DataFrame
|
|
585
|
+
jobs_df = jobs_df.rename(columns=column_name_mapping)
|
|
586
|
+
|
|
587
|
+
# Remove the original startTime and endTime columns since we now have Submit time, End time, and Run time
|
|
588
|
+
jobs_df = jobs_df.drop(columns=['startTime', 'endTime'], errors='ignore')
|
|
589
|
+
|
|
590
|
+
# 7. Define the desired order of columns
|
|
591
|
+
desired_order = [
|
|
592
|
+
"Status", "Name", "Project", "Owner", "Pipeline", "ID",
|
|
593
|
+
"Submit time", "End time", "Run time", "Commit", "Cost",
|
|
594
|
+
"Resources", "Storage type", "Pipeline url",
|
|
595
|
+
"Nextflow version", "Executor", "Storage size", "Job queue ID",
|
|
596
|
+
"Accelerated file staging"
|
|
597
|
+
]
|
|
598
|
+
|
|
599
|
+
# Reorder columns - only include columns that exist in the DataFrame
|
|
600
|
+
available_columns = [col for col in desired_order if col in jobs_df.columns]
|
|
601
|
+
# Add any remaining columns that aren't in the desired order
|
|
602
|
+
remaining_columns = [col for col in jobs_df.columns if col not in desired_order]
|
|
603
|
+
final_column_order = available_columns + remaining_columns
|
|
604
|
+
|
|
605
|
+
# Reorder the DataFrame
|
|
606
|
+
jobs_df = jobs_df[final_column_order]
|
|
607
|
+
return jobs_df
|
|
608
|
+
|
|
609
|
+
def save_job_list_to_csv(self, my_jobs_df, filename='my_jobs.csv'):
|
|
610
|
+
# Save to CSV
|
|
611
|
+
jobs_df = self.reorder_job_list(my_jobs_df, filename)
|
|
612
|
+
jobs_df.to_csv(filename, index=False)
|
|
613
|
+
print(f'\tJob list collected with a total of {len(jobs_df)} jobs.')
|
|
614
|
+
print(f'\tJob list saved to {filename}')
|
|
615
|
+
|
|
465
616
|
def get_workflow_list(self, workspace_id, verify=True, get_all=True,
|
|
466
617
|
page=1, page_size=10, max_page_size=100,
|
|
467
618
|
archived_status=False):
|
|
@@ -916,6 +1067,52 @@ class Cloudos:
|
|
|
916
1067
|
|
|
917
1068
|
return project_id
|
|
918
1069
|
|
|
1070
|
+
def create_project(self, workspace_id, project_name, verify=True):
|
|
1071
|
+
"""Create a new project in CloudOS.
|
|
1072
|
+
|
|
1073
|
+
Parameters
|
|
1074
|
+
----------
|
|
1075
|
+
workspace_id : str
|
|
1076
|
+
The CloudOS workspace ID where the project will be created.
|
|
1077
|
+
project_name : str
|
|
1078
|
+
The name for the new project.
|
|
1079
|
+
verify : [bool | str], optional
|
|
1080
|
+
Whether to use SSL verification or not. Alternatively, if
|
|
1081
|
+
a string is passed, it will be interpreted as the path to
|
|
1082
|
+
the SSL certificate file. Default is True.
|
|
1083
|
+
|
|
1084
|
+
Returns
|
|
1085
|
+
-------
|
|
1086
|
+
str
|
|
1087
|
+
The ID of the newly created project.
|
|
1088
|
+
|
|
1089
|
+
Raises
|
|
1090
|
+
------
|
|
1091
|
+
BadRequestException
|
|
1092
|
+
If the request to create the project fails with a status code
|
|
1093
|
+
indicating an error.
|
|
1094
|
+
"""
|
|
1095
|
+
data = {
|
|
1096
|
+
"name": project_name
|
|
1097
|
+
}
|
|
1098
|
+
headers = {
|
|
1099
|
+
"Content-type": "application/json",
|
|
1100
|
+
"apikey": self.apikey
|
|
1101
|
+
}
|
|
1102
|
+
r = retry_requests_post("{}/api/v1/projects?teamId={}".format(self.cloudos_url,
|
|
1103
|
+
workspace_id),
|
|
1104
|
+
json=data, headers=headers, verify=verify)
|
|
1105
|
+
if r.status_code == 401:
|
|
1106
|
+
raise ValueError('It seems your API key is not authorised. Please check if ' +
|
|
1107
|
+
'you have used the correct API key for the selected workspace')
|
|
1108
|
+
elif r.status_code == 409:
|
|
1109
|
+
raise ValueError(f'It seems that there is another project named "{project_name}" ' +
|
|
1110
|
+
'in your workspace, please use another name for the new project')
|
|
1111
|
+
elif r.status_code >= 400:
|
|
1112
|
+
raise BadRequestException(r)
|
|
1113
|
+
content = json.loads(r.content)
|
|
1114
|
+
return content['_id']
|
|
1115
|
+
|
|
919
1116
|
def get_workflow_max_pagination(self, workspace_id, workflow_name, verify=True):
|
|
920
1117
|
"""Retrieve the workflows max pages from API.
|
|
921
1118
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cloudos_cli
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.49.0
|
|
4
4
|
Summary: Python package for interacting with CloudOS
|
|
5
5
|
Home-page: https://github.com/lifebit-ai/cloudos-cli
|
|
6
6
|
Author: David Piñeyro
|
|
@@ -180,7 +180,7 @@ cloudos --help
|
|
|
180
180
|
│ configure CloudOS configuration. │
|
|
181
181
|
│ cromwell Cromwell server functionality: check status, start and stop. │
|
|
182
182
|
│ job CloudOS job functionality: run, check and abort jobs in CloudOS. │
|
|
183
|
-
│ project CloudOS project functionality: list projects in CloudOS.
|
|
183
|
+
│ project CloudOS project functionality: list and create projects in CloudOS. │
|
|
184
184
|
│ queue CloudOS job queue functionality. │
|
|
185
185
|
│ workflow CloudOS workflow functionality: list and import workflows. │
|
|
186
186
|
╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
@@ -775,16 +775,15 @@ This file can later be used when running a job with `cloudos job run --job-confi
|
|
|
775
775
|
> [!NOTE]
|
|
776
776
|
> Job details can only be retrieved for a single user, cannot see other user's job details.
|
|
777
777
|
|
|
778
|
-
#### Get a list of
|
|
778
|
+
#### Get a list of workspace jobs from a CloudOS
|
|
779
779
|
|
|
780
|
-
You can get a summary of
|
|
781
|
-
parameter) in two different formats:
|
|
780
|
+
You can get a summary of the workspace's last 30 submitted jobs (or a selected number of last jobs using `--last-n-jobs n` parameter) in two different formats:
|
|
782
781
|
|
|
783
782
|
- CSV: this is a table with a minimum predefined set of columns by default, or all the
|
|
784
783
|
available columns using the `--all-fields` argument.
|
|
785
|
-
- JSON: all the available information from
|
|
784
|
+
- JSON: all the available information from the workspace jobs, in JSON format (`--all-fields` is always enabled for this format).
|
|
786
785
|
|
|
787
|
-
To get a list with
|
|
786
|
+
To get a list with the workspace's last 30 submitted jobs, in CSV format, use
|
|
788
787
|
the following command:
|
|
789
788
|
|
|
790
789
|
```bash
|
|
@@ -806,7 +805,7 @@ Executing list...
|
|
|
806
805
|
|
|
807
806
|
In addition, a file named `joblist.csv` is created.
|
|
808
807
|
|
|
809
|
-
To get the same information, but for all
|
|
808
|
+
To get the same information, but for all the workspace's jobs and in JSON format, use the following command:
|
|
810
809
|
|
|
811
810
|
```bash
|
|
812
811
|
cloudos job list \
|
|
@@ -947,6 +946,25 @@ Executing list...
|
|
|
947
946
|
Workflow list saved to project_list.csv
|
|
948
947
|
```
|
|
949
948
|
|
|
949
|
+
#### Create a new project in CloudOS
|
|
950
|
+
|
|
951
|
+
You can create a new project in your CloudOS workspace using the `project create` command.
|
|
952
|
+
This command requires the name of the new project and will return the project ID upon successful creation.
|
|
953
|
+
|
|
954
|
+
```bash
|
|
955
|
+
cloudos project create \
|
|
956
|
+
--cloudos-url $CLOUDOS \
|
|
957
|
+
--apikey $MY_API_KEY \
|
|
958
|
+
--workspace-id $WORKSPACE_ID \
|
|
959
|
+
--new-project "My New Project"
|
|
960
|
+
```
|
|
961
|
+
|
|
962
|
+
The expected output is something similar to:
|
|
963
|
+
|
|
964
|
+
```console
|
|
965
|
+
Project "My New Project" created successfully with ID: 64f1a23b8e4c9d001234abcd
|
|
966
|
+
```
|
|
967
|
+
|
|
950
968
|
#### Get a list of the available job queues
|
|
951
969
|
|
|
952
970
|
Job queues are required for running jobs using AWS batch executor. The available job queues in your CloudOS workspace are
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
from click.testing import CliRunner
|
|
3
|
+
from cloudos_cli.__main__ import run_cloudos_cli
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def test_project_create_command_exists():
|
|
7
|
+
"""
|
|
8
|
+
Test that the 'project create' command exists and shows proper help
|
|
9
|
+
"""
|
|
10
|
+
runner = CliRunner()
|
|
11
|
+
|
|
12
|
+
# Test that project create command exists
|
|
13
|
+
result = runner.invoke(run_cloudos_cli, ['project', 'create', '--help'])
|
|
14
|
+
|
|
15
|
+
# Command should exist and not error out
|
|
16
|
+
assert result.exit_code == 0
|
|
17
|
+
|
|
18
|
+
# Check that the help text contains expected options
|
|
19
|
+
assert 'Create a new project in CloudOS' in result.output
|
|
20
|
+
assert '--new-project' in result.output
|
|
21
|
+
assert '--workspace-id' in result.output
|
|
22
|
+
assert '--apikey' in result.output
|
|
23
|
+
assert '--cloudos-url' in result.output
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_project_create_command_structure():
|
|
27
|
+
"""
|
|
28
|
+
Test that the 'project create' command has the correct structure and options
|
|
29
|
+
"""
|
|
30
|
+
runner = CliRunner()
|
|
31
|
+
|
|
32
|
+
# Test that the command exists and can show help without making API calls
|
|
33
|
+
result = runner.invoke(run_cloudos_cli, ['project', 'create', '--help'])
|
|
34
|
+
|
|
35
|
+
# Command should exist and show help properly
|
|
36
|
+
assert result.exit_code == 0
|
|
37
|
+
assert 'Create a new project in CloudOS' in result.output
|
|
38
|
+
assert '--new-project' in result.output
|
|
39
|
+
assert 'required' in result.output # Required arguments should be marked as such
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def test_project_group_contains_create_command():
|
|
43
|
+
"""
|
|
44
|
+
Test that the 'project' group contains the 'create' command
|
|
45
|
+
"""
|
|
46
|
+
runner = CliRunner()
|
|
47
|
+
|
|
48
|
+
# Test that project group shows create command
|
|
49
|
+
result = runner.invoke(run_cloudos_cli, ['project', '--help'])
|
|
50
|
+
|
|
51
|
+
assert result.exit_code == 0
|
|
52
|
+
assert 'create' in result.output
|
|
53
|
+
assert 'Create a new project in CloudOS' in result.output
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = '2.47.1'
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|