cloudos-cli 2.51.0__tar.gz → 2.56.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/PKG-INFO +75 -1
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/README.md +74 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli/__main__.py +381 -12
- cloudos_cli-2.56.0/cloudos_cli/_version.py +1 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli/clos.py +282 -35
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli/jobs/job.py +279 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli/utils/errors.py +18 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli.egg-info/PKG-INFO +75 -1
- cloudos_cli-2.51.0/cloudos_cli/_version.py +0 -1
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/LICENSE +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli/__init__.py +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli/configure/__init__.py +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli/configure/configure.py +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli/datasets/__init__.py +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli/datasets/datasets.py +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli/import_wf/__init__.py +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli/import_wf/import_wf.py +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli/jobs/__init__.py +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli/link/__init__.py +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli/link/link.py +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli/procurement/__init__.py +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli/procurement/images.py +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli/queue/__init__.py +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli/queue/queue.py +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli/utils/__init__.py +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli/utils/array_job.py +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli/utils/cloud.py +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli/utils/details.py +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli/utils/last_wf.py +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli/utils/requests.py +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli/utils/resources.py +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli.egg-info/SOURCES.txt +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli.egg-info/dependency_links.txt +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli.egg-info/entry_points.txt +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli.egg-info/requires.txt +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/cloudos_cli.egg-info/top_level.txt +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/setup.cfg +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/setup.py +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/tests/__init__.py +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/tests/functions_for_pytest.py +0 -0
- {cloudos_cli-2.51.0 → cloudos_cli-2.56.0}/tests/test_cli_project_create.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cloudos_cli
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.56.0
|
|
4
4
|
Summary: Python package for interacting with CloudOS
|
|
5
5
|
Home-page: https://github.com/lifebit-ai/cloudos-cli
|
|
6
6
|
Author: David Piñeyro
|
|
@@ -597,6 +597,34 @@ Executing results...
|
|
|
597
597
|
results: s3://path/to/location/of/results/results/
|
|
598
598
|
```
|
|
599
599
|
|
|
600
|
+
#### Query working directory of job
|
|
601
|
+
|
|
602
|
+
To get the working directory of a job submitted to CloudOS:
|
|
603
|
+
|
|
604
|
+
```shell
|
|
605
|
+
cloudos job workdir \
|
|
606
|
+
--apikey $MY_API_KEY \
|
|
607
|
+
--cloudos-url $CLOUDOS \
|
|
608
|
+
--job-id 62c83a1191fe06013b7ef355
|
|
609
|
+
```
|
|
610
|
+
|
|
611
|
+
Or with a defined profile:
|
|
612
|
+
|
|
613
|
+
```shell
|
|
614
|
+
cloudos job workdir \
|
|
615
|
+
--profile profile-name \
|
|
616
|
+
--job-id 62c83a1191fe06013b7ef355
|
|
617
|
+
```
|
|
618
|
+
|
|
619
|
+
The output should be something similar to:
|
|
620
|
+
|
|
621
|
+
```console
|
|
622
|
+
CloudOS job functionality: run, check and abort jobs in CloudOS.
|
|
623
|
+
|
|
624
|
+
Finding working directory path...
|
|
625
|
+
Working directory for job 68747bac9e7fe38ec6e022ad: az://123456789000.blob.core.windows.net/cloudos-987652349087/projects/455654676/jobs/54678856765/work
|
|
626
|
+
```
|
|
627
|
+
|
|
600
628
|
#### Abort single or multiple jobs from CloudOS
|
|
601
629
|
|
|
602
630
|
Aborts jobs in the CloudOS workspace that are either running or initialising. It can be used with one or more job IDs provided as a comma separated string using the `--job-ids` parameter.
|
|
@@ -615,6 +643,52 @@ Aborting jobs...
|
|
|
615
643
|
```
|
|
616
644
|
|
|
617
645
|
|
|
646
|
+
#### Clone/resume a job with optional parameter overrides
|
|
647
|
+
|
|
648
|
+
The `clone` and `resume` commands allows you to create a new job based on an existing job's configuration, with the ability to override specific parameters. This is useful for re-running jobs with slight modifications without having to specify all parameters from scratch.
|
|
649
|
+
|
|
650
|
+
Basic usage:
|
|
651
|
+
```console
|
|
652
|
+
cloudos job clone/resume \
|
|
653
|
+
--profile MY_PROFILE
|
|
654
|
+
--job-id "60a7b8c9d0e1f2g3h4i5j6k7"
|
|
655
|
+
```
|
|
656
|
+
|
|
657
|
+
Clone/resume with parameter overrides:
|
|
658
|
+
```console
|
|
659
|
+
cloudos job clone/resume \
|
|
660
|
+
--profile MY_PROFILE
|
|
661
|
+
--job-id "60a7b8c9d0e1f2g3h4i5j6k7" \
|
|
662
|
+
--job-queue "high-priority-queue" \
|
|
663
|
+
--cost-limit 50.0 \
|
|
664
|
+
--instance-type "c5.2xlarge" \
|
|
665
|
+
--job-name "cloned_analysis_v2" \
|
|
666
|
+
--nextflow-version "24.04.4" \
|
|
667
|
+
--git-branch "dev" \
|
|
668
|
+
--nextflow-profile "production" \
|
|
669
|
+
--do-not-save-logs true \
|
|
670
|
+
--accelerate-file-staging true \
|
|
671
|
+
--workflow-name "updated-workflow" \
|
|
672
|
+
-p "input=s3://new-bucket/input.csv" \
|
|
673
|
+
-p "output_dir=s3://new-bucket/results"
|
|
674
|
+
```
|
|
675
|
+
|
|
676
|
+
Available override options:
|
|
677
|
+
- `--job-queue`: Specify a different job queue
|
|
678
|
+
- `--cost-limit`: Set a new cost limit (use -1 for no limit)
|
|
679
|
+
- `--instance-type`: Change the master instance type
|
|
680
|
+
- `--job-name`: Assign a custom name to the cloned/resumed job
|
|
681
|
+
- `--nextflow-version`: Use a different Nextflow version
|
|
682
|
+
- `--git-branch`: Switch to a different git branch
|
|
683
|
+
- `--nextflow-profile`: Change the Nextflow profile
|
|
684
|
+
- `--do-not-save-logs`: Enable/disable log saving
|
|
685
|
+
- `--accelerate-file-staging`: Enable/disable fusion filesystem
|
|
686
|
+
- `--workflow-name`: Use a different workflow
|
|
687
|
+
- `-p, --parameter`: Override or add parameters (can be used multiple times)
|
|
688
|
+
|
|
689
|
+
> [!NOTE]
|
|
690
|
+
> Parameters can be overridden or new ones can be added using `-p` option
|
|
691
|
+
|
|
618
692
|
#### Executor support
|
|
619
693
|
|
|
620
694
|
CloudOS supports [AWS batch](https://www.nextflow.io/docs/latest/executor.html?highlight=executors#aws-batch) executor by default.
|
|
@@ -562,6 +562,34 @@ Executing results...
|
|
|
562
562
|
results: s3://path/to/location/of/results/results/
|
|
563
563
|
```
|
|
564
564
|
|
|
565
|
+
#### Query working directory of job
|
|
566
|
+
|
|
567
|
+
To get the working directory of a job submitted to CloudOS:
|
|
568
|
+
|
|
569
|
+
```shell
|
|
570
|
+
cloudos job workdir \
|
|
571
|
+
--apikey $MY_API_KEY \
|
|
572
|
+
--cloudos-url $CLOUDOS \
|
|
573
|
+
--job-id 62c83a1191fe06013b7ef355
|
|
574
|
+
```
|
|
575
|
+
|
|
576
|
+
Or with a defined profile:
|
|
577
|
+
|
|
578
|
+
```shell
|
|
579
|
+
cloudos job workdir \
|
|
580
|
+
--profile profile-name \
|
|
581
|
+
--job-id 62c83a1191fe06013b7ef355
|
|
582
|
+
```
|
|
583
|
+
|
|
584
|
+
The output should be something similar to:
|
|
585
|
+
|
|
586
|
+
```console
|
|
587
|
+
CloudOS job functionality: run, check and abort jobs in CloudOS.
|
|
588
|
+
|
|
589
|
+
Finding working directory path...
|
|
590
|
+
Working directory for job 68747bac9e7fe38ec6e022ad: az://123456789000.blob.core.windows.net/cloudos-987652349087/projects/455654676/jobs/54678856765/work
|
|
591
|
+
```
|
|
592
|
+
|
|
565
593
|
#### Abort single or multiple jobs from CloudOS
|
|
566
594
|
|
|
567
595
|
Aborts jobs in the CloudOS workspace that are either running or initialising. It can be used with one or more job IDs provided as a comma separated string using the `--job-ids` parameter.
|
|
@@ -580,6 +608,52 @@ Aborting jobs...
|
|
|
580
608
|
```
|
|
581
609
|
|
|
582
610
|
|
|
611
|
+
#### Clone/resume a job with optional parameter overrides
|
|
612
|
+
|
|
613
|
+
The `clone` and `resume` commands allows you to create a new job based on an existing job's configuration, with the ability to override specific parameters. This is useful for re-running jobs with slight modifications without having to specify all parameters from scratch.
|
|
614
|
+
|
|
615
|
+
Basic usage:
|
|
616
|
+
```console
|
|
617
|
+
cloudos job clone/resume \
|
|
618
|
+
--profile MY_PROFILE
|
|
619
|
+
--job-id "60a7b8c9d0e1f2g3h4i5j6k7"
|
|
620
|
+
```
|
|
621
|
+
|
|
622
|
+
Clone/resume with parameter overrides:
|
|
623
|
+
```console
|
|
624
|
+
cloudos job clone/resume \
|
|
625
|
+
--profile MY_PROFILE
|
|
626
|
+
--job-id "60a7b8c9d0e1f2g3h4i5j6k7" \
|
|
627
|
+
--job-queue "high-priority-queue" \
|
|
628
|
+
--cost-limit 50.0 \
|
|
629
|
+
--instance-type "c5.2xlarge" \
|
|
630
|
+
--job-name "cloned_analysis_v2" \
|
|
631
|
+
--nextflow-version "24.04.4" \
|
|
632
|
+
--git-branch "dev" \
|
|
633
|
+
--nextflow-profile "production" \
|
|
634
|
+
--do-not-save-logs true \
|
|
635
|
+
--accelerate-file-staging true \
|
|
636
|
+
--workflow-name "updated-workflow" \
|
|
637
|
+
-p "input=s3://new-bucket/input.csv" \
|
|
638
|
+
-p "output_dir=s3://new-bucket/results"
|
|
639
|
+
```
|
|
640
|
+
|
|
641
|
+
Available override options:
|
|
642
|
+
- `--job-queue`: Specify a different job queue
|
|
643
|
+
- `--cost-limit`: Set a new cost limit (use -1 for no limit)
|
|
644
|
+
- `--instance-type`: Change the master instance type
|
|
645
|
+
- `--job-name`: Assign a custom name to the cloned/resumed job
|
|
646
|
+
- `--nextflow-version`: Use a different Nextflow version
|
|
647
|
+
- `--git-branch`: Switch to a different git branch
|
|
648
|
+
- `--nextflow-profile`: Change the Nextflow profile
|
|
649
|
+
- `--do-not-save-logs`: Enable/disable log saving
|
|
650
|
+
- `--accelerate-file-staging`: Enable/disable fusion filesystem
|
|
651
|
+
- `--workflow-name`: Use a different workflow
|
|
652
|
+
- `-p, --parameter`: Override or add parameters (can be used multiple times)
|
|
653
|
+
|
|
654
|
+
> [!NOTE]
|
|
655
|
+
> Parameters can be overridden or new ones can be added using `-p` option
|
|
656
|
+
|
|
583
657
|
#### Executor support
|
|
584
658
|
|
|
585
659
|
CloudOS supports [AWS batch](https://www.nextflow.io/docs/latest/executor.html?highlight=executors#aws-batch) executor by default.
|
|
@@ -100,8 +100,11 @@ def run_cloudos_cli(ctx, debug):
|
|
|
100
100
|
'status': shared_config,
|
|
101
101
|
'list': shared_config,
|
|
102
102
|
'logs': shared_config,
|
|
103
|
+
'workdir': shared_config,
|
|
103
104
|
'results': shared_config,
|
|
104
|
-
'details': shared_config
|
|
105
|
+
'details': shared_config,
|
|
106
|
+
'clone': shared_config,
|
|
107
|
+
'resume': shared_config
|
|
105
108
|
},
|
|
106
109
|
'workflow': {
|
|
107
110
|
'list': shared_config,
|
|
@@ -161,8 +164,11 @@ def run_cloudos_cli(ctx, debug):
|
|
|
161
164
|
'status': shared_config,
|
|
162
165
|
'list': shared_config,
|
|
163
166
|
'logs': shared_config,
|
|
167
|
+
'workdir': shared_config,
|
|
164
168
|
'results': shared_config,
|
|
165
|
-
'details': shared_config
|
|
169
|
+
'details': shared_config,
|
|
170
|
+
'clone': shared_config,
|
|
171
|
+
'resume': shared_config
|
|
166
172
|
},
|
|
167
173
|
'workflow': {
|
|
168
174
|
'list': shared_config,
|
|
@@ -205,7 +211,7 @@ def run_cloudos_cli(ctx, debug):
|
|
|
205
211
|
|
|
206
212
|
@run_cloudos_cli.group()
|
|
207
213
|
def job():
|
|
208
|
-
"""CloudOS job functionality: run, check and abort jobs in CloudOS."""
|
|
214
|
+
"""CloudOS job functionality: run, clone, resume, check and abort jobs in CloudOS."""
|
|
209
215
|
print(job.__doc__ + '\n')
|
|
210
216
|
|
|
211
217
|
|
|
@@ -801,6 +807,83 @@ def job_status(ctx,
|
|
|
801
807
|
'or repeat the command you just used.')
|
|
802
808
|
|
|
803
809
|
|
|
810
|
+
@job.command('workdir')
|
|
811
|
+
@click.option('-k',
|
|
812
|
+
'--apikey',
|
|
813
|
+
help='Your CloudOS API key',
|
|
814
|
+
required=True)
|
|
815
|
+
@click.option('-c',
|
|
816
|
+
'--cloudos-url',
|
|
817
|
+
help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'),
|
|
818
|
+
default=CLOUDOS_URL,
|
|
819
|
+
required=True)
|
|
820
|
+
@click.option('--workspace-id',
|
|
821
|
+
help='The specific CloudOS workspace id.',
|
|
822
|
+
required=True)
|
|
823
|
+
@click.option('--job-id',
|
|
824
|
+
help='The job id in CloudOS to search for.',
|
|
825
|
+
required=True)
|
|
826
|
+
@click.option('--verbose',
|
|
827
|
+
help='Whether to print information messages or not.',
|
|
828
|
+
is_flag=True)
|
|
829
|
+
@click.option('--disable-ssl-verification',
|
|
830
|
+
help=('Disable SSL certificate verification. Please, remember that this option is ' +
|
|
831
|
+
'not generally recommended for security reasons.'),
|
|
832
|
+
is_flag=True)
|
|
833
|
+
@click.option('--ssl-cert',
|
|
834
|
+
help='Path to your SSL certificate file.')
|
|
835
|
+
@click.option('--profile', help='Profile to use from the config file', default=None)
|
|
836
|
+
@click.pass_context
|
|
837
|
+
def job_workdir(ctx,
|
|
838
|
+
apikey,
|
|
839
|
+
cloudos_url,
|
|
840
|
+
workspace_id,
|
|
841
|
+
job_id,
|
|
842
|
+
verbose,
|
|
843
|
+
disable_ssl_verification,
|
|
844
|
+
ssl_cert,
|
|
845
|
+
profile):
|
|
846
|
+
"""Get the path to the working directory of a specified job."""
|
|
847
|
+
profile = profile or ctx.default_map['job']['workdir']['profile']
|
|
848
|
+
# Create a dictionary with required and non-required params
|
|
849
|
+
required_dict = {
|
|
850
|
+
'apikey': True,
|
|
851
|
+
'workspace_id': True,
|
|
852
|
+
'workflow_name': False,
|
|
853
|
+
'project_name': False,
|
|
854
|
+
'procurement_id': False
|
|
855
|
+
}
|
|
856
|
+
# determine if the user provided all required parameters
|
|
857
|
+
config_manager = ConfigurationProfile()
|
|
858
|
+
user_options = (
|
|
859
|
+
config_manager.load_profile_and_validate_data(
|
|
860
|
+
ctx,
|
|
861
|
+
INIT_PROFILE,
|
|
862
|
+
CLOUDOS_URL,
|
|
863
|
+
profile=profile,
|
|
864
|
+
required_dict=required_dict,
|
|
865
|
+
apikey=apikey,
|
|
866
|
+
cloudos_url=cloudos_url,
|
|
867
|
+
workspace_id=workspace_id
|
|
868
|
+
)
|
|
869
|
+
)
|
|
870
|
+
apikey = user_options['apikey']
|
|
871
|
+
cloudos_url = user_options['cloudos_url']
|
|
872
|
+
workspace_id = user_options['workspace_id']
|
|
873
|
+
|
|
874
|
+
print('Finding working directory path...')
|
|
875
|
+
verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert)
|
|
876
|
+
if verbose:
|
|
877
|
+
print('\t...Preparing objects')
|
|
878
|
+
cl = Cloudos(cloudos_url, apikey, None)
|
|
879
|
+
if verbose:
|
|
880
|
+
print('\tThe following Cloudos object was created:')
|
|
881
|
+
print('\t' + str(cl) + '\n')
|
|
882
|
+
print(f'\tSearching for job id: {job_id}')
|
|
883
|
+
workdir = cl.get_job_workdir(job_id, workspace_id, verify_ssl)
|
|
884
|
+
print(f"Working directory for job {job_id}: {workdir}")
|
|
885
|
+
|
|
886
|
+
|
|
804
887
|
@job.command('logs')
|
|
805
888
|
@click.option('-k',
|
|
806
889
|
'--apikey',
|
|
@@ -1225,6 +1308,24 @@ def job_details(ctx,
|
|
|
1225
1308
|
@click.option('--archived',
|
|
1226
1309
|
help=('When this flag is used, only archived jobs list is collected.'),
|
|
1227
1310
|
is_flag=True)
|
|
1311
|
+
@click.option('--filter-status',
|
|
1312
|
+
help='Filter jobs by status (e.g., completed, running, failed, aborted).')
|
|
1313
|
+
@click.option('--filter-job-name',
|
|
1314
|
+
help='Filter jobs by job name ( case insensitive ).')
|
|
1315
|
+
@click.option('--filter-project',
|
|
1316
|
+
help='Filter jobs by project name.')
|
|
1317
|
+
@click.option('--filter-workflow',
|
|
1318
|
+
help='Filter jobs by workflow/pipeline name.')
|
|
1319
|
+
@click.option('--last',
|
|
1320
|
+
help=('When workflows are duplicated, use the latest imported workflow (by date).'),
|
|
1321
|
+
is_flag=True)
|
|
1322
|
+
@click.option('--filter-job-id',
|
|
1323
|
+
help='Filter jobs by specific job ID.')
|
|
1324
|
+
@click.option('--filter-only-mine',
|
|
1325
|
+
help='Filter to show only jobs belonging to the current user.',
|
|
1326
|
+
is_flag=True)
|
|
1327
|
+
@click.option('--filter-queue',
|
|
1328
|
+
help='Filter jobs by queue name. Only applies to jobs running in batch environment. Non-batch jobs are preserved in results.')
|
|
1228
1329
|
@click.option('--verbose',
|
|
1229
1330
|
help='Whether to print information messages or not.',
|
|
1230
1331
|
is_flag=True)
|
|
@@ -1246,6 +1347,15 @@ def list_jobs(ctx,
|
|
|
1246
1347
|
last_n_jobs,
|
|
1247
1348
|
page,
|
|
1248
1349
|
archived,
|
|
1350
|
+
filter_status,
|
|
1351
|
+
filter_job_name,
|
|
1352
|
+
filter_project,
|
|
1353
|
+
filter_workflow,
|
|
1354
|
+
last,
|
|
1355
|
+
filter_job_id,
|
|
1356
|
+
filter_only_mine,
|
|
1357
|
+
#filter_owner,
|
|
1358
|
+
filter_queue,
|
|
1249
1359
|
verbose,
|
|
1250
1360
|
disable_ssl_verification,
|
|
1251
1361
|
ssl_cert,
|
|
@@ -1300,7 +1410,15 @@ def list_jobs(ctx,
|
|
|
1300
1410
|
print("[ERROR] last-n-jobs value was not valid. Please use a positive int or 'all'")
|
|
1301
1411
|
raise
|
|
1302
1412
|
|
|
1303
|
-
my_jobs_r = cl.get_job_list(workspace_id, last_n_jobs, page, archived, verify_ssl
|
|
1413
|
+
my_jobs_r = cl.get_job_list(workspace_id, last_n_jobs, page, archived, verify_ssl,
|
|
1414
|
+
filter_status=filter_status,
|
|
1415
|
+
filter_job_name=filter_job_name,
|
|
1416
|
+
filter_project=filter_project,
|
|
1417
|
+
filter_workflow=filter_workflow,
|
|
1418
|
+
filter_job_id=filter_job_id,
|
|
1419
|
+
filter_only_mine=filter_only_mine,
|
|
1420
|
+
filter_queue=filter_queue,
|
|
1421
|
+
last=last)
|
|
1304
1422
|
if len(my_jobs_r) == 0:
|
|
1305
1423
|
if ctx.get_parameter_source('page') == click.core.ParameterSource.DEFAULT:
|
|
1306
1424
|
print('\t[Message] A total of 0 jobs collected. This is likely because your workspace ' +
|
|
@@ -1418,6 +1536,185 @@ def abort_jobs(ctx,
|
|
|
1418
1536
|
cl.abort_job(job, workspace_id, verify_ssl)
|
|
1419
1537
|
print(f"\tJob '{job}' aborted successfully.")
|
|
1420
1538
|
|
|
1539
|
+
@click.command()
|
|
1540
|
+
@click.option('-k',
|
|
1541
|
+
'--apikey',
|
|
1542
|
+
help='Your CloudOS API key',
|
|
1543
|
+
required=True)
|
|
1544
|
+
@click.option('-c',
|
|
1545
|
+
'--cloudos-url',
|
|
1546
|
+
help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'),
|
|
1547
|
+
default=CLOUDOS_URL,
|
|
1548
|
+
required=True)
|
|
1549
|
+
@click.option('--workspace-id',
|
|
1550
|
+
help='The specific CloudOS workspace id.',
|
|
1551
|
+
required=True)
|
|
1552
|
+
@click.option('--project-name',
|
|
1553
|
+
help='The name of a CloudOS project.')
|
|
1554
|
+
@click.option('-p',
|
|
1555
|
+
'--parameter',
|
|
1556
|
+
multiple=True,
|
|
1557
|
+
help=('A single parameter to pass to the job call. It should be in the ' +
|
|
1558
|
+
'following form: parameter_name=parameter_value. E.g.: ' +
|
|
1559
|
+
'-p input=s3://path_to_my_file. You can use this option as many ' +
|
|
1560
|
+
'times as parameters you want to include.'))
|
|
1561
|
+
@click.option('--nextflow-profile',
|
|
1562
|
+
help=('A comma separated string indicating the nextflow profile/s ' +
|
|
1563
|
+
'to use with your job.'))
|
|
1564
|
+
@click.option('--nextflow-version',
|
|
1565
|
+
help=('Nextflow version to use when executing the workflow in CloudOS. ' +
|
|
1566
|
+
'Default=22.10.8.'),
|
|
1567
|
+
type=click.Choice(['22.10.8', '24.04.4', '22.11.1-edge', 'latest']))
|
|
1568
|
+
@click.option('--git-branch',
|
|
1569
|
+
help=('The branch to run for the selected pipeline. ' +
|
|
1570
|
+
'If not specified it defaults to the last commit ' +
|
|
1571
|
+
'of the default branch.'))
|
|
1572
|
+
@click.option('--job-name',
|
|
1573
|
+
help='The name of the job. If not set, will take the name of the cloned job.')
|
|
1574
|
+
@click.option('--do-not-save-logs',
|
|
1575
|
+
help=('Avoids process log saving. If you select this option, your job process ' +
|
|
1576
|
+
'logs will not be stored.'),
|
|
1577
|
+
is_flag=True)
|
|
1578
|
+
@click.option('--job-queue',
|
|
1579
|
+
help=('Name of the job queue to use with a batch job. ' +
|
|
1580
|
+
'In Azure workspaces, this option is ignored.'))
|
|
1581
|
+
@click.option('--instance-type',
|
|
1582
|
+
help=('The type of compute instance to use as master node. ' +
|
|
1583
|
+
'Default=c5.xlarge(aws)|Standard_D4as_v4(azure).'))
|
|
1584
|
+
@click.option('--cost-limit',
|
|
1585
|
+
help='Add a cost limit to your job. Default=30.0 (For no cost limit please use -1).',
|
|
1586
|
+
type=float)
|
|
1587
|
+
@click.option('--job-id',
|
|
1588
|
+
help='The CloudOS job id of the job to be cloned.',
|
|
1589
|
+
required=True)
|
|
1590
|
+
@click.option('--accelerate-file-staging',
|
|
1591
|
+
help='Enables AWS S3 mountpoint for quicker file staging.',
|
|
1592
|
+
is_flag=True)
|
|
1593
|
+
@click.option('--resumable',
|
|
1594
|
+
help='Whether to make the job able to be resumed or not.',
|
|
1595
|
+
is_flag=True)
|
|
1596
|
+
@click.option('--verbose',
|
|
1597
|
+
help='Whether to print information messages or not.',
|
|
1598
|
+
is_flag=True)
|
|
1599
|
+
@click.option('--disable-ssl-verification',
|
|
1600
|
+
help=('Disable SSL certificate verification. Please, remember that this option is ' +
|
|
1601
|
+
'not generally recommended for security reasons.'),
|
|
1602
|
+
is_flag=True)
|
|
1603
|
+
@click.option('--ssl-cert',
|
|
1604
|
+
help='Path to your SSL certificate file.')
|
|
1605
|
+
@click.option('--profile',
|
|
1606
|
+
help='Profile to use from the config file',
|
|
1607
|
+
default=None)
|
|
1608
|
+
@click.pass_context
|
|
1609
|
+
def clone_resume(ctx,
|
|
1610
|
+
apikey,
|
|
1611
|
+
cloudos_url,
|
|
1612
|
+
workspace_id,
|
|
1613
|
+
project_name,
|
|
1614
|
+
parameter,
|
|
1615
|
+
nextflow_profile,
|
|
1616
|
+
nextflow_version,
|
|
1617
|
+
git_branch,
|
|
1618
|
+
job_name,
|
|
1619
|
+
do_not_save_logs,
|
|
1620
|
+
job_queue,
|
|
1621
|
+
instance_type,
|
|
1622
|
+
cost_limit,
|
|
1623
|
+
job_id,
|
|
1624
|
+
accelerate_file_staging,
|
|
1625
|
+
resumable,
|
|
1626
|
+
verbose,
|
|
1627
|
+
disable_ssl_verification,
|
|
1628
|
+
ssl_cert,
|
|
1629
|
+
profile):
|
|
1630
|
+
if ctx.info_name == "clone":
|
|
1631
|
+
mode, action = "clone", "cloning"
|
|
1632
|
+
elif ctx.info_name == "resume":
|
|
1633
|
+
mode, action = "resume", "resuming"
|
|
1634
|
+
|
|
1635
|
+
f"""{mode.capitalize()} an existing job with optional parameter overrides."""
|
|
1636
|
+
profile = profile or ctx.default_map['job'][mode]['profile']
|
|
1637
|
+
|
|
1638
|
+
# Create a dictionary with required and non-required params
|
|
1639
|
+
required_dict = {
|
|
1640
|
+
'apikey': True,
|
|
1641
|
+
'workspace_id': True,
|
|
1642
|
+
'workflow_name': False,
|
|
1643
|
+
'project_name': False,
|
|
1644
|
+
'procurement_id': False
|
|
1645
|
+
}
|
|
1646
|
+
|
|
1647
|
+
# Determine if the user provided all required parameters
|
|
1648
|
+
config_manager = ConfigurationProfile()
|
|
1649
|
+
user_options = (
|
|
1650
|
+
config_manager.load_profile_and_validate_data(
|
|
1651
|
+
ctx,
|
|
1652
|
+
INIT_PROFILE,
|
|
1653
|
+
CLOUDOS_URL,
|
|
1654
|
+
profile=profile,
|
|
1655
|
+
required_dict=required_dict,
|
|
1656
|
+
apikey=apikey,
|
|
1657
|
+
cloudos_url=cloudos_url,
|
|
1658
|
+
workspace_id=workspace_id,
|
|
1659
|
+
project_name=project_name
|
|
1660
|
+
)
|
|
1661
|
+
)
|
|
1662
|
+
apikey = user_options['apikey']
|
|
1663
|
+
cloudos_url = user_options['cloudos_url']
|
|
1664
|
+
workspace_id = user_options['workspace_id']
|
|
1665
|
+
|
|
1666
|
+
verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert)
|
|
1667
|
+
|
|
1668
|
+
print(f'{action.capitalize()} job...')
|
|
1669
|
+
if verbose:
|
|
1670
|
+
print('\t...Preparing objects')
|
|
1671
|
+
|
|
1672
|
+
# Create Job object (set dummy values for project_name and workflow_name, since they come from the cloned job)
|
|
1673
|
+
job_obj = jb.Job(cloudos_url, apikey, None, workspace_id, None, None, workflow_id=1234, project_id="None",
|
|
1674
|
+
mainfile=None, importsfile=None,verify=verify_ssl)
|
|
1675
|
+
|
|
1676
|
+
if verbose:
|
|
1677
|
+
print('\tThe following Job object was created:')
|
|
1678
|
+
print('\t' + str(job_obj) + '\n')
|
|
1679
|
+
print(f'\t{action.capitalize()} job {job_id} in workspace: {workspace_id}')
|
|
1680
|
+
|
|
1681
|
+
try:
|
|
1682
|
+
# Clone/resume the job with provided overrides
|
|
1683
|
+
cloned_resumed_job_id = job_obj.clone_or_resume_job(
|
|
1684
|
+
source_job_id=job_id,
|
|
1685
|
+
queue_name=job_queue,
|
|
1686
|
+
cost_limit=cost_limit,
|
|
1687
|
+
master_instance=instance_type,
|
|
1688
|
+
job_name=job_name,
|
|
1689
|
+
nextflow_version=nextflow_version,
|
|
1690
|
+
branch=git_branch,
|
|
1691
|
+
profile=nextflow_profile,
|
|
1692
|
+
do_not_save_logs=do_not_save_logs,
|
|
1693
|
+
use_fusion=accelerate_file_staging,
|
|
1694
|
+
resumable=resumable,
|
|
1695
|
+
# only when explicitly setting --project-name will be overridden, else using the original project
|
|
1696
|
+
project_name=project_name if ctx.get_parameter_source("project_name") == click.core.ParameterSource.COMMANDLINE else None,
|
|
1697
|
+
parameters=list(parameter) if parameter else None,
|
|
1698
|
+
verify=verify_ssl,
|
|
1699
|
+
mode=mode
|
|
1700
|
+
)
|
|
1701
|
+
if verbose:
|
|
1702
|
+
print(f'\t{mode.capitalize()}d job ID: {cloned_resumed_job_id}')
|
|
1703
|
+
|
|
1704
|
+
print(f"Job successfully {mode}d. New job ID: {cloned_resumed_job_id}")
|
|
1705
|
+
|
|
1706
|
+
except BadRequestException as e:
|
|
1707
|
+
if verbose:
|
|
1708
|
+
print(f'\tError details: {e}')
|
|
1709
|
+
raise ValueError(f"Failed to {mode} job: {e}")
|
|
1710
|
+
except Exception as e:
|
|
1711
|
+
if verbose:
|
|
1712
|
+
print(f'\tError details: {e}')
|
|
1713
|
+
raise ValueError(f"An error occurred while {action} the job: {e}")
|
|
1714
|
+
# Register the same function under two names
|
|
1715
|
+
job.add_command(clone_resume, "clone")
|
|
1716
|
+
job.add_command(clone_resume, "resume")
|
|
1717
|
+
|
|
1421
1718
|
|
|
1422
1719
|
@workflow.command('list')
|
|
1423
1720
|
@click.option('-k',
|
|
@@ -2827,7 +3124,7 @@ def run_bash_array_job(ctx,
|
|
|
2827
3124
|
@click.option('--details',
|
|
2828
3125
|
help=('When selected, it prints the details of the listed files. ' +
|
|
2829
3126
|
'Details contains "Type", "Owner", "Size", "Last Updated", ' +
|
|
2830
|
-
'"
|
|
3127
|
+
'"Virtual Name", "Storage Path".'),
|
|
2831
3128
|
is_flag=True)
|
|
2832
3129
|
@click.pass_context
|
|
2833
3130
|
def list_files(ctx,
|
|
@@ -2896,7 +3193,7 @@ def list_files(ctx,
|
|
|
2896
3193
|
table.add_column("Owner", style="white")
|
|
2897
3194
|
table.add_column("Size", style="magenta")
|
|
2898
3195
|
table.add_column("Last Updated", style="green")
|
|
2899
|
-
table.add_column("
|
|
3196
|
+
table.add_column("Virtual Name", style="bold", overflow="fold")
|
|
2900
3197
|
table.add_column("Storage Path", style="dim", no_wrap=False, overflow="fold", ratio=2)
|
|
2901
3198
|
|
|
2902
3199
|
for item in contents:
|
|
@@ -3087,7 +3384,7 @@ def move_files(ctx, source_path, destination_path, apikey, cloudos_url, workspac
|
|
|
3087
3384
|
if folder_type in ("VirtualFolder", "Folder"):
|
|
3088
3385
|
target_kind = "Folder"
|
|
3089
3386
|
elif folder_type=="S3Folder":
|
|
3090
|
-
click.echo(f"[ERROR]
|
|
3387
|
+
click.echo(f"[ERROR] Unable to move item '{source_item_name}' to '{destination_path}'. The destination is an S3 folder, and only virtual folders can be selected as valid move destinations.",
|
|
3091
3388
|
err=True)
|
|
3092
3389
|
sys.exit(1)
|
|
3093
3390
|
elif isinstance(folder_type, bool) and folder_type: # legacy dataset structure
|
|
@@ -3577,20 +3874,20 @@ def rm_item(ctx, target_path, apikey, cloudos_url,
|
|
|
3577
3874
|
click.echo(f"[ERROR] Item '{item_name}' could not be removed as the parent folder is not modifiable.",
|
|
3578
3875
|
err=True)
|
|
3579
3876
|
sys.exit(1)
|
|
3580
|
-
click.echo(f"
|
|
3877
|
+
click.echo(f"Removing {kind} '{item_name}' from '{parent_path or '[root]'}'...")
|
|
3581
3878
|
try:
|
|
3582
3879
|
response = client.delete_item(item_id=item_id, kind=kind)
|
|
3583
3880
|
if response.ok:
|
|
3584
3881
|
click.secho(
|
|
3585
|
-
f"[SUCCESS] {kind} '{item_name}' was
|
|
3882
|
+
f"[SUCCESS] {kind} '{item_name}' was removed from '{parent_path or '[root]'}'.",
|
|
3586
3883
|
fg="green", bold=True
|
|
3587
3884
|
)
|
|
3588
3885
|
click.secho("This item will still be available on your Cloud Provider.", fg="yellow")
|
|
3589
3886
|
else:
|
|
3590
|
-
click.echo(f"[ERROR]
|
|
3887
|
+
click.echo(f"[ERROR] Removal failed: {response.status_code} - {response.text}", err=True)
|
|
3591
3888
|
sys.exit(1)
|
|
3592
3889
|
except Exception as e:
|
|
3593
|
-
click.echo(f"[ERROR]
|
|
3890
|
+
click.echo(f"[ERROR] Remove operation failed: {str(e)}", err=True)
|
|
3594
3891
|
sys.exit(1)
|
|
3595
3892
|
|
|
3596
3893
|
|
|
@@ -3664,7 +3961,79 @@ def link(ctx, path, apikey, cloudos_url, project_name, workspace_id, session_id,
|
|
|
3664
3961
|
project_name=project_name,
|
|
3665
3962
|
verify=verify_ssl
|
|
3666
3963
|
)
|
|
3667
|
-
|
|
3964
|
+
|
|
3965
|
+
# Minimal folder validation and improved error messages
|
|
3966
|
+
is_s3 = path.startswith("s3://")
|
|
3967
|
+
is_folder = True
|
|
3968
|
+
|
|
3969
|
+
if is_s3:
|
|
3970
|
+
# S3 path validation - use heuristics to determine if it's likely a folder
|
|
3971
|
+
try:
|
|
3972
|
+
# If path ends with '/', it's likely a folder
|
|
3973
|
+
if path.endswith('/'):
|
|
3974
|
+
is_folder = True
|
|
3975
|
+
else:
|
|
3976
|
+
# Check the last part of the path
|
|
3977
|
+
path_parts = path.rstrip("/").split("/")
|
|
3978
|
+
if path_parts:
|
|
3979
|
+
last_part = path_parts[-1]
|
|
3980
|
+
# If the last part has no dot, it's likely a folder
|
|
3981
|
+
if '.' not in last_part:
|
|
3982
|
+
is_folder = True
|
|
3983
|
+
else:
|
|
3984
|
+
# If it has a dot, it might be a file - set to None for warning
|
|
3985
|
+
is_folder = None
|
|
3986
|
+
else:
|
|
3987
|
+
# Empty path parts, set to None for uncertainty
|
|
3988
|
+
is_folder = None
|
|
3989
|
+
except Exception:
|
|
3990
|
+
# If we can't parse the S3 path, set to None for uncertainty
|
|
3991
|
+
is_folder = None
|
|
3992
|
+
else:
|
|
3993
|
+
# File Explorer path validation (existing logic)
|
|
3994
|
+
try:
|
|
3995
|
+
datasets = Datasets(
|
|
3996
|
+
cloudos_url=cloudos_url,
|
|
3997
|
+
apikey=apikey,
|
|
3998
|
+
workspace_id=workspace_id,
|
|
3999
|
+
project_name=project_name,
|
|
4000
|
+
verify=verify_ssl,
|
|
4001
|
+
cromwell_token=None
|
|
4002
|
+
)
|
|
4003
|
+
parts = path.strip("/").split("/")
|
|
4004
|
+
parent_path = "/".join(parts[:-1]) if len(parts) > 1 else ""
|
|
4005
|
+
item_name = parts[-1]
|
|
4006
|
+
contents = datasets.list_folder_content(parent_path)
|
|
4007
|
+
found = None
|
|
4008
|
+
for item in contents.get("folders", []):
|
|
4009
|
+
if item.get("name") == item_name:
|
|
4010
|
+
found = item
|
|
4011
|
+
break
|
|
4012
|
+
if not found:
|
|
4013
|
+
for item in contents.get("files", []):
|
|
4014
|
+
if item.get("name") == item_name:
|
|
4015
|
+
found = item
|
|
4016
|
+
break
|
|
4017
|
+
if found and ("folderType" not in found):
|
|
4018
|
+
is_folder = False
|
|
4019
|
+
except Exception:
|
|
4020
|
+
is_folder = None
|
|
4021
|
+
|
|
4022
|
+
if is_folder is False:
|
|
4023
|
+
if is_s3:
|
|
4024
|
+
click.echo("[ERROR] The S3 path appears to point to a file, not a folder. You can only link folders. Please link the parent folder instead.", err=True)
|
|
4025
|
+
else:
|
|
4026
|
+
click.echo("[ERROR] Linking is only supported for folders, not individual files. Please link the parent folder instead.", err=True)
|
|
4027
|
+
return
|
|
4028
|
+
elif is_folder is None and is_s3:
|
|
4029
|
+
click.echo("[WARNING] Unable to verify whether the S3 path is a folder. Proceeding with linking; however, if the operation fails, please confirm that you are linking a folder rather than a file.", err=True)
|
|
4030
|
+
|
|
4031
|
+
try:
|
|
4032
|
+
link_p.link_folder(path, session_id)
|
|
4033
|
+
except Exception as e:
|
|
4034
|
+
click.echo(f"[ERROR] Could not link folder: {e}", err=True)
|
|
4035
|
+
if is_s3:
|
|
4036
|
+
click.echo("If you are linking an S3 path, please ensure it is a folder.", err=True)
|
|
3668
4037
|
|
|
3669
4038
|
|
|
3670
4039
|
@images.command(name="ls")
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = '2.56.0'
|