dxpy 0.389.0__tar.gz → 0.391.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dxpy-0.389.0 → dxpy-0.391.0}/PKG-INFO +1 -1
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/dxfile.py +2 -1
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/dxfile_functions.py +103 -30
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/cli/download.py +3 -1
- dxpy-0.391.0/dxpy/nextflow/awscli_assets.json +9 -0
- dxpy-0.391.0/dxpy/nextflow/awscli_assets.staging.json +9 -0
- dxpy-0.391.0/dxpy/nextflow/nextaur_assets.json +9 -0
- dxpy-0.391.0/dxpy/nextflow/nextaur_assets.staging.json +9 -0
- dxpy-0.391.0/dxpy/nextflow/nextflow_assets.json +9 -0
- dxpy-0.391.0/dxpy/nextflow/nextflow_assets.staging.json +9 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/nextflow/nextflow_builder.py +27 -28
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/nextflow/nextflow_utils.py +32 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/templating/templates/nextflow/dxapp.json +1 -1
- dxpy-0.391.0/dxpy/toolkit_version.py +1 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy.egg-info/PKG-INFO +1 -1
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy.egg-info/requires.txt +2 -1
- {dxpy-0.389.0 → dxpy-0.391.0}/requirements.txt +2 -1
- {dxpy-0.389.0 → dxpy-0.391.0}/test/test_nextflow.py +2 -1
- dxpy-0.389.0/dxpy/nextflow/awscli_assets.json +0 -9
- dxpy-0.389.0/dxpy/nextflow/awscli_assets.staging.json +0 -9
- dxpy-0.389.0/dxpy/nextflow/nextaur_assets.json +0 -9
- dxpy-0.389.0/dxpy/nextflow/nextaur_assets.staging.json +0 -9
- dxpy-0.389.0/dxpy/nextflow/nextflow_assets.json +0 -9
- dxpy-0.389.0/dxpy/nextflow/nextflow_assets.staging.json +0 -9
- dxpy-0.389.0/dxpy/toolkit_version.py +0 -1
- {dxpy-0.389.0 → dxpy-0.391.0}/MANIFEST.in +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/Readme.md +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/__init__.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/api.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/app_builder.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/app_categories.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/asset_builder.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/__init__.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/apollo/__init__.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/apollo/cmd_line_options_validator.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/apollo/data_transformations.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/apollo/dataset.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/apollo/json_validation_by_schema.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/apollo/schemas/__init__.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/apollo/schemas/assay_filtering_conditions.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/apollo/schemas/assay_filtering_json_schemas.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/apollo/schemas/input_arguments_validation_schemas.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/apollo/vizclient.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/apollo/vizserver_filters_from_json_parser.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/apollo/vizserver_payload_builder.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/auth.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/download_all_inputs.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/dxanalysis.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/dxapp.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/dxapp_container_functions.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/dxapplet.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/dxdatabase.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/dxdatabase_functions.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/dxdataobject_functions.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/dxglobalworkflow.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/dxjob.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/dxproject.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/dxrecord.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/dxworkflow.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/mount_all_inputs.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/search.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/cli/__init__.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/cli/cp.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/cli/dataset_utilities.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/cli/exec_io.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/cli/help_messages.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/cli/org.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/cli/output_handling.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/cli/parsers.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/cli/workflow.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/compat.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/dx_extract_utils/Homo_sapiens_genes_manifest.json +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/dx_extract_utils/Homo_sapiens_genes_manifest_staging.json +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/dx_extract_utils/Homo_sapiens_genes_manifest_staging_vep.json +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/dx_extract_utils/Homo_sapiens_genes_manifest_vep.json +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/dx_extract_utils/__init__.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/dx_extract_utils/cohort_filter_payload.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/dx_extract_utils/column_conditions.json +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/dx_extract_utils/column_conversion.json +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/dx_extract_utils/filter_to_payload.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/dx_extract_utils/germline_utils.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/dx_extract_utils/input_validation.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/dx_extract_utils/input_validation_somatic.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/dx_extract_utils/retrieve_allele_schema.json +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/dx_extract_utils/retrieve_annotation_schema.json +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/dx_extract_utils/retrieve_bins.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/dx_extract_utils/retrieve_genotype_schema.json +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/dx_extract_utils/return_columns_allele.json +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/dx_extract_utils/return_columns_annotation.json +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/dx_extract_utils/return_columns_genotype.json +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/dx_extract_utils/return_columns_genotype_only.json +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/dx_extract_utils/somatic_filter_payload.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/dxlog.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/exceptions.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/executable_builder.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/nextflow/ImageRef.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/nextflow/ImageRefFactory.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/nextflow/__init__.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/nextflow/collect_images.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/nextflow/nextflow_templates.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/packages/__init__.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/scripts/__init__.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/scripts/dx.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/scripts/dx_app_wizard.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/scripts/dx_build_app.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/scripts/dx_build_applet.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/ssh_tunnel_app_support.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/system_requirements.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/templating/__init__.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/templating/bash.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/templating/python.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/templating/templates/Readme.md +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/templating/templates/bash/basic/dxapp.json +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/templating/templates/bash/basic/src/code.sh +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/templating/templates/bash/parallelized/dxapp.json +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/templating/templates/bash/parallelized/src/code.sh +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/templating/templates/bash/scatter-process-gather/dxapp.json +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/templating/templates/bash/scatter-process-gather/src/code.sh +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/templating/templates/nextflow/src/nextflow.sh +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/templating/templates/python/basic/dxapp.json +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/templating/templates/python/basic/src/code.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/templating/templates/python/basic/test/test.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/templating/templates/python/parallelized/dxapp.json +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/templating/templates/python/parallelized/src/code.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/templating/templates/python/parallelized/test/test.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/templating/templates/python/scatter-process-gather/dxapp.json +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/templating/templates/python/scatter-process-gather/src/code.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/templating/templates/python/scatter-process-gather/test/test.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/templating/utils.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/utils/__init__.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/utils/batch_utils.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/utils/completer.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/utils/config.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/utils/describe.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/utils/exec_utils.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/utils/executable_unbuilder.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/utils/file_handle.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/utils/file_load_utils.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/utils/genomic_utils.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/utils/job_log_client.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/utils/local_exec_utils.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/utils/pathmatch.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/utils/pretty_print.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/utils/printing.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/utils/resolver.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/utils/spelling_corrector.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/utils/version.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy/workflow_builder.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy.egg-info/SOURCES.txt +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy.egg-info/dependency_links.txt +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy.egg-info/entry_points.txt +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy.egg-info/not-zip-safe +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/dxpy.egg-info/top_level.txt +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/requirements_setuptools.txt +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/requirements_test.txt +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/scripts/dx-clone-asset +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/scripts/dx-docker +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/scripts/dx-download-all-inputs +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/scripts/dx-fetch-bundled-depends +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/scripts/dx-generate-dxapp +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/scripts/dx-jobutil-add-output +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/scripts/dx-jobutil-dxlink +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/scripts/dx-jobutil-get-identity-token +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/scripts/dx-jobutil-new-job +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/scripts/dx-jobutil-parse-link +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/scripts/dx-jobutil-report-error +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/scripts/dx-log-stream +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/scripts/dx-mount-all-inputs +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/scripts/dx-notebook-reconnect +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/scripts/dx-print-bash-vars +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/scripts/dx-upload-all-outputs +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/setup.cfg +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/setup.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/test/test_batch.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/test/test_create_cohort.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/test/test_describe.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/test/test_dx-docker.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/test/test_dx_app_wizard.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/test/test_dx_bash_helpers.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/test/test_dx_completion.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/test/test_dx_symlink.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/test/test_dxabs.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/test/test_dxasset.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/test/test_dxclient.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/test/test_dxpy.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/test/test_dxpy_utils.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/test/test_dxunpack.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/test/test_extract_assay.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/test/test_extract_dataset.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/test/test_extract_expression.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/test/test_extract_somatic.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/test/test_nextflow_ImageRef.py +0 -0
- {dxpy-0.389.0 → dxpy-0.391.0}/test/test_nextflow_ImageRefFactory.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: dxpy
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.391.0
|
|
4
4
|
Summary: DNAnexus Platform API bindings for Python
|
|
5
5
|
Home-page: https://github.com/dnanexus/dx-toolkit
|
|
6
6
|
Author: Aleksandra Zalcman, Andrey Kislyuk, Anurag Biyani, Geet Duggal, Katherine Lai, Kurt Jensen, Marek Hrvol, Ohad Rodeh, Phil Sung
|
|
@@ -47,6 +47,7 @@ if dxpy.JOB_ID:
|
|
|
47
47
|
|
|
48
48
|
MD5_READ_CHUNK_SIZE = 1024*1024*4
|
|
49
49
|
FILE_REQUEST_TIMEOUT = 60
|
|
50
|
+
PART_UPLOAD_REQUEST_TIMEOUT_SECONDS = 300
|
|
50
51
|
|
|
51
52
|
|
|
52
53
|
def _validate_headers(headers):
|
|
@@ -715,7 +716,7 @@ class DXFile(DXDataObject):
|
|
|
715
716
|
jsonify_data=False,
|
|
716
717
|
prepend_srv=False,
|
|
717
718
|
always_retry=True,
|
|
718
|
-
timeout=
|
|
719
|
+
timeout=PART_UPLOAD_REQUEST_TIMEOUT_SECONDS,
|
|
719
720
|
auth=None,
|
|
720
721
|
method='PUT')
|
|
721
722
|
|
|
@@ -32,6 +32,9 @@ from collections import defaultdict
|
|
|
32
32
|
import multiprocessing
|
|
33
33
|
from random import randint
|
|
34
34
|
from time import sleep
|
|
35
|
+
import crc32c
|
|
36
|
+
import zlib
|
|
37
|
+
import base64
|
|
35
38
|
|
|
36
39
|
import dxpy
|
|
37
40
|
from .. import logger
|
|
@@ -41,6 +44,7 @@ from ..exceptions import DXError, DXFileError, DXPartLengthMismatchError, DXChec
|
|
|
41
44
|
from ..compat import open, md5_hasher, USING_PYTHON2
|
|
42
45
|
from ..utils import response_iterator
|
|
43
46
|
import subprocess
|
|
47
|
+
import concurrent.futures
|
|
44
48
|
|
|
45
49
|
def open_dxfile(dxid, project=None, mode=None, read_buffer_size=dxfile.DEFAULT_BUFFER_SIZE):
|
|
46
50
|
'''
|
|
@@ -176,7 +180,6 @@ def _verify(filename, md5digest):
|
|
|
176
180
|
err_exit("Checksum doesn't match " + str(actual_md5) + " expected:" + str(md5digest))
|
|
177
181
|
print("Checksum correct")
|
|
178
182
|
|
|
179
|
-
|
|
180
183
|
# [dxid] is a symbolic link. Create a preauthenticated URL,
|
|
181
184
|
# and download it
|
|
182
185
|
def _download_symbolic_link(dxid, md5digest, project, dest_filename, symlink_max_tries=15):
|
|
@@ -226,6 +229,98 @@ def _download_symbolic_link(dxid, md5digest, project, dest_filename, symlink_max
|
|
|
226
229
|
if md5digest is not None:
|
|
227
230
|
_verify(dest_filename, md5digest)
|
|
228
231
|
|
|
232
|
+
|
|
233
|
+
def _verify_per_part_checksum_on_downloaded_file(filename, dxfile_desc, show_progress=False):
|
|
234
|
+
parts = dxfile_desc["parts"]
|
|
235
|
+
parts_to_get = sorted(parts, key=int)
|
|
236
|
+
file_size = dxfile_desc.get("size")
|
|
237
|
+
per_part_checksum = dxfile_desc.get('perPartCheckSum')
|
|
238
|
+
_bytes = 0
|
|
239
|
+
|
|
240
|
+
if per_part_checksum is None:
|
|
241
|
+
return
|
|
242
|
+
|
|
243
|
+
offset = 0
|
|
244
|
+
for part_id in parts_to_get:
|
|
245
|
+
parts[part_id]["start"] = offset
|
|
246
|
+
offset += parts[part_id]["size"]
|
|
247
|
+
|
|
248
|
+
def read_chunk(filename, start, size, part_id):
|
|
249
|
+
with open(filename, 'rb') as f:
|
|
250
|
+
f.seek(start)
|
|
251
|
+
chunk = f.read(size)
|
|
252
|
+
return (chunk, part_id)
|
|
253
|
+
|
|
254
|
+
def process_file_in_parallel(filename):
|
|
255
|
+
chunks = []
|
|
256
|
+
for part_id in parts_to_get:
|
|
257
|
+
part_info = parts[part_id]
|
|
258
|
+
start = part_info["start"]
|
|
259
|
+
size = part_info["size"]
|
|
260
|
+
chunks.append((start, size, part_id))
|
|
261
|
+
|
|
262
|
+
with concurrent.futures.ThreadPoolExecutor() as executor:
|
|
263
|
+
futures = [executor.submit(read_chunk, filename, start, size, part_id) for start, size, part_id in chunks]
|
|
264
|
+
for future in concurrent.futures.as_completed(futures):
|
|
265
|
+
yield future.result()
|
|
266
|
+
|
|
267
|
+
for (chunk, part_id) in process_file_in_parallel(filename):
|
|
268
|
+
_verify_per_part_checksum(parts, part_id, chunk, per_part_checksum, dxfile_desc['id'])
|
|
269
|
+
if show_progress:
|
|
270
|
+
_bytes += parts[part_id]["size"]
|
|
271
|
+
_print_progress(_bytes, file_size, filename, action="Verified")
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _verify_per_part_checksum(parts, part_id, chunk_data, per_part_checksum, dxfile_id):
|
|
275
|
+
if per_part_checksum is None:
|
|
276
|
+
return
|
|
277
|
+
|
|
278
|
+
part = parts[part_id]
|
|
279
|
+
expected_checksum = part.get('checksum')
|
|
280
|
+
verifiers = {
|
|
281
|
+
'CRC32': lambda data: zlib.crc32(data).to_bytes(4, 'big'),
|
|
282
|
+
'CRC32C': lambda data: crc32c.crc32c(data).to_bytes(4, 'big'),
|
|
283
|
+
'SHA1': lambda data: hashlib.sha1(data).digest(),
|
|
284
|
+
'SHA256': lambda data: hashlib.sha256(data).digest()
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
if per_part_checksum not in verifiers:
|
|
288
|
+
raise DXFileError("Unsupported per-part checksum type: {}".format(per_part_checksum))
|
|
289
|
+
if expected_checksum is None:
|
|
290
|
+
raise DXFileError("{} checksum not found in part {}".format(per_part_checksum, part_id))
|
|
291
|
+
|
|
292
|
+
expected_checksum = base64.b64decode(expected_checksum)
|
|
293
|
+
got_checksum = verifiers[per_part_checksum](chunk_data)
|
|
294
|
+
if got_checksum != expected_checksum:
|
|
295
|
+
raise DXChecksumMismatchError("Checksum mismatch in {} part {} (expected {}, got {})".format(dxfile_id, part_id, expected_checksum, got_checksum))
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def _print_progress(bytes_downloaded, file_size, filename, action="Downloaded"):
|
|
299
|
+
num_ticks = 60
|
|
300
|
+
|
|
301
|
+
effective_file_size = file_size or 1
|
|
302
|
+
if bytes_downloaded > effective_file_size:
|
|
303
|
+
effective_file_size = bytes_downloaded
|
|
304
|
+
|
|
305
|
+
ticks = int(round((bytes_downloaded / float(effective_file_size)) * num_ticks))
|
|
306
|
+
percent = int(math.floor((bytes_downloaded / float(effective_file_size)) * 100))
|
|
307
|
+
|
|
308
|
+
fmt = "[{done}{pending}] {action} {done_bytes:,}{remaining} bytes ({percent}%) {name}"
|
|
309
|
+
# Erase the line and return the cursor to the start of the line.
|
|
310
|
+
# The following VT100 escape sequence will erase the current line.
|
|
311
|
+
sys.stderr.write("\33[2K")
|
|
312
|
+
sys.stderr.write(fmt.format(action=action,
|
|
313
|
+
done=("=" * (ticks - 1) + ">") if ticks > 0 else "",
|
|
314
|
+
pending=" " * (num_ticks - ticks),
|
|
315
|
+
done_bytes=bytes_downloaded,
|
|
316
|
+
remaining=" of {size:,}".format(size=file_size) if file_size else "",
|
|
317
|
+
percent=percent,
|
|
318
|
+
name=filename))
|
|
319
|
+
sys.stderr.flush()
|
|
320
|
+
sys.stderr.write("\r")
|
|
321
|
+
sys.stderr.flush()
|
|
322
|
+
|
|
323
|
+
|
|
229
324
|
def _download_dxfile(dxid, filename, part_retry_counter,
|
|
230
325
|
chunksize=dxfile.DEFAULT_BUFFER_SIZE, append=False, show_progress=False,
|
|
231
326
|
project=None, describe_output=None, symlink_max_tries=15, **kwargs):
|
|
@@ -238,30 +333,6 @@ def _download_dxfile(dxid, filename, part_retry_counter,
|
|
|
238
333
|
- False means the download was stopped because of a retryable error
|
|
239
334
|
- Exception raised for other errors
|
|
240
335
|
'''
|
|
241
|
-
def print_progress(bytes_downloaded, file_size, action="Downloaded"):
|
|
242
|
-
num_ticks = 60
|
|
243
|
-
|
|
244
|
-
effective_file_size = file_size or 1
|
|
245
|
-
if bytes_downloaded > effective_file_size:
|
|
246
|
-
effective_file_size = bytes_downloaded
|
|
247
|
-
|
|
248
|
-
ticks = int(round((bytes_downloaded / float(effective_file_size)) * num_ticks))
|
|
249
|
-
percent = int(math.floor((bytes_downloaded / float(effective_file_size)) * 100))
|
|
250
|
-
|
|
251
|
-
fmt = "[{done}{pending}] {action} {done_bytes:,}{remaining} bytes ({percent}%) {name}"
|
|
252
|
-
# Erase the line and return the cursor to the start of the line.
|
|
253
|
-
# The following VT100 escape sequence will erase the current line.
|
|
254
|
-
sys.stderr.write("\33[2K")
|
|
255
|
-
sys.stderr.write(fmt.format(action=action,
|
|
256
|
-
done=("=" * (ticks - 1) + ">") if ticks > 0 else "",
|
|
257
|
-
pending=" " * (num_ticks - ticks),
|
|
258
|
-
done_bytes=bytes_downloaded,
|
|
259
|
-
remaining=" of {size:,}".format(size=file_size) if file_size else "",
|
|
260
|
-
percent=percent,
|
|
261
|
-
name=filename))
|
|
262
|
-
sys.stderr.flush()
|
|
263
|
-
sys.stderr.write("\r")
|
|
264
|
-
sys.stderr.flush()
|
|
265
336
|
|
|
266
337
|
_bytes = 0
|
|
267
338
|
|
|
@@ -283,6 +354,7 @@ def _download_dxfile(dxid, filename, part_retry_counter,
|
|
|
283
354
|
else:
|
|
284
355
|
md5 = None
|
|
285
356
|
_download_symbolic_link(dxid, md5, project, filename, symlink_max_tries=symlink_max_tries)
|
|
357
|
+
_verify_per_part_checksum_on_downloaded_file(filename, dxfile_desc, show_progress)
|
|
286
358
|
return True
|
|
287
359
|
|
|
288
360
|
parts = dxfile_desc["parts"]
|
|
@@ -303,7 +375,7 @@ def _download_dxfile(dxid, filename, part_retry_counter,
|
|
|
303
375
|
fh = open(filename, "wb")
|
|
304
376
|
|
|
305
377
|
if show_progress:
|
|
306
|
-
|
|
378
|
+
_print_progress(0, None, filename)
|
|
307
379
|
|
|
308
380
|
def get_chunk(part_id_to_get, start, end):
|
|
309
381
|
url, headers = dxfile.get_download_url(project=project, **kwargs)
|
|
@@ -333,6 +405,7 @@ def _download_dxfile(dxid, filename, part_retry_counter,
|
|
|
333
405
|
msg = "Checksum mismatch in {} part {} (expected {}, got {})"
|
|
334
406
|
msg = msg.format(dxfile.get_id(), _part_id, parts[_part_id]["md5"], hasher.hexdigest())
|
|
335
407
|
raise DXChecksumMismatchError(msg)
|
|
408
|
+
|
|
336
409
|
|
|
337
410
|
with fh:
|
|
338
411
|
last_verified_pos = 0
|
|
@@ -361,7 +434,7 @@ def _download_dxfile(dxid, filename, part_retry_counter,
|
|
|
361
434
|
last_verified_pos = fh.tell()
|
|
362
435
|
if show_progress:
|
|
363
436
|
_bytes += part_info["size"]
|
|
364
|
-
|
|
437
|
+
_print_progress(_bytes, file_size, filename, action="Verified")
|
|
365
438
|
except (IOError, DXFileError) as e:
|
|
366
439
|
logger.debug(e)
|
|
367
440
|
fh.seek(last_verified_pos)
|
|
@@ -369,7 +442,7 @@ def _download_dxfile(dxid, filename, part_retry_counter,
|
|
|
369
442
|
if last_verified_part is not None:
|
|
370
443
|
del parts_to_get[:parts_to_get.index(last_verified_part)+1]
|
|
371
444
|
if show_progress and len(parts_to_get) < len(parts):
|
|
372
|
-
|
|
445
|
+
_print_progress(last_verified_pos, file_size, filename, action="Resuming at")
|
|
373
446
|
logger.debug("Verified %s/%d downloaded parts", last_verified_part, len(parts_to_get))
|
|
374
447
|
|
|
375
448
|
try:
|
|
@@ -387,10 +460,10 @@ def _download_dxfile(dxid, filename, part_retry_counter,
|
|
|
387
460
|
fh.write(chunk_data)
|
|
388
461
|
if show_progress:
|
|
389
462
|
_bytes += len(chunk_data)
|
|
390
|
-
|
|
463
|
+
_print_progress(_bytes, file_size, filename)
|
|
391
464
|
verify_part(cur_part, got_bytes, hasher)
|
|
392
465
|
if show_progress:
|
|
393
|
-
|
|
466
|
+
_print_progress(_bytes, file_size, filename, action="Completed")
|
|
394
467
|
except DXFileError:
|
|
395
468
|
print(traceback.format_exc(), file=sys.stderr)
|
|
396
469
|
part_retry_counter[cur_part] -= 1
|
|
@@ -199,7 +199,9 @@ def download(args):
|
|
|
199
199
|
resolver_kwargs.update({"describe": {"parts": True,
|
|
200
200
|
"size": True,
|
|
201
201
|
"drive": True,
|
|
202
|
-
"md5": True
|
|
202
|
+
"md5": True,
|
|
203
|
+
"perPartCheckSum": True
|
|
204
|
+
}})
|
|
203
205
|
|
|
204
206
|
project, folderpath, matching_files = try_call(resolve_existing_path, path, **resolver_kwargs)
|
|
205
207
|
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
{
|
|
2
|
+
"aws:ap-southeast-2": "record-Gyv6Z6j5BzGXJBqy9BpBjJ6z",
|
|
3
|
+
"aws:eu-central-1": "record-Gyv6Zzj4qKFB410Z8vk9GXf7",
|
|
4
|
+
"aws:eu-west-2-g": "record-Gyv6ykBKQqy2K41YGfqpxYvF",
|
|
5
|
+
"aws:me-south-1": "record-Gyv70B932Vy4bg78PFvbB4Q7",
|
|
6
|
+
"aws:us-east-1": "record-Gyv6Q9j0PPgJQJ72226Gz7j7",
|
|
7
|
+
"azure:westeurope": "record-Gyv73f0BKJkvjx5FPJq6F22b",
|
|
8
|
+
"azure:westus": "record-Gyv77Vj9zv30Pfx8V890xQ6Q"
|
|
9
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
{
|
|
2
|
+
"aws:ap-southeast-2": "record-GyqkQ205V8yY469GKKXqybZG",
|
|
3
|
+
"aws:eu-central-1": "record-GyqkVp04VKZGgPBPYK78qGyx",
|
|
4
|
+
"aws:eu-west-2-g": "record-GyqkvqpK06ffFb9V8g80f7Gg",
|
|
5
|
+
"aws:me-south-1": "record-Gyqp5F13fKQ44GfVP0VkKZ0z",
|
|
6
|
+
"aws:us-east-1": "record-GyqkB3j08Gb14Pf0P8x5Xq5Q",
|
|
7
|
+
"azure:westeurope": "record-Gyqp8g8B3BVZ4GfVP0VkKZ15",
|
|
8
|
+
"azure:westus": "record-GyqpG7Q9Z3044GfVP0VkKZ1J"
|
|
9
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
{
|
|
2
|
+
"aws:ap-southeast-2": "record-Gyv791859gf3p9vxp7570612",
|
|
3
|
+
"aws:eu-central-1": "record-Gyv7B3j4BjY98Kk504Qj88QX",
|
|
4
|
+
"aws:eu-west-2-g": "record-Gyv7FQ2K68k82kgBGPx7vK97",
|
|
5
|
+
"aws:me-south-1": "record-Gyv7GxV3Z498b9VbyqJpgx7q",
|
|
6
|
+
"aws:us-east-1": "record-Gyv6V780kGJJXGY1vz4qJFQ9",
|
|
7
|
+
"azure:westeurope": "record-Gyv7QpQB9ZyxF3q6B86QQKVP",
|
|
8
|
+
"azure:westus": "record-Gyv7Zq09GFy7p6XJP24bVjY9"
|
|
9
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
{
|
|
2
|
+
"aws:ap-southeast-2": "record-GyqpK0851406Fb9V8g80f7JG",
|
|
3
|
+
"aws:eu-central-1": "record-GyqpPpj41Jj57Kk4fYYJ8yfB",
|
|
4
|
+
"aws:eu-west-2-g": "record-GyqpVXBK75YZZ110qZJbfbZ7",
|
|
5
|
+
"aws:me-south-1": "record-GyqpY3931Vq0vP9F7fQ22jxy",
|
|
6
|
+
"aws:us-east-1": "record-GyqkG980bJ6b7Kk4fYYJ8yZb",
|
|
7
|
+
"azure:westeurope": "record-GyqpfJ0BxQ4zPfQKqFkx8VG1",
|
|
8
|
+
"azure:westus": "record-GyqppG891Yv4Z110qZJbfbb0"
|
|
9
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
{
|
|
2
|
+
"aws:ap-southeast-2": "record-Gyv7f7858VGjqZ1jjJjyjJpv",
|
|
3
|
+
"aws:eu-central-1": "record-Gyv7g28474vK3JPK4g5BP1PB",
|
|
4
|
+
"aws:eu-west-2-g": "record-Gyv810BKZkKyyxkJGzbK2yV4",
|
|
5
|
+
"aws:me-south-1": "record-Gyv82vk306Q8b9VbyqJpjbpj",
|
|
6
|
+
"aws:us-east-1": "record-Gyv6XQ00zzv5bfPgGqqPJ6Gq",
|
|
7
|
+
"azure:westeurope": "record-Gyv86X8B9bQ8160VF0Gfp8QX",
|
|
8
|
+
"azure:westus": "record-Gyv8B3j94xjy8K0PK914Pvfp"
|
|
9
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
{
|
|
2
|
+
"aws:ap-southeast-2": "record-Gyqpv0Q5yv814Pf0P8x5Xq79",
|
|
3
|
+
"aws:eu-central-1": "record-Gyqpxv842Yx54X2y8zVkQzb5",
|
|
4
|
+
"aws:eu-west-2-g": "record-Gyqpz9pKP7qXqyjjGP2Vx4Bp",
|
|
5
|
+
"aws:me-south-1": "record-Gyqq12k3q795K9pb8Pfqvzf2",
|
|
6
|
+
"aws:us-east-1": "record-GyqkJ5j0z0fqyxpy40ygBGpK",
|
|
7
|
+
"azure:westeurope": "record-Gyqq478BqQQGgPBPYK78qJ1J",
|
|
8
|
+
"azure:westus": "record-Gyqq7zQ9JP62qyjjGP2Vx4F3"
|
|
9
|
+
}
|
|
@@ -6,9 +6,11 @@ import argparse
|
|
|
6
6
|
from glob import glob
|
|
7
7
|
import shutil
|
|
8
8
|
import tempfile
|
|
9
|
+
from functools import partial
|
|
9
10
|
|
|
10
11
|
from dxpy.nextflow.nextflow_templates import (get_nextflow_dxapp, get_nextflow_src)
|
|
11
|
-
from dxpy.nextflow.nextflow_utils import (get_template_dir, write_exec, write_dxapp, get_importer_name,
|
|
12
|
+
from dxpy.nextflow.nextflow_utils import (get_template_dir, write_exec, write_dxapp, get_importer_name,
|
|
13
|
+
create_readme, get_nested, get_allowed_extra_fields_mapping)
|
|
12
14
|
from dxpy.cli.exec_io import parse_obj
|
|
13
15
|
from dxpy.cli import try_call
|
|
14
16
|
from dxpy.utils.resolver import resolve_existing_path
|
|
@@ -49,44 +51,41 @@ def build_pipeline_with_npi(
|
|
|
49
51
|
Runs the Nextflow Pipeline Importer app, which creates a Nextflow applet from a given Git repository.
|
|
50
52
|
"""
|
|
51
53
|
|
|
52
|
-
def parse_extra_args(
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
if extra_args.get("details", {}).get("whatsNew") is not None:
|
|
63
|
-
dx_input["whats_new"] = extra_args.get("details", {}).get("whatsNew")
|
|
64
|
-
return dx_input
|
|
54
|
+
def parse_extra_args(args):
|
|
55
|
+
"""
|
|
56
|
+
:param args: extra args from command input
|
|
57
|
+
:returns: overridable fields from extra_args
|
|
58
|
+
"""
|
|
59
|
+
return {
|
|
60
|
+
target_key: val
|
|
61
|
+
for arg_path, target_key in get_allowed_extra_fields_mapping()
|
|
62
|
+
if (val := get_nested(args, arg_path)) is not None
|
|
63
|
+
}
|
|
65
64
|
|
|
66
65
|
extra_args = extra_args or {}
|
|
67
66
|
build_project_id = dxpy.WORKSPACE_ID
|
|
68
67
|
build_folder = None
|
|
69
68
|
input_hash = parse_extra_args(extra_args)
|
|
70
69
|
input_hash["repository_url"] = repository
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
70
|
+
|
|
71
|
+
# { NPI_input_name: (raw value, transformation function),...}
|
|
72
|
+
input_updates = {
|
|
73
|
+
"repository_tag": (tag, None),
|
|
74
|
+
"config_profile": (profile, None),
|
|
75
|
+
"cache_docker": (cache_docker, None),
|
|
76
|
+
"nextflow_pipeline_params": (nextflow_pipeline_params, None),
|
|
77
|
+
"docker_secrets": (docker_secrets, partial(parse_obj, klass="file")),
|
|
78
|
+
"github_credentials": (git_creds, partial(parse_obj, klass="file")),
|
|
79
|
+
}
|
|
80
|
+
for key, (raw_value, transform) in input_updates.items():
|
|
81
|
+
if raw_value:
|
|
82
|
+
input_hash[key] = transform(raw_value) if transform else raw_value
|
|
83
|
+
|
|
77
84
|
if destination:
|
|
78
85
|
build_project_id, build_folder, _ = try_call(resolve_existing_path, destination, expected='folder')
|
|
79
|
-
if docker_secrets:
|
|
80
|
-
input_hash["docker_secrets"] = parse_obj(docker_secrets, "file")
|
|
81
|
-
if cache_docker:
|
|
82
|
-
input_hash["cache_docker"] = cache_docker
|
|
83
|
-
if nextflow_pipeline_params:
|
|
84
|
-
input_hash["nextflow_pipeline_params"] = nextflow_pipeline_params
|
|
85
|
-
|
|
86
86
|
if build_project_id is None:
|
|
87
87
|
parser.error(
|
|
88
88
|
"Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project")
|
|
89
|
-
|
|
90
89
|
nf_builder_job = dxpy.DXApp(name=get_importer_name()).run(app_input=input_hash, project=build_project_id,
|
|
91
90
|
folder=build_folder,
|
|
92
91
|
name="Nextflow build of %s" % (repository), detach=True)
|
|
@@ -155,3 +155,35 @@ def get_nextflow_assets(region):
|
|
|
155
155
|
|
|
156
156
|
with open(nextaur_assets, 'r') as nextaur_f, open(nextflow_assets, 'r') as nextflow_f, open(awscli_assets, 'r') as awscli_f:
|
|
157
157
|
return json.load(nextaur_f)[region], json.load(nextflow_f)[region], json.load(awscli_f)[region]
|
|
158
|
+
|
|
159
|
+
def get_nested(args, arg_path):
|
|
160
|
+
"""
|
|
161
|
+
:param args: extra args from command input
|
|
162
|
+
:type args: dict
|
|
163
|
+
:param arg_path: list of a dxapp.json location of an allowed extra_arg (eg. ["runSpec", "timeoutPolicy"])
|
|
164
|
+
:type arg_path: tuple/list
|
|
165
|
+
:returns: nested arg value if it exists in args, otherwise None
|
|
166
|
+
"""
|
|
167
|
+
for key in arg_path:
|
|
168
|
+
if not isinstance(args, dict):
|
|
169
|
+
return None
|
|
170
|
+
args = args.get(key)
|
|
171
|
+
if args is None:
|
|
172
|
+
return None
|
|
173
|
+
return args
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def get_allowed_extra_fields_mapping():
|
|
177
|
+
"""
|
|
178
|
+
:returns: tuple (arg_path, target_key)
|
|
179
|
+
arg_path is a list of a dxapp.json location of an allowed extra_arg, target_key is name of an argument for a remote build
|
|
180
|
+
"""
|
|
181
|
+
return [
|
|
182
|
+
(["name"], "name"),
|
|
183
|
+
(["title"], "title"),
|
|
184
|
+
(["summary"], "summary"),
|
|
185
|
+
(["runSpec", "timeoutPolicy"], "timeout_policy"),
|
|
186
|
+
(["runSpec", "release"], "release"),
|
|
187
|
+
(["details", "whatsNew"], "whats_new"),
|
|
188
|
+
]
|
|
189
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
version = '0.391.0'
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: dxpy
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.391.0
|
|
4
4
|
Summary: DNAnexus Platform API bindings for Python
|
|
5
5
|
Home-page: https://github.com/dnanexus/dx-toolkit
|
|
6
6
|
Author: Aleksandra Zalcman, Andrey Kislyuk, Anurag Biyani, Geet Duggal, Katherine Lai, Kurt Jensen, Marek Hrvol, Ohad Rodeh, Phil Sung
|
|
@@ -3,7 +3,8 @@ argcomplete>=1.9.4,<2.0.0; python_version < "3.10"
|
|
|
3
3
|
websocket-client>=1.6.0,<1.8.0
|
|
4
4
|
python-dateutil>=2.5
|
|
5
5
|
psutil>=5.9.3
|
|
6
|
-
certifi
|
|
6
|
+
certifi>=2024.7.4
|
|
7
7
|
urllib3>=1.25,<2.2
|
|
8
8
|
pyreadline3==3.4.1; sys_platform == "win32"
|
|
9
9
|
colorama>=0.4.4,<=0.4.6; sys_platform == "win32"
|
|
10
|
+
crc32c>=2.7.1
|
|
@@ -241,7 +241,7 @@ class TestDXBuildNextflowApplet(DXTestCaseBuildNextflowApps):
|
|
|
241
241
|
pipeline_name, existing_nf_file_path=self.base_nextflow_nf)
|
|
242
242
|
|
|
243
243
|
# Override metadata values
|
|
244
|
-
extra_args = '{"name": "name-9Oxvx2tCZe", "title": "Title VsnhPeFBqt", "summary": "Summary 3E7fFfEXdB"}'
|
|
244
|
+
extra_args = '{"name": "name-9Oxvx2tCZe", "title": "Title VsnhPeFBqt", "summary": "Summary 3E7fFfEXdB", "runSpec": {"release": "20.04"}}'
|
|
245
245
|
applet_id = json.loads(run(
|
|
246
246
|
"dx build --nextflow '{}' --json --extra-args '{}'".format(applet_dir, extra_args)))["id"]
|
|
247
247
|
|
|
@@ -250,6 +250,7 @@ class TestDXBuildNextflowApplet(DXTestCaseBuildNextflowApps):
|
|
|
250
250
|
self.assertEqual(desc["name"], json.loads(extra_args)["name"])
|
|
251
251
|
self.assertEqual(desc["title"], json.loads(extra_args)["title"])
|
|
252
252
|
self.assertEqual(desc["summary"], json.loads(extra_args)["summary"])
|
|
253
|
+
self.assertEqual(desc["runSpec"]["release"], json.loads(extra_args)["runSpec"]["release"])
|
|
253
254
|
|
|
254
255
|
details = applet.get_details()
|
|
255
256
|
self.assertEqual(details["repository"], "local")
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"aws:ap-southeast-2": "record-Ggjx0YQ5qP61v7Q7Qq4QQ2f7",
|
|
3
|
-
"aws:eu-central-1": "record-Ggjx2Jj4jbKyKxGXYfZgpJYy",
|
|
4
|
-
"aws:eu-west-2-g": "record-Ggjx4qBKYFQ35Gp87ybKV28Z",
|
|
5
|
-
"aws:me-south-1": "record-GxFKpy135qFZjpBqPyPV0Z4F",
|
|
6
|
-
"aws:us-east-1": "record-GgjvvP00GFJ5JjG1gX3Jbx6j",
|
|
7
|
-
"azure:westeurope": "record-Ggjx8Z8BxzQqXY9Fv4vXBqpj",
|
|
8
|
-
"azure:westus": "record-GgjxGf89ZB67Bz7f3815qk11"
|
|
9
|
-
}
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"aws:ap-southeast-2": "record-Ggjp4Fj50y9Vp27Fx8gB65fY",
|
|
3
|
-
"aws:eu-central-1": "record-Ggjp62Q4PFG1K0g562PKZgbV",
|
|
4
|
-
"aws:eu-west-2-g": "record-Ggjp7K2K60b11G4p2X4X8B98",
|
|
5
|
-
"aws:me-south-1": "record-GxFJy0V3kVvxjKfZF7p3JbGV",
|
|
6
|
-
"aws:us-east-1": "record-Ggjp2p009010jB06F520vZQG",
|
|
7
|
-
"azure:westeurope": "record-GgjpB2QB7Vp4Kz8FV25Q68Jv",
|
|
8
|
-
"azure:westus": "record-GgjpJ509bvJ1p27Fx8gB65jg"
|
|
9
|
-
}
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"aws:ap-southeast-2": "record-GpzQzk852Z6F46605Q45G2fk",
|
|
3
|
-
"aws:eu-central-1": "record-GpzV15j4Gp5b5GpyJ67GX24x",
|
|
4
|
-
"aws:eu-west-2-g": "record-GpzV2xXKqzvbxk4z6884j8jQ",
|
|
5
|
-
"aws:me-south-1": "record-GxFKf693XYGq0ZbVf1FjpjkP",
|
|
6
|
-
"aws:us-east-1": "record-GpzQxg00Zb6YVFZ1bBJ0QFQY",
|
|
7
|
-
"azure:westeurope": "record-GpzV6kQBGXPfFK2jBq1P5fqK",
|
|
8
|
-
"azure:westus": "record-GpzVFv894pgvz7qyjKY3GJbG"
|
|
9
|
-
}
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"aws:ap-southeast-2": "record-GpyvF1Q5Ky99FPzYYgZKK1VP",
|
|
3
|
-
"aws:eu-central-1": "record-GpyvGVj4gv75gPB1F7jP18kx",
|
|
4
|
-
"aws:eu-west-2-g": "record-GpyvJbBKVkbzYxqfQ409202F",
|
|
5
|
-
"aws:me-south-1": "record-GxFJbvk3jfbfyPj9VjK660qp",
|
|
6
|
-
"aws:us-east-1": "record-Gpyv8jj023vP2x068G1bZ7Zz",
|
|
7
|
-
"azure:westeurope": "record-GpyvQvQB69bFPJZFgY0J3807",
|
|
8
|
-
"azure:westus": "record-GpyvZ3Q9vf40pyGqgb5G7Kvp"
|
|
9
|
-
}
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"aws:ap-southeast-2": "record-GbqXZx85YkJYkXbJGx2712B0",
|
|
3
|
-
"aws:eu-central-1": "record-GbqXbKQ4f06Fb6yFVJ2jf91z",
|
|
4
|
-
"aws:eu-west-2-g": "record-GbqXffpKbPkPGyqxBKj4bkGP",
|
|
5
|
-
"aws:me-south-1": "record-GxFKjXV3v1BFj0jJ8Bz53k5Q",
|
|
6
|
-
"aws:us-east-1": "record-GbqX3YQ0zF96QY6XkKgGj6zJ",
|
|
7
|
-
"azure:westeurope": "record-GbqXq50By9ZZF5JpzzvZVX3F",
|
|
8
|
-
"azure:westus": "record-GbqXy9Q91ggZQg52Z4Ffk84q"
|
|
9
|
-
}
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"aws:ap-southeast-2": "record-GbqVfV85P3j22G98Qk3Pbp54",
|
|
3
|
-
"aws:eu-central-1": "record-GbqVj684y262YXY4gkXP12YK",
|
|
4
|
-
"aws:eu-west-2-g": "record-GbqVkkpKqzF471k20pKZkpZJ",
|
|
5
|
-
"aws:me-south-1": "record-GxFJpgk3gX2Kf655z3KP1x80",
|
|
6
|
-
"aws:us-east-1": "record-GbqVBJ80g9g6vzxj51xYzYj6",
|
|
7
|
-
"azure:westeurope": "record-GbqVv98B1PXF5g7GGYVqfFjx",
|
|
8
|
-
"azure:westus": "record-GbqVz1Q9K7yp85ZVKQVbVgz7"
|
|
9
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
version = '0.389.0'
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{dxpy-0.389.0 → dxpy-0.391.0}/dxpy/bindings/apollo/schemas/input_arguments_validation_schemas.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|