latch 2.40.4.dev4__tar.gz → 2.40.4.dev6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- latch-2.40.4.dev6/PKG-INFO +39 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/account.py +7 -11
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/executions.py +2 -2
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/registry/project.py +7 -11
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/registry/table.py +14 -20
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/resources/dynamic.py +17 -5
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/resources/tasks.py +14 -4
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/verified/deseq2.py +18 -28
- latch-2.40.4.dev6/latch.egg-info/PKG-INFO +39 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch.egg-info/SOURCES.txt +0 -3
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/centromere/utils.py +4 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/common/serialize.py +5 -1
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/nextflow/build.py +34 -9
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/nextflow/channel.py +10 -2
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/nextflow/file_persistence.py +26 -2
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/nextflow/serialize.py +1 -1
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/nextflow/tasks/adapters.py +9 -5
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/nextflow/tasks/base.py +4 -1
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/nextflow/tasks/map.py +7 -1
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/nextflow/tasks/operator.py +12 -2
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/nextflow/tasks/process.py +147 -72
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/main.py +24 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/execute/utils.py +1 -1
- latch-2.40.4.dev6/latch_cli/services/init/__pycache__/__init__.cpython-310.pyc +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/__pycache__/init.cpython-310.pyc +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/register/register.py +2 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/utils/__init__.py +2 -3
- latch-2.40.4.dev6/latch_cli/utils/workflow.py +44 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/setup.py +1 -1
- latch-2.40.4.dev4/PKG-INFO +0 -13
- latch-2.40.4.dev4/latch.egg-info/PKG-INFO +0 -13
- latch-2.40.4.dev4/latch_cli/services/init/__pycache__/__init__.cpython-310.pyc +0 -0
- latch-2.40.4.dev4/latch_cli/services/init/example_docker/.__init__.py.swn +0 -0
- latch-2.40.4.dev4/latch_cli/services/init/example_docker/.__init__.py.swo +0 -0
- latch-2.40.4.dev4/latch_cli/services/init/example_docker/.__init__.py.swp +0 -0
- latch-2.40.4.dev4/latch_cli/utils/workflow.py +0 -84
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/LICENSE +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/MANIFEST.in +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/README.md +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/functions/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/functions/messages.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/functions/operators.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/functions/secrets.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/ldata/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/ldata/_transfer/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/ldata/_transfer/download.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/ldata/_transfer/manager.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/ldata/_transfer/node.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/ldata/_transfer/progress.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/ldata/_transfer/remote_copy.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/ldata/_transfer/throttle.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/ldata/_transfer/upload.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/ldata/_transfer/utils.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/ldata/path.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/ldata/type.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/registry/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/registry/record.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/registry/types.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/registry/upstream_types/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/registry/upstream_types/types.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/registry/upstream_types/values.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/registry/utils.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/resources/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/resources/conditional.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/resources/launch_plan.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/resources/map_tasks.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/resources/reference_workflow.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/resources/workflow.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/types/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/types/directory.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/types/file.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/types/glob.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/types/json.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/types/metadata.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/types/utils.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/utils.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/verified/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/verified/mafft.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/verified/pathway.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/verified/rnaseq.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch/verified/trim_galore.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch.egg-info/dependency_links.txt +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch.egg-info/entry_points.txt +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch.egg-info/requires.txt +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch.egg-info/top_level.txt +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/auth/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/auth/csrf.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/auth/oauth2.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/auth/pkce.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/auth/utils.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/centromere/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/centromere/ctx.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/click_utils.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/constants.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/docker_utils/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/exceptions/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/exceptions/cache.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/exceptions/errors.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/exceptions/handler.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/exceptions/traceback.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/common/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/common/config/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/common/config/parser.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/common/config/utils.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/common/utils.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/nextflow/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/nextflow/config.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/nextflow/dag.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/nextflow/tasks/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/nextflow/tasks/conditional.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/nextflow/tasks/merge.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/nextflow/workflow.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/snakemake/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/snakemake/config.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/snakemake/serialize.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/snakemake/single_task_snakemake.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/snakemake/utils.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/extras/snakemake/workflow.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/menus.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/cp/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/cp/autocomplete.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/cp/glob.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/cp/main.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/cp/utils.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/execute/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/execute/main.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/get.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/get_executions.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/get_params.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/assemble_and_sort/.env +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/assemble_and_sort/LICENSE +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/assemble_and_sort/README.md +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/assemble_and_sort/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/assemble_and_sort/assemble.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/assemble_and_sort/sort.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/assemble_and_sort/system-requirements.txt +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/common/.dockerignore +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/example_conda/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/example_conda/conda_task.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/example_conda/environment.yaml +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/example_docker/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/example_docker/task.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/example_nextflow/main.nf +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/example_nextflow/nextflow.config +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/example_nextflow/workflow.nf +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/example_nfcore/Dockerfile +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/example_nfcore/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/example_nfcore/task.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/example_r/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/example_r/environment.R +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/example_r/r_task.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/example_snakemake/.latch/latch_entrypoint +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/example_snakemake/Dockerfile +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/example_snakemake/Snakefile +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/example_snakemake/config.yaml +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/example_snakemake/environment.yaml +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/example_snakemake/latch_metadata.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/example_snakemake/scripts/plot-quals.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/example_snakemake/version +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/init.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/template/LICENSE +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/template/README.md +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/template/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/init/template/task.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/launch.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/local_dev.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/local_dev_old.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/login.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/ls.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/mkdir.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/move.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/preview.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/register/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/register/constants.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/register/utils.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/rm.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/stop_pod.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/sync.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/test_data/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/test_data/ls.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/test_data/remove.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/test_data/upload.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/test_data/utils.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/services/workspace.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/tinyrequests.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/tui/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/utils/path.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/latch_cli/workflow_config.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/pyproject.toml +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/setup.cfg +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/tests/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/tests/cp/__init__.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/tests/fixtures.py +0 -0
- {latch-2.40.4.dev4 → latch-2.40.4.dev6}/tests/test_ls.py +0 -0
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: latch
|
|
3
|
+
Version: 2.40.4.dev6
|
|
4
|
+
Summary: The Latch SDK
|
|
5
|
+
Author-email: kenny@latch.bio
|
|
6
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
7
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
8
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
10
|
+
Requires-Python: >=3.8,<3.12
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Requires-Dist: kubernetes>=24.2.0
|
|
13
|
+
Requires-Dist: pyjwt>=0.2.0
|
|
14
|
+
Requires-Dist: requests>=2.28.1
|
|
15
|
+
Requires-Dist: click>=8.0
|
|
16
|
+
Requires-Dist: docker>=5.0
|
|
17
|
+
Requires-Dist: paramiko>=3.2.0
|
|
18
|
+
Requires-Dist: scp>=0.14.0
|
|
19
|
+
Requires-Dist: boto3>=1.26.0
|
|
20
|
+
Requires-Dist: tqdm>=4.63.0
|
|
21
|
+
Requires-Dist: lytekit==0.15.6
|
|
22
|
+
Requires-Dist: lytekitplugins-pods==0.6.2
|
|
23
|
+
Requires-Dist: typing-extensions==4.7.1
|
|
24
|
+
Requires-Dist: apscheduler==3.9.1
|
|
25
|
+
Requires-Dist: gql==3.4.0
|
|
26
|
+
Requires-Dist: graphql-core==3.2.3
|
|
27
|
+
Requires-Dist: requests-toolbelt==0.10.1
|
|
28
|
+
Requires-Dist: latch-sdk-gql==0.0.6
|
|
29
|
+
Requires-Dist: latch-sdk-config==0.0.4
|
|
30
|
+
Requires-Dist: python-dateutil>=2.8
|
|
31
|
+
Requires-Dist: aioconsole==0.6.1
|
|
32
|
+
Requires-Dist: asyncssh==2.13.2
|
|
33
|
+
Requires-Dist: websockets==11.0.3
|
|
34
|
+
Requires-Dist: watchfiles==0.19.0
|
|
35
|
+
Provides-Extra: snakemake
|
|
36
|
+
Requires-Dist: snakemake<7.30.2,>=7.18.0; extra == "snakemake"
|
|
37
|
+
Requires-Dist: pulp<2.8,>=2.0; extra == "snakemake"
|
|
38
|
+
Provides-Extra: pandas
|
|
39
|
+
Requires-Dist: pandas>=2.0.0; extra == "pandas"
|
|
@@ -258,12 +258,10 @@ class AccountUpdate:
|
|
|
258
258
|
|
|
259
259
|
args = l.ArgumentNode()
|
|
260
260
|
args.name = _name_node("input")
|
|
261
|
-
args.value = _json_value(
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
}
|
|
266
|
-
)
|
|
261
|
+
args.value = _json_value({
|
|
262
|
+
"argOwnerId": self.account.id,
|
|
263
|
+
"argDisplayNames": display_names,
|
|
264
|
+
})
|
|
267
265
|
|
|
268
266
|
res.alias = _name_node(f"upd{len(mutations)}")
|
|
269
267
|
res.arguments = tuple([args])
|
|
@@ -299,11 +297,9 @@ class AccountUpdate:
|
|
|
299
297
|
|
|
300
298
|
args = l.ArgumentNode()
|
|
301
299
|
args.name = _name_node("input")
|
|
302
|
-
args.value = _json_value(
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
}
|
|
306
|
-
)
|
|
300
|
+
args.value = _json_value({
|
|
301
|
+
"argIds": ids,
|
|
302
|
+
})
|
|
307
303
|
|
|
308
304
|
res.alias = _name_node(f"upd{len(mutations)}")
|
|
309
305
|
res.arguments = tuple([args])
|
|
@@ -228,12 +228,10 @@ class ProjectUpdate:
|
|
|
228
228
|
|
|
229
229
|
args = l.ArgumentNode()
|
|
230
230
|
args.name = _name_node("input")
|
|
231
|
-
args.value = _json_value(
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
}
|
|
236
|
-
)
|
|
231
|
+
args.value = _json_value({
|
|
232
|
+
"argProjectId": self.project.id,
|
|
233
|
+
"argDisplayNames": display_names,
|
|
234
|
+
})
|
|
237
235
|
|
|
238
236
|
res.alias = _name_node(f"upd{len(mutations)}")
|
|
239
237
|
res.arguments = tuple([args])
|
|
@@ -267,11 +265,9 @@ class ProjectUpdate:
|
|
|
267
265
|
|
|
268
266
|
args = l.ArgumentNode()
|
|
269
267
|
args.name = _name_node("input")
|
|
270
|
-
args.value = _json_value(
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
}
|
|
274
|
-
)
|
|
268
|
+
args.value = _json_value({
|
|
269
|
+
"argIds": ids,
|
|
270
|
+
})
|
|
275
271
|
|
|
276
272
|
res.alias = _name_node(f"upd{len(mutations)}")
|
|
277
273
|
res.arguments = tuple([args])
|
|
@@ -496,13 +496,11 @@ class TableUpdate:
|
|
|
496
496
|
|
|
497
497
|
args = l.ArgumentNode()
|
|
498
498
|
args.name = _name_node("input")
|
|
499
|
-
args.value = _json_value(
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
}
|
|
505
|
-
)
|
|
499
|
+
args.value = _json_value({
|
|
500
|
+
"argExperimentId": self.table.id,
|
|
501
|
+
"argNames": names,
|
|
502
|
+
"argData": _var_node(argDataVar),
|
|
503
|
+
})
|
|
506
504
|
|
|
507
505
|
res.alias = _name_node(f"upd{len(mutations)}")
|
|
508
506
|
res.arguments = tuple([args])
|
|
@@ -537,12 +535,10 @@ class TableUpdate:
|
|
|
537
535
|
|
|
538
536
|
args = l.ArgumentNode()
|
|
539
537
|
args.name = _name_node("input")
|
|
540
|
-
args.value = _json_value(
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
}
|
|
545
|
-
)
|
|
538
|
+
args.value = _json_value({
|
|
539
|
+
"argExperimentId": self.table.id,
|
|
540
|
+
"argNames": names,
|
|
541
|
+
})
|
|
546
542
|
|
|
547
543
|
res.alias = _name_node(f"upd{len(mutations)}")
|
|
548
544
|
res.arguments = tuple([args])
|
|
@@ -690,13 +686,11 @@ class TableUpdate:
|
|
|
690
686
|
|
|
691
687
|
args = l.ArgumentNode()
|
|
692
688
|
args.name = _name_node("input")
|
|
693
|
-
args.value = _json_value(
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
}
|
|
699
|
-
)
|
|
689
|
+
args.value = _json_value({
|
|
690
|
+
"argExperimentId": self.table.id,
|
|
691
|
+
"argKeys": keys,
|
|
692
|
+
"argTypes": _var_node(argTypesVar),
|
|
693
|
+
})
|
|
700
694
|
|
|
701
695
|
res.alias = _name_node(f"upd{len(mutations)}")
|
|
702
696
|
res.arguments = tuple([args])
|
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import json
|
|
2
1
|
import os
|
|
3
2
|
from dataclasses import dataclass
|
|
4
|
-
from typing import Any, Callable, Dict, Union
|
|
3
|
+
from typing import Any, Callable, Dict, Optional, Union
|
|
5
4
|
|
|
6
5
|
import gql
|
|
7
6
|
from flytekit.configuration import SerializationSettings
|
|
@@ -63,9 +62,15 @@ def _override_task_resources(task_config: Pod) -> None:
|
|
|
63
62
|
|
|
64
63
|
|
|
65
64
|
def _dynamic_resource_task(
|
|
66
|
-
|
|
65
|
+
dynamic_func: Optional[Callable],
|
|
66
|
+
cpu: Union[int, Callable],
|
|
67
|
+
memory: Union[int, Callable],
|
|
68
|
+
disk: Union[int, Callable],
|
|
67
69
|
):
|
|
68
70
|
def f(**kwargs):
|
|
71
|
+
if dynamic_func is not None:
|
|
72
|
+
dynamic_func(**kwargs)
|
|
73
|
+
|
|
69
74
|
res: Dict[str, int] = {
|
|
70
75
|
"cpu": cpu(**kwargs) if callable(cpu) else cpu,
|
|
71
76
|
"memory": memory(**kwargs) if callable(memory) else memory,
|
|
@@ -85,6 +90,7 @@ def _dynamic_resource_task(
|
|
|
85
90
|
|
|
86
91
|
@dataclass(frozen=True)
|
|
87
92
|
class DynamicTaskConfig:
|
|
93
|
+
dynamic_func: Optional[Callable]
|
|
88
94
|
cpu: Union[Callable, int]
|
|
89
95
|
memory: Union[Callable, int]
|
|
90
96
|
storage: Union[Callable, int]
|
|
@@ -93,7 +99,10 @@ class DynamicTaskConfig:
|
|
|
93
99
|
|
|
94
100
|
class DynamicPythonFunctionTask(PodFunctionTask):
|
|
95
101
|
def __init__(
|
|
96
|
-
self,
|
|
102
|
+
self,
|
|
103
|
+
task_config: DynamicTaskConfig,
|
|
104
|
+
task_function: Callable,
|
|
105
|
+
**kwargs,
|
|
97
106
|
):
|
|
98
107
|
# validate that the task function inputs are the same as the resource functions
|
|
99
108
|
for resource in [task_config.cpu, task_config.memory, task_config.storage]:
|
|
@@ -113,7 +122,10 @@ class DynamicPythonFunctionTask(PodFunctionTask):
|
|
|
113
122
|
)
|
|
114
123
|
|
|
115
124
|
self._pre_task_function = _dynamic_resource_task(
|
|
116
|
-
task_config.
|
|
125
|
+
task_config.dynamic_func,
|
|
126
|
+
task_config.cpu,
|
|
127
|
+
task_config.memory,
|
|
128
|
+
task_config.storage,
|
|
117
129
|
)
|
|
118
130
|
|
|
119
131
|
super().__init__(
|
|
@@ -26,7 +26,7 @@ exported decorators.
|
|
|
26
26
|
|
|
27
27
|
import datetime
|
|
28
28
|
import functools
|
|
29
|
-
from typing import Callable, Union
|
|
29
|
+
from typing import Callable, Optional, Union
|
|
30
30
|
from warnings import warn
|
|
31
31
|
|
|
32
32
|
from flytekit import task
|
|
@@ -470,6 +470,9 @@ def custom_task(
|
|
|
470
470
|
*,
|
|
471
471
|
storage_gib: Union[Callable, int] = 500,
|
|
472
472
|
timeout: Union[datetime.timedelta, int] = 0,
|
|
473
|
+
cache: bool = False,
|
|
474
|
+
retries: int = 0,
|
|
475
|
+
pre_execute: Optional[Callable] = None,
|
|
473
476
|
):
|
|
474
477
|
"""Returns a custom task configuration requesting
|
|
475
478
|
the specified CPU/RAM allocations
|
|
@@ -481,13 +484,20 @@ def custom_task(
|
|
|
481
484
|
"""
|
|
482
485
|
if callable(cpu) or callable(memory) or callable(storage_gib):
|
|
483
486
|
task_config = DynamicTaskConfig(
|
|
487
|
+
dynamic_func=pre_execute,
|
|
484
488
|
cpu=cpu,
|
|
485
489
|
memory=memory,
|
|
486
490
|
storage=storage_gib,
|
|
487
|
-
pod_config=
|
|
491
|
+
pod_config=_custom_task_config(1, 1, 50),
|
|
492
|
+
)
|
|
493
|
+
return functools.partial(
|
|
494
|
+
task, task_config=task_config, timeout=timeout, cache=cache, retries=retries
|
|
488
495
|
)
|
|
489
|
-
return functools.partial(task, task_config=task_config, timeout=timeout)
|
|
490
496
|
|
|
491
497
|
return functools.partial(
|
|
492
|
-
task,
|
|
498
|
+
task,
|
|
499
|
+
task_config=_custom_task_config(cpu, memory, storage_gib),
|
|
500
|
+
timeout=timeout,
|
|
501
|
+
cache=cache,
|
|
502
|
+
retries=retries,
|
|
493
503
|
)
|
|
@@ -19,17 +19,13 @@ def deseq2_wf(
|
|
|
19
19
|
raw_count_table: Optional[
|
|
20
20
|
Annotated[
|
|
21
21
|
LatchFile,
|
|
22
|
-
FlyteAnnotation(
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
"
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
}
|
|
30
|
-
],
|
|
31
|
-
}
|
|
32
|
-
),
|
|
22
|
+
FlyteAnnotation({
|
|
23
|
+
"_tmp_hack_deseq2": "counts_table",
|
|
24
|
+
"rules": [{
|
|
25
|
+
"regex": r".*\.(csv|tsv|xlsx)$",
|
|
26
|
+
"message": "Expected a CSV, TSV, or XLSX file",
|
|
27
|
+
}],
|
|
28
|
+
}),
|
|
33
29
|
]
|
|
34
30
|
] = None,
|
|
35
31
|
raw_count_tables: List[LatchFile] = [],
|
|
@@ -44,17 +40,13 @@ def deseq2_wf(
|
|
|
44
40
|
conditions_table: Optional[
|
|
45
41
|
Annotated[
|
|
46
42
|
LatchFile,
|
|
47
|
-
FlyteAnnotation(
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
"
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
}
|
|
55
|
-
],
|
|
56
|
-
}
|
|
57
|
-
),
|
|
43
|
+
FlyteAnnotation({
|
|
44
|
+
"_tmp_hack_deseq2": "design_matrix",
|
|
45
|
+
"rules": [{
|
|
46
|
+
"regex": r".*\.(csv|tsv|xlsx)$",
|
|
47
|
+
"message": "Expected a CSV, TSV, or XLSX file",
|
|
48
|
+
}],
|
|
49
|
+
}),
|
|
58
50
|
]
|
|
59
51
|
] = None,
|
|
60
52
|
design_matrix_sample_id_column: Optional[
|
|
@@ -62,12 +54,10 @@ def deseq2_wf(
|
|
|
62
54
|
] = None,
|
|
63
55
|
design_formula: Annotated[
|
|
64
56
|
List[List[str]],
|
|
65
|
-
FlyteAnnotation(
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
}
|
|
70
|
-
),
|
|
57
|
+
FlyteAnnotation({
|
|
58
|
+
"_tmp_hack_deseq2": "design_formula",
|
|
59
|
+
"_tmp_hack_deseq2_allow_clustering": True,
|
|
60
|
+
}),
|
|
71
61
|
] = [],
|
|
72
62
|
number_of_genes_to_plot: int = 30,
|
|
73
63
|
) -> LatchDir: ...
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: latch
|
|
3
|
+
Version: 2.40.4.dev6
|
|
4
|
+
Summary: The Latch SDK
|
|
5
|
+
Author-email: kenny@latch.bio
|
|
6
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
7
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
8
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
10
|
+
Requires-Python: >=3.8,<3.12
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Requires-Dist: kubernetes>=24.2.0
|
|
13
|
+
Requires-Dist: pyjwt>=0.2.0
|
|
14
|
+
Requires-Dist: requests>=2.28.1
|
|
15
|
+
Requires-Dist: click>=8.0
|
|
16
|
+
Requires-Dist: docker>=5.0
|
|
17
|
+
Requires-Dist: paramiko>=3.2.0
|
|
18
|
+
Requires-Dist: scp>=0.14.0
|
|
19
|
+
Requires-Dist: boto3>=1.26.0
|
|
20
|
+
Requires-Dist: tqdm>=4.63.0
|
|
21
|
+
Requires-Dist: lytekit==0.15.6
|
|
22
|
+
Requires-Dist: lytekitplugins-pods==0.6.2
|
|
23
|
+
Requires-Dist: typing-extensions==4.7.1
|
|
24
|
+
Requires-Dist: apscheduler==3.9.1
|
|
25
|
+
Requires-Dist: gql==3.4.0
|
|
26
|
+
Requires-Dist: graphql-core==3.2.3
|
|
27
|
+
Requires-Dist: requests-toolbelt==0.10.1
|
|
28
|
+
Requires-Dist: latch-sdk-gql==0.0.6
|
|
29
|
+
Requires-Dist: latch-sdk-config==0.0.4
|
|
30
|
+
Requires-Dist: python-dateutil>=2.8
|
|
31
|
+
Requires-Dist: aioconsole==0.6.1
|
|
32
|
+
Requires-Dist: asyncssh==2.13.2
|
|
33
|
+
Requires-Dist: websockets==11.0.3
|
|
34
|
+
Requires-Dist: watchfiles==0.19.0
|
|
35
|
+
Provides-Extra: snakemake
|
|
36
|
+
Requires-Dist: snakemake<7.30.2,>=7.18.0; extra == "snakemake"
|
|
37
|
+
Requires-Dist: pulp<2.8,>=2.0; extra == "snakemake"
|
|
38
|
+
Provides-Extra: pandas
|
|
39
|
+
Requires-Dist: pandas>=2.0.0; extra == "pandas"
|
|
@@ -148,9 +148,6 @@ latch_cli/services/init/common/.dockerignore
|
|
|
148
148
|
latch_cli/services/init/example_conda/__init__.py
|
|
149
149
|
latch_cli/services/init/example_conda/conda_task.py
|
|
150
150
|
latch_cli/services/init/example_conda/environment.yaml
|
|
151
|
-
latch_cli/services/init/example_docker/.__init__.py.swn
|
|
152
|
-
latch_cli/services/init/example_docker/.__init__.py.swo
|
|
153
|
-
latch_cli/services/init/example_docker/.__init__.py.swp
|
|
154
151
|
latch_cli/services/init/example_docker/__init__.py
|
|
155
152
|
latch_cli/services/init/example_docker/task.py
|
|
156
153
|
latch_cli/services/init/example_nextflow/main.nf
|
|
@@ -2,6 +2,7 @@ import builtins
|
|
|
2
2
|
import contextlib
|
|
3
3
|
import os
|
|
4
4
|
import random
|
|
5
|
+
import shutil
|
|
5
6
|
import string
|
|
6
7
|
import sys
|
|
7
8
|
import tempfile
|
|
@@ -261,6 +262,9 @@ class MaybeRemoteDir:
|
|
|
261
262
|
return
|
|
262
263
|
|
|
263
264
|
if self._tempdir is not None and not isinstance(self._tempdir, str):
|
|
265
|
+
if os.environ.get("LATCH_SAVE_PROTO") is not None:
|
|
266
|
+
shutil.copytree(Path(self._tempdir.name), Path.cwd() / "protos")
|
|
267
|
+
|
|
264
268
|
self._tempdir.cleanup()
|
|
265
269
|
|
|
266
270
|
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import os
|
|
1
2
|
import sys
|
|
2
3
|
from dataclasses import fields
|
|
3
4
|
from pathlib import Path
|
|
@@ -302,8 +303,11 @@ def serialize(
|
|
|
302
303
|
image_name: str,
|
|
303
304
|
dkr_repo: str,
|
|
304
305
|
*,
|
|
305
|
-
write_spec: bool =
|
|
306
|
+
write_spec: Optional[bool] = None,
|
|
306
307
|
):
|
|
308
|
+
if write_spec is None:
|
|
309
|
+
write_spec = os.environ.get("LATCH_DUMP_SPEC") is not None
|
|
310
|
+
|
|
307
311
|
image_name_no_version, version = image_name.split(":")
|
|
308
312
|
default_img = Image(
|
|
309
313
|
name=image_name,
|
|
@@ -29,7 +29,7 @@ from latch_cli.extras.nextflow.tasks.base import NextflowBaseTask, NFTaskType
|
|
|
29
29
|
|
|
30
30
|
from ...click_utils import italic
|
|
31
31
|
from ...menus import select_tui
|
|
32
|
-
from ...utils import
|
|
32
|
+
from ...utils import identifier_from_str
|
|
33
33
|
from ..common.serialize import binding_from_python
|
|
34
34
|
from ..common.utils import is_samplesheet_param, reindent
|
|
35
35
|
from .dag import DAG, VertexType
|
|
@@ -50,7 +50,10 @@ def get_node_name(vertex_id: str) -> str:
|
|
|
50
50
|
|
|
51
51
|
|
|
52
52
|
def build_from_nextflow_dag(
|
|
53
|
-
wf: NextflowWorkflow,
|
|
53
|
+
wf: NextflowWorkflow,
|
|
54
|
+
*,
|
|
55
|
+
execution_profile: Optional[str] = None,
|
|
56
|
+
ephemeral_storage_gib: int = 500,
|
|
54
57
|
):
|
|
55
58
|
global_start_node = Node(
|
|
56
59
|
id=_common_constants.GLOBAL_INPUT_NODE_ID,
|
|
@@ -105,18 +108,22 @@ def build_from_nextflow_dag(
|
|
|
105
108
|
else:
|
|
106
109
|
input_name = f"channel_{dep.id}"
|
|
107
110
|
|
|
108
|
-
|
|
111
|
+
dep_output_name = "res"
|
|
109
112
|
if len(dep.outputNames) > 0:
|
|
110
113
|
idx = int(edge.label or "0")
|
|
111
114
|
input_name = f"{input_name}_{idx}"
|
|
112
|
-
|
|
115
|
+
dep_output_name = dep.outputNames[idx]
|
|
113
116
|
|
|
114
117
|
if vertex.type == VertexType.Merge:
|
|
115
|
-
|
|
118
|
+
vertex_output_name = "res"
|
|
119
|
+
if len(vertex.outputNames) > 0:
|
|
120
|
+
vertex_output_name = dep_output_name
|
|
121
|
+
|
|
122
|
+
merge_sources[vertex_output_name].append(input_name)
|
|
116
123
|
|
|
117
124
|
task_inputs[input_name] = Optional[str]
|
|
118
125
|
|
|
119
|
-
node = NodeOutput(node=node_map[dep.id], var=
|
|
126
|
+
node = NodeOutput(node=node_map[dep.id], var=dep_output_name)
|
|
120
127
|
|
|
121
128
|
task_bindings.append(
|
|
122
129
|
literals_models.Binding(
|
|
@@ -177,6 +184,7 @@ def build_from_nextflow_dag(
|
|
|
177
184
|
wf=wf,
|
|
178
185
|
cpu=vertex.cpu,
|
|
179
186
|
memory=vertex.memoryBytes,
|
|
187
|
+
storage_gib=ephemeral_storage_gib,
|
|
180
188
|
)
|
|
181
189
|
|
|
182
190
|
wf.nextflow_tasks.append(process_task)
|
|
@@ -220,9 +228,21 @@ def build_from_nextflow_dag(
|
|
|
220
228
|
),
|
|
221
229
|
).ref
|
|
222
230
|
),
|
|
223
|
-
)
|
|
231
|
+
),
|
|
232
|
+
literals_models.Binding(
|
|
233
|
+
var="is_skipped",
|
|
234
|
+
binding=literals_models.BindingData(
|
|
235
|
+
promise=Promise(
|
|
236
|
+
var="is_skipped",
|
|
237
|
+
val=NodeOutput(
|
|
238
|
+
node=pre_adapter_node,
|
|
239
|
+
var="is_skipped",
|
|
240
|
+
),
|
|
241
|
+
).ref
|
|
242
|
+
),
|
|
243
|
+
),
|
|
224
244
|
],
|
|
225
|
-
upstream_nodes=[mapped_process_node],
|
|
245
|
+
upstream_nodes=[mapped_process_node, pre_adapter_node],
|
|
226
246
|
flyte_entity=post_adapter_task,
|
|
227
247
|
)
|
|
228
248
|
|
|
@@ -396,6 +416,7 @@ def build_nf_wf(
|
|
|
396
416
|
*,
|
|
397
417
|
redownload_dependencies: bool = False,
|
|
398
418
|
execution_profile: Optional[str] = None,
|
|
419
|
+
ephemeral_storage_gib: int = 500,
|
|
399
420
|
) -> NextflowWorkflow:
|
|
400
421
|
ensure_nf_dependencies(pkg_root, force_redownload=redownload_dependencies)
|
|
401
422
|
|
|
@@ -474,7 +495,11 @@ def build_nf_wf(
|
|
|
474
495
|
|
|
475
496
|
wf = NextflowWorkflow(pkg_root, nf_script, version, main_dag)
|
|
476
497
|
|
|
477
|
-
build_from_nextflow_dag(
|
|
498
|
+
build_from_nextflow_dag(
|
|
499
|
+
wf,
|
|
500
|
+
execution_profile=execution_profile,
|
|
501
|
+
ephemeral_storage_gib=ephemeral_storage_gib,
|
|
502
|
+
)
|
|
478
503
|
|
|
479
504
|
return wf
|
|
480
505
|
|
|
@@ -64,8 +64,16 @@ def get_mapper_inputs(
|
|
|
64
64
|
NT = TypeVar("NT", bound=NamedTuple)
|
|
65
65
|
|
|
66
66
|
|
|
67
|
-
def get_mapper_outputs(
|
|
68
|
-
|
|
67
|
+
def get_mapper_outputs(
|
|
68
|
+
adapter_output_cls: Type[NT], mapper_outputs: List[DC], is_skipped: bool
|
|
69
|
+
) -> NT:
|
|
70
|
+
|
|
71
|
+
kwargs: Dict[str, Optional[str]] = {}
|
|
72
|
+
if is_skipped:
|
|
73
|
+
for f in adapter_output_cls._fields:
|
|
74
|
+
kwargs[f] = None
|
|
75
|
+
|
|
76
|
+
return adapter_output_cls(**kwargs)
|
|
69
77
|
|
|
70
78
|
if len(mapper_outputs) == 1:
|
|
71
79
|
output = mapper_outputs[0]
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
+
import json
|
|
1
2
|
import os
|
|
2
|
-
import shutil
|
|
3
3
|
from dataclasses import dataclass
|
|
4
4
|
from pathlib import Path
|
|
5
|
-
from typing import Dict, List, Optional, Set, Union
|
|
5
|
+
from typing import Dict, List, Optional, Set, Union, cast
|
|
6
6
|
from urllib.parse import urlparse
|
|
7
7
|
|
|
8
8
|
import click
|
|
@@ -17,6 +17,7 @@ from latch.ldata.type import LDataNodeType
|
|
|
17
17
|
from latch.types.directory import LatchDir
|
|
18
18
|
from latch.types.file import LatchFile
|
|
19
19
|
from latch_cli.utils import urljoins
|
|
20
|
+
from latch_cli.utils.path import is_remote_path
|
|
20
21
|
|
|
21
22
|
JSONValue: TypeAlias = Union[
|
|
22
23
|
int,
|
|
@@ -76,7 +77,17 @@ def _extract_paths(parameter: JSONValue, res: List[PathData]):
|
|
|
76
77
|
return
|
|
77
78
|
|
|
78
79
|
res.append(PathData(parameter=parameter, local=local, remote=remote))
|
|
80
|
+
elif "string" in parameter:
|
|
81
|
+
v = parameter["string"]
|
|
82
|
+
assert isinstance(v, str)
|
|
83
|
+
|
|
84
|
+
if not is_remote_path(v):
|
|
85
|
+
return
|
|
79
86
|
|
|
87
|
+
parameter["path"] = v
|
|
88
|
+
del parameter["string"]
|
|
89
|
+
|
|
90
|
+
_extract_paths(parameter, res)
|
|
80
91
|
elif "list" in parameter:
|
|
81
92
|
v = parameter["list"]
|
|
82
93
|
assert isinstance(v, List)
|
|
@@ -230,5 +241,18 @@ def upload_files(channels: Dict[str, List[JSONValue]], outdir: LatchDir):
|
|
|
230
241
|
local_to_remote[local] = remote
|
|
231
242
|
data.parameter["remote"] = remote
|
|
232
243
|
|
|
244
|
+
published_files: List[str] = []
|
|
245
|
+
try:
|
|
246
|
+
with open(".latch/published.json", "r") as f:
|
|
247
|
+
data = f.read()
|
|
248
|
+
conf = json.loads(data)
|
|
249
|
+
published_files = conf.get("files", [])
|
|
250
|
+
except FileNotFoundError:
|
|
251
|
+
pass
|
|
252
|
+
|
|
253
|
+
for file in published_files:
|
|
254
|
+
relative_path = Path(file).relative_to(Path.home())
|
|
255
|
+
local_to_remote[file] = urljoins(remote_parent, str(relative_path))
|
|
256
|
+
|
|
233
257
|
for local, remote in local_to_remote.items():
|
|
234
258
|
_upload(local, remote)
|
|
@@ -3,10 +3,10 @@ from typing import cast
|
|
|
3
3
|
import gql
|
|
4
4
|
from latch_sdk_gql.execute import execute
|
|
5
5
|
|
|
6
|
+
from latch.utils import current_workspace
|
|
6
7
|
from latch_cli.extras.common.serialize import serialize
|
|
7
8
|
from latch_cli.extras.nextflow.tasks.base import NextflowBaseTask
|
|
8
9
|
from latch_cli.extras.nextflow.workflow import NextflowWorkflow
|
|
9
|
-
from latch_cli.utils import current_workspace
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
def serialize_nf(
|
|
@@ -66,7 +66,7 @@ class NextflowProcessPreAdapterTask(NextflowBaseTask):
|
|
|
66
66
|
|
|
67
67
|
super().__init__(
|
|
68
68
|
inputs,
|
|
69
|
-
{"default": List[self.dataclass]},
|
|
69
|
+
{"default": List[self.dataclass], "is_skipped": bool},
|
|
70
70
|
id,
|
|
71
71
|
name,
|
|
72
72
|
branches,
|
|
@@ -96,6 +96,7 @@ class NextflowProcessPreAdapterTask(NextflowBaseTask):
|
|
|
96
96
|
|
|
97
97
|
class Res_{self.id}(NamedTuple):
|
|
98
98
|
default: {type_repr(output_typ)}
|
|
99
|
+
is_skipped: bool
|
|
99
100
|
|
|
100
101
|
""",
|
|
101
102
|
0,
|
|
@@ -114,7 +115,9 @@ class NextflowProcessPreAdapterTask(NextflowBaseTask):
|
|
|
114
115
|
return res
|
|
115
116
|
|
|
116
117
|
def get_fn_return_stmt(self):
|
|
117
|
-
return reindent(
|
|
118
|
+
return reindent(
|
|
119
|
+
f"return Res_{self.id}(default=result, is_skipped = not cond)", 1
|
|
120
|
+
)
|
|
118
121
|
|
|
119
122
|
def get_fn_code(self, nf_script_path_in_container: Path):
|
|
120
123
|
code_block = self.get_fn_interface()
|
|
@@ -173,7 +176,7 @@ class NextflowProcessPostAdapterTask(NextflowBaseTask):
|
|
|
173
176
|
self.dataclass = dataclass_from_python_params(outputs, id)
|
|
174
177
|
|
|
175
178
|
super().__init__(
|
|
176
|
-
{"default": List[self.dataclass]},
|
|
179
|
+
{"default": List[self.dataclass], "is_skipped": bool},
|
|
177
180
|
outputs,
|
|
178
181
|
id,
|
|
179
182
|
name,
|
|
@@ -210,7 +213,8 @@ class NextflowProcessPostAdapterTask(NextflowBaseTask):
|
|
|
210
213
|
rf"""
|
|
211
214
|
@task(cache=True)
|
|
212
215
|
def {self.name}(
|
|
213
|
-
default: List[Dataclass_{self.id}]
|
|
216
|
+
default: List[Dataclass_{self.id}],
|
|
217
|
+
is_skipped: bool,
|
|
214
218
|
) -> Res{self.name}:
|
|
215
219
|
""",
|
|
216
220
|
0,
|
|
@@ -221,7 +225,7 @@ class NextflowProcessPostAdapterTask(NextflowBaseTask):
|
|
|
221
225
|
def get_fn_return_stmt(self):
|
|
222
226
|
return reindent(
|
|
223
227
|
rf"""
|
|
224
|
-
return get_mapper_outputs(Res{self.name}, default)
|
|
228
|
+
return get_mapper_outputs(Res{self.name}, default, is_skipped)
|
|
225
229
|
""",
|
|
226
230
|
1,
|
|
227
231
|
)
|