eva-sub-cli 0.4__tar.gz → 0.4.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. eva_sub_cli-0.4.6/.github/workflows/publish.yml +51 -0
  2. eva_sub_cli-0.4.6/PKG-INFO +213 -0
  3. eva_sub_cli-0.4.6/README.md +184 -0
  4. eva_sub_cli-0.4.6/conda/build.sh +20 -0
  5. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/conda/meta.yaml +17 -11
  6. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/docker/Dockerfile +1 -1
  7. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/__init__.py +1 -0
  8. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/_version.py +9 -4
  9. eva_sub_cli-0.4.6/eva_sub_cli/etc/EVA_Submission_Example.xlsx +0 -0
  10. eva_sub_cli-0.4.6/eva_sub_cli/etc/EVA_Submission_template.xlsx +0 -0
  11. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/etc/spreadsheet2json_conf.yaml +3 -2
  12. eva_sub_cli-0.4.6/eva_sub_cli/exceptions/invalid_file_type_exception.py +4 -0
  13. eva_sub_cli-0.4.6/eva_sub_cli/exceptions/metadata_template_version_exception.py +10 -0
  14. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/executables/cli.py +37 -6
  15. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/file_utils.py +45 -0
  16. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/jinja_templates/html_report.html +4 -0
  17. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/nextflow/validation.nf +1 -2
  18. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/orchestrator.py +120 -14
  19. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/semantic_metadata.py +13 -0
  20. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/submit.py +13 -0
  21. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/validators/docker_validator.py +41 -43
  22. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/validators/native_validator.py +11 -9
  23. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/validators/validation_results_parsers.py +10 -3
  24. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/validators/validator.py +49 -22
  25. eva_sub_cli-0.4.6/eva_sub_cli.egg-info/PKG-INFO +213 -0
  26. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli.egg-info/SOURCES.txt +6 -1
  27. eva_sub_cli-0.4.6/eva_sub_cli.egg-info/requires.txt +6 -0
  28. eva_sub_cli-0.4.6/requirements.txt +6 -0
  29. eva_sub_cli-0.4.6/tests/resources/EVA_Submission_test.xlsx +0 -0
  30. eva_sub_cli-0.4.6/tests/resources/EVA_Submission_test_fails.xlsx +0 -0
  31. eva_sub_cli-0.4.6/tests/resources/EVA_Submission_test_version_missing.xlsx +0 -0
  32. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/test_cli.py +19 -4
  33. eva_sub_cli-0.4.6/tests/test_docker_validator.py +207 -0
  34. eva_sub_cli-0.4.6/tests/test_file_utils.py +82 -0
  35. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/test_orchestrator.py +105 -30
  36. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/test_report.py +6 -1
  37. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/test_semantic_metadata.py +22 -2
  38. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/test_submit.py +26 -2
  39. eva_sub_cli-0.4.6/tests/test_validaton_results_parsers.py +44 -0
  40. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/test_validator.py +6 -6
  41. eva_sub_cli-0.4/PKG-INFO +0 -169
  42. eva_sub_cli-0.4/README.md +0 -140
  43. eva_sub_cli-0.4/conda/build.sh +0 -48
  44. eva_sub_cli-0.4/eva_sub_cli/etc/EVA_Submission_template.xlsx +0 -0
  45. eva_sub_cli-0.4/eva_sub_cli.egg-info/PKG-INFO +0 -169
  46. eva_sub_cli-0.4/eva_sub_cli.egg-info/requires.txt +0 -7
  47. eva_sub_cli-0.4/requirements.txt +0 -7
  48. eva_sub_cli-0.4/tests/resources/EVA_Submission_test.xlsx +0 -0
  49. eva_sub_cli-0.4/tests/resources/EVA_Submission_test_fails.xlsx +0 -0
  50. eva_sub_cli-0.4/tests/test_docker_validator.py +0 -151
  51. eva_sub_cli-0.4/tests/test_file_utils.py +0 -46
  52. eva_sub_cli-0.4/tests/test_validaton_results_parsers.py +0 -32
  53. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/.github/workflows/publish_packages.yml +0 -0
  54. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/.github/workflows/tests.yml +0 -0
  55. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/.gitignore +0 -0
  56. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/LICENSE +0 -0
  57. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6/docs}/Getting_Started_with_eva_sub_cli.md +0 -0
  58. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/auth.py +0 -0
  59. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/date_utils.py +0 -0
  60. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/etc/eva-biosamples.json +0 -0
  61. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/etc/eva_logo.png +0 -0
  62. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/etc/eva_schema.json +0 -0
  63. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/exceptions/__init__.py +0 -0
  64. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/exceptions/submission_not_found_exception.py +0 -0
  65. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/exceptions/submission_status_exception.py +0 -0
  66. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/executables/__init__.py +0 -0
  67. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/executables/check_fasta_insdc.py +0 -0
  68. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/executables/check_metadata_semantics.py +0 -0
  69. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/executables/samples_checker.py +0 -0
  70. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/executables/trim_down.py +0 -0
  71. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/executables/xlsx2json.py +1 -1
  72. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/jinja_templates/fasta_check.html +0 -0
  73. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/jinja_templates/file_validation.html +0 -0
  74. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/jinja_templates/metadata_validation.html +0 -0
  75. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/jinja_templates/project_details.html +0 -0
  76. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/jinja_templates/sample_name_check.html +0 -0
  77. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/jinja_templates/shallow_validation.html +0 -0
  78. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/metadata_utils.py +0 -0
  79. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/report.py +0 -0
  80. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/submission_ws.py +0 -0
  81. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli/validators/__init__.py +0 -0
  82. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli.egg-info/dependency_links.txt +0 -0
  83. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli.egg-info/entry_points.txt +0 -0
  84. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/eva_sub_cli.egg-info/top_level.txt +0 -0
  85. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/pyproject.toml +0 -0
  86. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/pytest.ini +0 -0
  87. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/setup.cfg +0 -0
  88. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/setup.py +0 -0
  89. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/ignore_test_nextflow_validation.yml +0 -0
  90. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/resources/EVA_Submission_test.json +0 -0
  91. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/resources/EVA_Submission_test_with_asm_report.json +0 -0
  92. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/resources/assembly_check/invalid.vcf.assembly_check.log +0 -0
  93. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/resources/assembly_check/invalid.vcf.text_assembly_report.txt +0 -0
  94. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/resources/assembly_reports/input_passed.txt +0 -0
  95. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/resources/fasta_files/Saccharomyces_cerevisiae_I.fa +0 -0
  96. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/resources/fasta_files/input_passed.fa +0 -0
  97. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/resources/sample_checker/example1.vcf.gz +0 -0
  98. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/resources/sample_checker/example2.vcf +0 -0
  99. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/resources/sample_checker/example3.vcf +0 -0
  100. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/resources/sample_checker/metadata.json +0 -0
  101. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/resources/validation_reports/expected_report_metadata_json.html +0 -0
  102. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/resources/validation_reports/expected_report_metadata_xlsx.html +0 -0
  103. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.html +0 -0
  104. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/resources/validation_reports/validation_output/assembly_check/input_passed.vcf.assembly_check.log +0 -0
  105. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/resources/validation_reports/validation_output/assembly_check/input_passed.vcf.text_assembly_report.1680102476700.txt +0 -0
  106. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/resources/validation_reports/validation_output/assembly_check/input_passed.vcf.valid_assembly_report.1680102476700.txt +0 -0
  107. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/resources/validation_reports/validation_output/other_validations/input_passed.fa_check.yml +0 -0
  108. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/resources/validation_reports/validation_output/other_validations/metadata_conversion_errors.yml +0 -0
  109. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/resources/validation_reports/validation_output/other_validations/metadata_semantic_check.yml +0 -0
  110. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/resources/validation_reports/validation_output/other_validations/metadata_validation.txt +0 -0
  111. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/resources/validation_reports/validation_output/other_validations/sample_checker.yml +0 -0
  112. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/resources/validation_reports/validation_output/vcf_format/input_passed.vcf.errors.1680102476854.db +0 -0
  113. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/resources/validation_reports/validation_output/vcf_format/input_passed.vcf.errors.1680102476854.txt +0 -0
  114. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/resources/validation_reports/validation_output/vcf_format/input_passed.vcf.vcf_format.log +0 -0
  115. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/resources/vcf_files/example2.vcf.gz +0 -0
  116. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/resources/vcf_files/input_passed.vcf +0 -0
  117. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/test_auth.py +0 -0
  118. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/test_check_fasta_insdc.py +0 -0
  119. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/test_samples_checker.py +0 -0
  120. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/test_utils.py +0 -0
  121. {eva_sub_cli-0.4 → eva_sub_cli-0.4.6}/tests/test_xlsx2json.py +0 -0
@@ -0,0 +1,51 @@
1
+ name: Publish Python Package
2
+
3
+ on:
4
+ push:
5
+ tags:
6
+ - '**'
7
+
8
+ jobs:
9
+ test:
10
+ runs-on: ubuntu-latest
11
+ strategy:
12
+ matrix:
13
+ python-version: ["3.9"]
14
+ steps:
15
+ - uses: actions/checkout@v4
16
+ - name: Set up Python ${{ matrix.python-version }}
17
+ uses: actions/setup-python@v5
18
+ with:
19
+ python-version: ${{ matrix.python-version }}
20
+ cache: pip
21
+ cache-dependency-path: '**/pyproject.toml'
22
+ - name: Install dependencies
23
+ run: |
24
+ python -m pip install --upgrade pip
25
+ pip install pytest
26
+ python -m pip install .
27
+ - name: Run tests
28
+ run: |
29
+ PYTHONPATH=. pytest tests -k "not integration"
30
+ deploy:
31
+ runs-on: ubuntu-latest
32
+ needs: [test]
33
+ environment: release
34
+ permissions:
35
+ id-token: write
36
+ steps:
37
+ - uses: actions/checkout@v4
38
+ - name: Set up Python
39
+ uses: actions/setup-python@v5
40
+ with:
41
+ python-version: '3.9'
42
+ cache: pip
43
+ cache-dependency-path: '**/pyproject.toml'
44
+ - name: Install dependencies
45
+ run: |
46
+ pip install setuptools wheel build
47
+ - name: Build
48
+ run: |
49
+ python -m build
50
+ - name: Publish
51
+ uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,213 @@
1
+ Metadata-Version: 2.4
2
+ Name: eva_sub_cli
3
+ Version: 0.4.6
4
+ Summary: EBI EVA - validation and submission command line tool
5
+ Classifier: Development Status :: 5 - Production/Stable
6
+ Classifier: Environment :: Console
7
+ Classifier: Intended Audience :: Science/Research
8
+ Classifier: Natural Language :: English
9
+ Classifier: Operating System :: POSIX :: Linux
10
+ Classifier: Operating System :: MacOS :: MacOS X
11
+ Classifier: Programming Language :: Python :: 3.6
12
+ Classifier: Programming Language :: Python :: 3.7
13
+ Classifier: Programming Language :: Python :: 3.8
14
+ Classifier: Programming Language :: Python :: 3.9
15
+ Classifier: Programming Language :: Python :: 3.10
16
+ Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
17
+ Classifier: Topic :: Communications :: File Sharing
18
+ Classifier: License :: OSI Approved :: Apache Software License
19
+ Requires-Python: >=3.6
20
+ Description-Content-Type: text/markdown
21
+ License-File: LICENSE
22
+ Requires-Dist: ebi_eva_common_pyutils==0.*,>0.6.11
23
+ Requires-Dist: jinja2==3.*,>=3.0.0
24
+ Requires-Dist: jsonschema==4.*
25
+ Requires-Dist: openpyxl==3.*
26
+ Requires-Dist: pyyaml==6.*
27
+ Requires-Dist: requests==2.*
28
+ Dynamic: license-file
29
+
30
+ # eva-sub-cli
31
+ The eva-sub-cli tool is a command line interface tool for data validation and upload. The tool transforms the submission process at EVA by enabling users to take control of data validation process. Previously handled by our helpdesk team, validation can now be performed directly by users, streamlining and improving the overall submission workflow at the EVA.
32
+
33
+
34
+ ## Installation
35
+
36
+ There are currently three ways to install and run the tool :
37
+ - Using conda
38
+ - From source using Docker
39
+ - From source natively (i.e. installing dependencies yourself)
40
+
41
+ ### 1. Conda
42
+
43
+ The most straightforward way to install eva-sub-cli and its dependencies is through conda.
44
+ For instance, the following commands install eva-sub-cli in a new environment called `eva`, activate the environment, and print
45
+ the help message:
46
+ ```bash
47
+ conda create -n eva -c conda-forge -c bioconda eva-sub-cli
48
+ conda activate eva
49
+ eva-sub-cli.py --help
50
+ ````
51
+
52
+ ### 2. From source using Docker
53
+
54
+ Docker provides an easy way to run eva-sub-cli without installing dependencies separately.
55
+ This method requires just Python 3.8+ and [Docker](https://docs.docker.com/engine/install/) to be installed.
56
+ Then you can install the most recent version from [PyPI](https://pypi.org/project/eva-sub-cli/) in a virtual environment:
57
+ ```bash
58
+ pip install eva-sub-cli
59
+ ```
60
+
61
+ To verify that the cli tool is installed correctly, run the following command, and you should see the help message displayed :
62
+ ```bash
63
+ eva-sub-cli.py -h
64
+ ```
65
+
66
+ ### 3. From source natively
67
+
68
+ This installation method requires the following :
69
+ * Python 3.8+
70
+ * [Nextflow](https://www.nextflow.io/docs/latest/getstarted.html) 21.10+
71
+ * [biovalidator](https://github.com/elixir-europe/biovalidator) 2.1.0+
72
+ * [vcf-validator](https://github.com/EBIvariation/vcf-validator) 0.9.7+
73
+
74
+ Install each of these and ensure they are included in your PATH. Then install the latest release as previously described.
75
+
76
+ ## Getting started with the eva-sub-cli tool
77
+
78
+ The ["Getting Started" guide](docs/Getting_Started_with_eva_sub_cli.md) serves as an introduction for users of the eva-sub-cli tool. It includes instructions on how to prepare your data and metadata, ensuring that users are equipped with the necessary information to successfully submit variant data. This guide is essential for new users, offering practical advice and tips for a smooth onboarding experience with the eva-sub-cli tool.
79
+
80
+ ## Options and parameters guide
81
+
82
+ The eva-sub-cli tool provides several options and parameters that you can use to tailor its functionality to your needs.
83
+ You can view all the available parameters with the command `eva-sub-cli.py -h` and view detailed explanations for the
84
+ input file requirements in the ["Getting Started" guide](docs/Getting_Started_with_eva_sub_cli.md).
85
+ Below is an overview of the key parameters.
86
+
87
+ ### Submission directory
88
+
89
+ This is the directory where all processing will take place, and where configuration and reports will be saved.
90
+ Crucially, the eva-sub-cli tool requires that there be **only one submission per directory** and that the submission directory not be reused.
91
+ Running multiple submissions from a single directory can result in data loss during validation and submission.
92
+
93
+ ### Metadata file
94
+
95
+ Metadata can be provided in one of two files.
96
+
97
+ #### The metadata spreadsheet
98
+
99
+ The metadata template can be found within the [etc folder](eva_sub_cli/etc/EVA_Submission_template.xlsx). It should be populated following the instructions provided within the template.
100
+ This is passed using the option `--metadata_xlsx`.
101
+
102
+ #### The metadata JSON
103
+
104
+ The metadata can also be provided via a JSON file, which should conform to the schema located [here](eva_sub_cli/etc/eva_schema.json).
105
+ This is passed using the option `--metadata_json`.
106
+
107
+ ### VCF files and Reference FASTA
108
+
109
+ These can be provided either in the metadata file directly, or on the command line using the `--vcf_files` and `--reference_fata` options.
110
+ Note that if you are using more than one reference FASTA, you **cannot** use the command line options; you must specify which VCF files use which FASTA files in the metadata.
111
+
112
+ VCF files can be either uncompressed or compressed using bgzip.
113
+ Other types of compression are not allowed and will result in errors during validation.
114
+ FASTA files must be uncompressed.
115
+
116
+ ## Execution
117
+
118
+ ### Validate only
119
+
120
+ To validate and not submit, run the following command:
121
+
122
+ ```shell
123
+ eva-sub-cli.py --metadata_xlsx metadata_spreadsheet.xlsx --submission_dir submission_dir --tasks VALIDATE
124
+ ```
125
+
126
+ **Note for Docker users:**
127
+
128
+ Make sure that Docker is running in the background, e.g. by opening Docker Desktop.
129
+ For each of the below commands, add the command line option `--executor docker`, which will
130
+ fetch and manage the Docker container for you.
131
+
132
+ ```shell
133
+ eva-sub-cli.py --metadata_xlsx metadata_spreadsheet.xlsx --submission_dir submission_dir --tasks VALIDATE --executor docker
134
+ ```
135
+
136
+ ### Validate and submit your dataset
137
+
138
+ To validate and submit, run the following command:
139
+
140
+ ```shell
141
+ eva-sub-cli.py --metadata_xlsx metadata_spreadsheet.xlsx \
142
+ --vcf_files vcf_file1.vcf vcf_file2.vcf --reference_fasta assembly.fa --submission_dir submission_dir
143
+ ```
144
+
145
+
146
+ ### Submit only
147
+
148
+ All submissions must have been validated. You cannot run the submission without validation. Once validated, execute the following command:
149
+
150
+ ```shell
151
+ eva-sub-cli.py --metadata_xlsx metadata_spreadsheet.xlsx --submission_dir submission_dir
152
+ ```
153
+ or
154
+ ```shell
155
+ eva-sub-cli.py --metadata_xlsx metadata_spreadsheet.xlsx --submission_dir submission_dir --tasks SUBMIT
156
+ ```
157
+ This will only submit the data and not validate.
158
+
159
+ ### Shallow validation
160
+
161
+ If you are working with large VCF files and find that validation takes a very long time, you can add the
162
+ argument `--shallow` to the command, which will validate only the first 10,000 lines in each VCF. Note that running
163
+ shallow validation will **not** be sufficient for actual submission.
164
+
165
+
166
+ ## Leveraging Nextflow to parallelize the validation process
167
+
168
+ Nextflow is a common workflow management system that helps orchestrate tasks and interface with the execution engine (like HPC or cloud). When running natively (i.e. not using Docker), eva-sub-cli will use Nextflow to run all the validation steps. In this section we'll see how it can be parameterised to work with your compute infrastructure.
169
+
170
+ When no options are provided, Nextflow will run as many tasks as there are available CPUs on the machine executing it. To modify how many tasks can start and how Nextflow will process each one, you can provide a Nextflow configuration file in several ways.
171
+
172
+ From the command line you can use `--nextflow_config <path>` to specify the Nextflow config file you want to apply. The configuration can also be picked up from other places directly by Nextflow. Please refer to [the nextflow documentation](https://www.nextflow.io/docs/latest/config.html) for more details.
173
+
174
+ ### Basic Nextflow configuration.
175
+
176
+ There are many options to configure Nextflow so we will not provide them all. Please refer to [the documentation](https://www.nextflow.io/docs/latest/reference/config.html) for advanced features.
177
+ Below is a very basic Nextflow configuration file that will request 2 cpus for each process, essentially limiting the number of process to half the number of available CPUs
178
+ ```
179
+ process {
180
+ executor="local"
181
+ cpus=2
182
+ }
183
+ ```
184
+ In this configuration, all the process will be running on the same machine where eva-sub-cli was started as described in the schema below.
185
+ ```
186
+ (Local machine)
187
+ eva-sub-cli
188
+ |_ nextflow
189
+ |_ task1
190
+ |_ task2
191
+ ```
192
+
193
+ ### Basic Nextflow configuration for HPC use.
194
+
195
+ If you have access to High Performance Compute (HPC) environment, Nextflow supports the main resource managers such as [SLURM](https://www.nextflow.io/docs/latest/executor.html#slurm), [SGE](https://www.nextflow.io/docs/latest/executor.html#sge), [LSF](https://www.nextflow.io/docs/latest/executor.html#lsf) and others.
196
+ In the configuration below, we're assuming that you are using SLURM. It would work similarly with other resource managers.
197
+ ```
198
+ process {
199
+ executor="slurm"
200
+ queue="my_production_queue"
201
+ }
202
+ ```
203
+
204
+ In this configuration, the subtasks will be performed in other machines as specified by your SLURM resource manager as described in the schema below.
205
+ ```
206
+ (Local machine)
207
+ eva-sub-cli
208
+ |_ nextflow
209
+ (Other compute node)
210
+ task1
211
+ (Other compute node)
212
+ task2
213
+ ```
@@ -0,0 +1,184 @@
1
+ # eva-sub-cli
2
+ The eva-sub-cli tool is a command line interface tool for data validation and upload. The tool transforms the submission process at EVA by enabling users to take control of data validation process. Previously handled by our helpdesk team, validation can now be performed directly by users, streamlining and improving the overall submission workflow at the EVA.
3
+
4
+
5
+ ## Installation
6
+
7
+ There are currently three ways to install and run the tool :
8
+ - Using conda
9
+ - From source using Docker
10
+ - From source natively (i.e. installing dependencies yourself)
11
+
12
+ ### 1. Conda
13
+
14
+ The most straightforward way to install eva-sub-cli and its dependencies is through conda.
15
+ For instance, the following commands install eva-sub-cli in a new environment called `eva`, activate the environment, and print
16
+ the help message:
17
+ ```bash
18
+ conda create -n eva -c conda-forge -c bioconda eva-sub-cli
19
+ conda activate eva
20
+ eva-sub-cli.py --help
21
+ ````
22
+
23
+ ### 2. From source using Docker
24
+
25
+ Docker provides an easy way to run eva-sub-cli without installing dependencies separately.
26
+ This method requires just Python 3.8+ and [Docker](https://docs.docker.com/engine/install/) to be installed.
27
+ Then you can install the most recent version from [PyPI](https://pypi.org/project/eva-sub-cli/) in a virtual environment:
28
+ ```bash
29
+ pip install eva-sub-cli
30
+ ```
31
+
32
+ To verify that the cli tool is installed correctly, run the following command, and you should see the help message displayed :
33
+ ```bash
34
+ eva-sub-cli.py -h
35
+ ```
36
+
37
+ ### 3. From source natively
38
+
39
+ This installation method requires the following :
40
+ * Python 3.8+
41
+ * [Nextflow](https://www.nextflow.io/docs/latest/getstarted.html) 21.10+
42
+ * [biovalidator](https://github.com/elixir-europe/biovalidator) 2.1.0+
43
+ * [vcf-validator](https://github.com/EBIvariation/vcf-validator) 0.9.7+
44
+
45
+ Install each of these and ensure they are included in your PATH. Then install the latest release as previously described.
46
+
47
+ ## Getting started with the eva-sub-cli tool
48
+
49
+ The ["Getting Started" guide](docs/Getting_Started_with_eva_sub_cli.md) serves as an introduction for users of the eva-sub-cli tool. It includes instructions on how to prepare your data and metadata, ensuring that users are equipped with the necessary information to successfully submit variant data. This guide is essential for new users, offering practical advice and tips for a smooth onboarding experience with the eva-sub-cli tool.
50
+
51
+ ## Options and parameters guide
52
+
53
+ The eva-sub-cli tool provides several options and parameters that you can use to tailor its functionality to your needs.
54
+ You can view all the available parameters with the command `eva-sub-cli.py -h` and view detailed explanations for the
55
+ input file requirements in the ["Getting Started" guide](docs/Getting_Started_with_eva_sub_cli.md).
56
+ Below is an overview of the key parameters.
57
+
58
+ ### Submission directory
59
+
60
+ This is the directory where all processing will take place, and where configuration and reports will be saved.
61
+ Crucially, the eva-sub-cli tool requires that there be **only one submission per directory** and that the submission directory not be reused.
62
+ Running multiple submissions from a single directory can result in data loss during validation and submission.
63
+
64
+ ### Metadata file
65
+
66
+ Metadata can be provided in one of two files.
67
+
68
+ #### The metadata spreadsheet
69
+
70
+ The metadata template can be found within the [etc folder](eva_sub_cli/etc/EVA_Submission_template.xlsx). It should be populated following the instructions provided within the template.
71
+ This is passed using the option `--metadata_xlsx`.
72
+
73
+ #### The metadata JSON
74
+
75
+ The metadata can also be provided via a JSON file, which should conform to the schema located [here](eva_sub_cli/etc/eva_schema.json).
76
+ This is passed using the option `--metadata_json`.
77
+
78
+ ### VCF files and Reference FASTA
79
+
80
+ These can be provided either in the metadata file directly, or on the command line using the `--vcf_files` and `--reference_fata` options.
81
+ Note that if you are using more than one reference FASTA, you **cannot** use the command line options; you must specify which VCF files use which FASTA files in the metadata.
82
+
83
+ VCF files can be either uncompressed or compressed using bgzip.
84
+ Other types of compression are not allowed and will result in errors during validation.
85
+ FASTA files must be uncompressed.
86
+
87
+ ## Execution
88
+
89
+ ### Validate only
90
+
91
+ To validate and not submit, run the following command:
92
+
93
+ ```shell
94
+ eva-sub-cli.py --metadata_xlsx metadata_spreadsheet.xlsx --submission_dir submission_dir --tasks VALIDATE
95
+ ```
96
+
97
+ **Note for Docker users:**
98
+
99
+ Make sure that Docker is running in the background, e.g. by opening Docker Desktop.
100
+ For each of the below commands, add the command line option `--executor docker`, which will
101
+ fetch and manage the Docker container for you.
102
+
103
+ ```shell
104
+ eva-sub-cli.py --metadata_xlsx metadata_spreadsheet.xlsx --submission_dir submission_dir --tasks VALIDATE --executor docker
105
+ ```
106
+
107
+ ### Validate and submit your dataset
108
+
109
+ To validate and submit, run the following command:
110
+
111
+ ```shell
112
+ eva-sub-cli.py --metadata_xlsx metadata_spreadsheet.xlsx \
113
+ --vcf_files vcf_file1.vcf vcf_file2.vcf --reference_fasta assembly.fa --submission_dir submission_dir
114
+ ```
115
+
116
+
117
+ ### Submit only
118
+
119
+ All submissions must have been validated. You cannot run the submission without validation. Once validated, execute the following command:
120
+
121
+ ```shell
122
+ eva-sub-cli.py --metadata_xlsx metadata_spreadsheet.xlsx --submission_dir submission_dir
123
+ ```
124
+ or
125
+ ```shell
126
+ eva-sub-cli.py --metadata_xlsx metadata_spreadsheet.xlsx --submission_dir submission_dir --tasks SUBMIT
127
+ ```
128
+ This will only submit the data and not validate.
129
+
130
+ ### Shallow validation
131
+
132
+ If you are working with large VCF files and find that validation takes a very long time, you can add the
133
+ argument `--shallow` to the command, which will validate only the first 10,000 lines in each VCF. Note that running
134
+ shallow validation will **not** be sufficient for actual submission.
135
+
136
+
137
+ ## Leveraging Nextflow to parallelize the validation process
138
+
139
+ Nextflow is a common workflow management system that helps orchestrate tasks and interface with the execution engine (like HPC or cloud). When running natively (i.e. not using Docker), eva-sub-cli will use Nextflow to run all the validation steps. In this section we'll see how it can be parameterised to work with your compute infrastructure.
140
+
141
+ When no options are provided, Nextflow will run as many tasks as there are available CPUs on the machine executing it. To modify how many tasks can start and how Nextflow will process each one, you can provide a Nextflow configuration file in several ways.
142
+
143
+ From the command line you can use `--nextflow_config <path>` to specify the Nextflow config file you want to apply. The configuration can also be picked up from other places directly by Nextflow. Please refer to [the nextflow documentation](https://www.nextflow.io/docs/latest/config.html) for more details.
144
+
145
+ ### Basic Nextflow configuration.
146
+
147
+ There are many options to configure Nextflow so we will not provide them all. Please refer to [the documentation](https://www.nextflow.io/docs/latest/reference/config.html) for advanced features.
148
+ Below is a very basic Nextflow configuration file that will request 2 cpus for each process, essentially limiting the number of process to half the number of available CPUs
149
+ ```
150
+ process {
151
+ executor="local"
152
+ cpus=2
153
+ }
154
+ ```
155
+ In this configuration, all the process will be running on the same machine where eva-sub-cli was started as described in the schema below.
156
+ ```
157
+ (Local machine)
158
+ eva-sub-cli
159
+ |_ nextflow
160
+ |_ task1
161
+ |_ task2
162
+ ```
163
+
164
+ ### Basic Nextflow configuration for HPC use.
165
+
166
+ If you have access to High Performance Compute (HPC) environment, Nextflow supports the main resource managers such as [SLURM](https://www.nextflow.io/docs/latest/executor.html#slurm), [SGE](https://www.nextflow.io/docs/latest/executor.html#sge), [LSF](https://www.nextflow.io/docs/latest/executor.html#lsf) and others.
167
+ In the configuration below, we're assuming that you are using SLURM. It would work similarly with other resource managers.
168
+ ```
169
+ process {
170
+ executor="slurm"
171
+ queue="my_production_queue"
172
+ }
173
+ ```
174
+
175
+ In this configuration, the subtasks will be performed in other machines as specified by your SLURM resource manager as described in the schema below.
176
+ ```
177
+ (Local machine)
178
+ eva-sub-cli
179
+ |_ nextflow
180
+ (Other compute node)
181
+ task1
182
+ (Other compute node)
183
+ task2
184
+ ```
@@ -0,0 +1,20 @@
1
+ #!/bin/bash
2
+
3
+ BIOVALIDATOR_VERSION=2.2.1
4
+
5
+ EVA_SUB_CLI="${PREFIX}/share/${PKG_NAME}-${PKG_VERSION}"
6
+ mkdir -p ${EVA_SUB_CLI}
7
+
8
+ # Install eva-sub-cli
9
+ $PYTHON -m pip install .
10
+ echo "Done with eva-sub-cli"
11
+
12
+ cd ${EVA_SUB_CLI}
13
+
14
+ # Install biovalidator from source
15
+ curl -Lo biovalidator.zip https://github.com/elixir-europe/biovalidator/archive/refs/tags/v${BIOVALIDATOR_VERSION}.zip \
16
+ && unzip -q biovalidator.zip && rm biovalidator.zip \
17
+ && cd biovalidator-${BIOVALIDATOR_VERSION} \
18
+ && npm install && npm install -g \
19
+ && cd ..
20
+ echo "Done with biovalidator"
@@ -1,37 +1,42 @@
1
1
  {% set name = "eva-sub-cli" %}
2
- {% set version = "0.2" %}
2
+ {% set version = "0.4" %}
3
3
 
4
4
  package:
5
5
  name: {{ name }}
6
6
  version: {{ version }}
7
7
 
8
8
  source:
9
- path: ../
10
- # For submission to bioconda recipes, replace 'path' with the following (filling in the appropriate sha256):
11
- # url: https://github.com/EBIvariation/eva-sub-cli/archive/v{{version}}.tar.gz
12
- # sha256:
9
+ url: https://github.com/EBIvariation/eva-sub-cli/archive/v{{version}}.tar.gz
10
+ sha256: be4d4109d9879ad94ccc01a3bd534ff6077775f126977dfb3f2d6fd9cfb251ba
13
11
 
14
12
  build:
15
13
  number: 0
16
14
  noarch: generic
17
- preserve_egg_dir: True
18
- # For submission to bioconda recipes, replace 'preserve_egg_dir' with the following:
19
- # run_exports:
20
- # - {{ pin_subpackage(name, max_pin="x.x") }}
15
+ script_env:
16
+ - SETUPTOOLS_SCM_PRETEND_VERSION={{ version }}
17
+ run_exports:
18
+ - {{ pin_subpackage(name, max_pin="x.x") }}
21
19
 
22
20
  requirements:
23
21
  host:
24
22
  - nextflow >=21.10.0
25
23
  - python >=3.8
26
24
  - nodejs >=10.19.1
25
+ - vcf-validator >=0.10.0
26
+ - ebi-eva-common-pyutils >=0.6.1
27
27
  - pyyaml
28
28
  - jinja2
29
29
  - openpyxl
30
30
  - requests
31
31
  - jsonschema
32
+ - unzip
33
+ - setuptools-scm
32
34
  run:
33
35
  - nextflow >=21.10.0
34
36
  - python >=3.8
37
+ - nodejs >=10.19.1
38
+ - vcf-validator >=0.9.7
39
+ - ebi-eva-common-pyutils >=0.6.1
35
40
  - pyyaml
36
41
  - jinja2
37
42
  - openpyxl
@@ -43,7 +48,6 @@ test:
43
48
  - eva_sub_cli
44
49
  commands:
45
50
  - biovalidator --help
46
- - vcf_validator --help
47
51
  - eva-sub-cli.py --help
48
52
 
49
53
  about:
@@ -54,4 +58,6 @@ about:
54
58
 
55
59
  extra:
56
60
  recipe-maintainers:
57
- - apriltuesday
61
+ - apriltuesday
62
+ - tcezard
63
+ - ebi-variation
@@ -1,6 +1,6 @@
1
1
  FROM python:3.10
2
2
 
3
- ENV vcf_validator_version=0.9.7
3
+ ENV vcf_validator_version=0.10.0
4
4
  ENV NXF_VER=22.10.6
5
5
 
6
6
  WORKDIR /opt
@@ -6,6 +6,7 @@ PACKAGE_DIR = os.path.dirname(os.path.abspath(__file__))
6
6
  NEXTFLOW_DIR = os.path.join(PACKAGE_DIR, 'nextflow')
7
7
  ETC_DIR = os.path.join(PACKAGE_DIR, 'etc')
8
8
  LSRI_CLIENT_ID = "aa0fcc42-096a-4f9d-b871-aceb1a97d174"
9
+ DEFAULT_METADATA_XLSX_TEMPLATE_VERSION = '1.1.6'
9
10
 
10
11
  SUB_CLI_CONFIG_FILE = ".eva_sub_cli_config.yml"
11
12
 
@@ -1,8 +1,13 @@
1
- # file generated by setuptools_scm
1
+ # file generated by setuptools-scm
2
2
  # don't change, don't track in version control
3
+
4
+ __all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
5
+
3
6
  TYPE_CHECKING = False
4
7
  if TYPE_CHECKING:
5
- from typing import Tuple, Union
8
+ from typing import Tuple
9
+ from typing import Union
10
+
6
11
  VERSION_TUPLE = Tuple[Union[int, str], ...]
7
12
  else:
8
13
  VERSION_TUPLE = object
@@ -12,5 +17,5 @@ __version__: str
12
17
  __version_tuple__: VERSION_TUPLE
13
18
  version_tuple: VERSION_TUPLE
14
19
 
15
- __version__ = version = '0.4'
16
- __version_tuple__ = version_tuple = (0, 4)
20
+ __version__ = version = '0.4.6'
21
+ __version_tuple__ = version_tuple = (0, 4, 6)
@@ -32,7 +32,7 @@ Project:
32
32
  Project Title: title
33
33
  Description: description
34
34
  Center: centre
35
- Tax ID: taxId
35
+ Taxonomy ID: taxId
36
36
  optional:
37
37
  Publication(s): publications
38
38
  Parent Project: parentProject
@@ -74,6 +74,7 @@ Analysis:
74
74
  Imputation: boolean
75
75
  Phasing: boolean
76
76
  Date: date
77
+ Software: list
77
78
 
78
79
  Sample:
79
80
  header_row: 3
@@ -89,7 +90,7 @@ Sample:
89
90
  Prefix: prefix
90
91
  Subject: subject
91
92
  Derived From: derivedFrom
92
- Tax Id: taxId
93
+ Taxonomy ID: taxId
93
94
  Scientific Name: scientificName
94
95
  Common Name: commonName
95
96
  mating_type: matingType
@@ -0,0 +1,4 @@
1
+ class InvalidFileTypeError(Exception):
2
+ def __init__(self, message):
3
+ self.message = message
4
+ super().__init__(self.message)
@@ -0,0 +1,10 @@
1
+ class MetadataTemplateVersionException(Exception):
2
+ def __init__(self, message):
3
+ self.message = message
4
+ super().__init__(self.message)
5
+
6
+
7
+ class MetadataTemplateVersionNotFoundException(Exception):
8
+ def __init__(self, message):
9
+ self.message = message
10
+ super().__init__(self.message)