eva-sub-cli 0.4.14__tar.gz → 0.5.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/.github/workflows/tests.yml +1 -1
  2. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/PKG-INFO +1 -1
  3. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/docker/Dockerfile +1 -1
  4. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/docs/how_to_submit.md +15 -0
  5. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/docs/input_file_overview.md +9 -9
  6. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/docs/validation_overview.md +1 -1
  7. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/_version.py +3 -3
  8. eva_sub_cli-0.5.1/eva_sub_cli/etc/EVA_Submission_Example.xlsx +0 -0
  9. eva_sub_cli-0.5.1/eva_sub_cli/etc/EVA_Submission_template.xlsx +0 -0
  10. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/etc/eva_schema.json +71 -56
  11. eva_sub_cli-0.5.1/eva_sub_cli/etc/spreadsheet2json_conf.yaml +139 -0
  12. eva_sub_cli-0.4.14/eva_sub_cli/etc/spreadsheet2json_conf.yaml → eva_sub_cli-0.5.1/eva_sub_cli/etc/spreadsheet2json_conf_V2.yaml +5 -3
  13. eva_sub_cli-0.5.1/eva_sub_cli/exceptions/submission_upload_exception.py +5 -0
  14. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/executables/cli.py +24 -8
  15. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/executables/xlsx2json.py +11 -4
  16. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/jinja_templates/html/file_validation.html +2 -2
  17. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/jinja_templates/html/report.html +42 -14
  18. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/jinja_templates/html/shallow_validation.html +1 -1
  19. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/jinja_templates/text/file_validation.txt +4 -2
  20. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/jinja_templates/text/report.txt +31 -7
  21. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/jinja_templates/text/shallow_validation.txt +1 -1
  22. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/nextflow/validation.nf +45 -29
  23. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/orchestrator.py +34 -13
  24. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/report.py +5 -1
  25. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/semantic_metadata.py +11 -7
  26. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/submission_ws.py +9 -4
  27. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/submit.py +5 -2
  28. eva_sub_cli-0.5.1/eva_sub_cli/utils.py +20 -0
  29. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/validators/docker_validator.py +13 -9
  30. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/validators/native_validator.py +8 -4
  31. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/validators/validator.py +179 -81
  32. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli.egg-info/PKG-INFO +1 -1
  33. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli.egg-info/SOURCES.txt +9 -1
  34. eva_sub_cli-0.5.1/tests/resources/EVA_Submission_Docker_Test.xlsx +0 -0
  35. eva_sub_cli-0.5.1/tests/resources/EVA_Submission_test.xlsx +0 -0
  36. eva_sub_cli-0.5.1/tests/resources/EVA_Submission_test_V2.xlsx +0 -0
  37. eva_sub_cli-0.5.1/tests/resources/EVA_Submission_test_fails.xlsx +0 -0
  38. eva_sub_cli-0.5.1/tests/resources/EVA_Submission_test_version_missing.xlsx +0 -0
  39. eva_sub_cli-0.5.1/tests/resources/EVA_Submission_test_with_project_accession.json +151 -0
  40. eva_sub_cli-0.5.1/tests/resources/EVA_Submission_test_with_project_accession.xlsx +0 -0
  41. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/metadata_with_filename.json +7 -1
  42. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/validation_reports/expected_report_metadata_json.html +7 -7
  43. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/validation_reports/expected_report_metadata_json.txt +7 -7
  44. eva_sub_cli-0.5.1/tests/resources/validation_reports/expected_report_metadata_json_process_not_run.html +142 -0
  45. eva_sub_cli-0.5.1/tests/resources/validation_reports/expected_report_metadata_json_process_not_run.txt +44 -0
  46. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/validation_reports/expected_report_metadata_xlsx.html +7 -7
  47. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/validation_reports/expected_report_metadata_xlsx.txt +7 -7
  48. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.html +7 -7
  49. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.txt +7 -7
  50. eva_sub_cli-0.5.1/tests/resources/validation_reports/validation_output/other_validations/input_passed_trim_down.yml +3 -0
  51. eva_sub_cli-0.5.1/tests/test_cli.py +116 -0
  52. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/test_docker_validator.py +5 -1
  53. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/test_orchestrator.py +91 -17
  54. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/test_report.py +67 -6
  55. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/test_semantic_metadata.py +27 -0
  56. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/test_submit.py +2 -0
  57. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/test_validator.py +206 -21
  58. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/test_xlsx2json.py +62 -1
  59. eva_sub_cli-0.4.14/.github/workflows/publish_packages.yml +0 -45
  60. eva_sub_cli-0.4.14/eva_sub_cli/etc/EVA_Submission_Example.xlsx +0 -0
  61. eva_sub_cli-0.4.14/eva_sub_cli/etc/EVA_Submission_template.xlsx +0 -0
  62. eva_sub_cli-0.4.14/tests/resources/EVA_Submission_Docker_Test.xlsx +0 -0
  63. eva_sub_cli-0.4.14/tests/resources/EVA_Submission_test.xlsx +0 -0
  64. eva_sub_cli-0.4.14/tests/resources/EVA_Submission_test_fails.xlsx +0 -0
  65. eva_sub_cli-0.4.14/tests/resources/EVA_Submission_test_version_missing.xlsx +0 -0
  66. eva_sub_cli-0.4.14/tests/test_cli.py +0 -67
  67. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/.github/workflows/publish.yml +0 -0
  68. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/.github/workflows/publish_docs.yml +0 -0
  69. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/.github/workflows/trigger-eva-sub-cli-integration-tests.yml +0 -0
  70. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/.gitignore +0 -0
  71. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/LICENSE +0 -0
  72. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/README.md +0 -0
  73. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/conda/build.sh +0 -0
  74. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/conda/meta.yaml +0 -0
  75. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/docs/README.md +0 -0
  76. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/docs/_config.yml +0 -0
  77. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/docs/installation.md +0 -0
  78. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/docs/using_nextflow.md +0 -0
  79. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/__init__.py +0 -0
  80. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/auth.py +0 -0
  81. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/date_utils.py +0 -0
  82. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/etc/eva-biosamples.json +0 -0
  83. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/etc/eva_logo.png +0 -0
  84. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/exceptions/__init__.py +0 -0
  85. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/exceptions/invalid_file_type_exception.py +0 -0
  86. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/exceptions/metadata_template_version_exception.py +0 -0
  87. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/exceptions/submission_not_found_exception.py +0 -0
  88. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/exceptions/submission_status_exception.py +0 -0
  89. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/executables/__init__.py +0 -0
  90. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/executables/check_fasta_insdc.py +0 -0
  91. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/executables/check_metadata_semantics.py +0 -0
  92. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/executables/evidence_type_checker.py +0 -0
  93. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/executables/samples_checker.py +0 -0
  94. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/executables/trim_down.py +0 -0
  95. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/file_utils.py +0 -0
  96. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/jinja_templates/html/fasta_check.html +0 -0
  97. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/jinja_templates/html/metadata_validation.html +0 -0
  98. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/jinja_templates/html/project_details.html +0 -0
  99. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/jinja_templates/html/sample_name_check.html +0 -0
  100. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/jinja_templates/text/fasta_check.txt +0 -0
  101. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/jinja_templates/text/metadata_validation.txt +0 -0
  102. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/jinja_templates/text/project_details.txt +0 -0
  103. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/jinja_templates/text/sample_name_check.txt +0 -0
  104. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/metadata.py +0 -0
  105. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/validators/__init__.py +0 -0
  106. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli/validators/validation_results_parsers.py +0 -0
  107. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli.egg-info/dependency_links.txt +0 -0
  108. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli.egg-info/entry_points.txt +0 -0
  109. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli.egg-info/requires.txt +0 -0
  110. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/eva_sub_cli.egg-info/top_level.txt +0 -0
  111. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/pyproject.toml +0 -0
  112. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/pytest.ini +0 -0
  113. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/requirements.txt +0 -0
  114. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/setup.cfg +0 -0
  115. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/setup.py +0 -0
  116. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/build_and_test_docker_locally.py +0 -0
  117. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/ignore_test_nextflow_validation.yml +0 -0
  118. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/EVA_Submission_test.json +0 -0
  119. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/EVA_Submission_test_with_asm_report.json +0 -0
  120. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/assembly_check/invalid.vcf.assembly_check.log +0 -0
  121. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/assembly_check/invalid.vcf.text_assembly_report.txt +0 -0
  122. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/assembly_reports/input_passed.txt +0 -0
  123. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/fasta_files/Saccharomyces_cerevisiae_I.fa +0 -0
  124. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/fasta_files/input_passed.fa +0 -0
  125. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/fasta_files/multi_seq.fa +0 -0
  126. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/sample_checker/example1.vcf.gz +0 -0
  127. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/sample_checker/example2.vcf +0 -0
  128. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/sample_checker/example3.vcf +0 -0
  129. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/sample_checker/metadata.json +0 -0
  130. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/validation_reports/validation_output/assembly_check/input_passed.vcf.assembly_check.log +0 -0
  131. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/validation_reports/validation_output/assembly_check/input_passed.vcf.text_assembly_report.1680102476700.txt +0 -0
  132. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/validation_reports/validation_output/assembly_check/input_passed.vcf.valid_assembly_report.1680102476700.txt +0 -0
  133. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/validation_reports/validation_output/other_validations/evidence_type_checker.yml +0 -0
  134. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/validation_reports/validation_output/other_validations/input_passed.fa_check.yml +0 -0
  135. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/validation_reports/validation_output/other_validations/metadata_conversion_errors.yml +0 -0
  136. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/validation_reports/validation_output/other_validations/metadata_semantic_check.yml +0 -0
  137. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/validation_reports/validation_output/other_validations/metadata_validation.txt +0 -0
  138. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/validation_reports/validation_output/other_validations/sample_checker.yml +0 -0
  139. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/validation_reports/validation_output/vcf_format/input_passed.vcf.errors.1680102476854.db +0 -0
  140. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/validation_reports/validation_output/vcf_format/input_passed.vcf.errors.1680102476854.txt +0 -0
  141. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/validation_reports/validation_output/vcf_format/input_passed.vcf.vcf_format.log +0 -0
  142. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/vcf_files/example2.vcf.gz +0 -0
  143. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/resources/vcf_files/input_passed.vcf +0 -0
  144. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/test_auth.py +0 -0
  145. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/test_check_fasta_insdc.py +0 -0
  146. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/test_evidence_type_checker.py +0 -0
  147. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/test_file_utils.py +0 -0
  148. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/test_metadata.py +0 -0
  149. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/test_samples_checker.py +0 -0
  150. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/test_trim_down.py +0 -0
  151. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/test_utils.py +0 -0
  152. {eva_sub_cli-0.4.14 → eva_sub_cli-0.5.1}/tests/test_validaton_results_parsers.py +0 -0
@@ -4,7 +4,7 @@ on:
4
4
  push:
5
5
  branches: [ main ]
6
6
  pull_request:
7
- branches: [ main ]
7
+ branches: [ main, add-validation-tasks ]
8
8
 
9
9
  jobs:
10
10
  build:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: eva_sub_cli
3
- Version: 0.4.14
3
+ Version: 0.5.1
4
4
  Summary: EBI EVA - validation and submission command line tool
5
5
  Classifier: Development Status :: 5 - Production/Stable
6
6
  Classifier: Environment :: Console
@@ -1,7 +1,7 @@
1
1
  FROM python:3.10
2
2
 
3
3
  ENV vcf_validator_version=0.10.2
4
- ENV NXF_VER=22.10.6
4
+ ENV NXF_VER=23.10.0
5
5
 
6
6
  WORKDIR /opt
7
7
 
@@ -65,6 +65,21 @@ If you are working with large VCF files and find that validation takes a very lo
65
65
  argument `--shallow` to the command, which will validate only the first 10,000 lines in each VCF. Note that running
66
66
  shallow validation will **not** be sufficient for actual submission but will allow you to identify most validation concerns without long run time.
67
67
 
68
+ ### Validation tasks
69
+ To run only a subset of the validation steps, you can use the `--validation_tasks` argument.
70
+ This can be useful if you want to avoid re-running long-running validations that have already passed.
71
+ Note that all validation tasks must pass in order to submit. The report will aggregate the results of previous runs and new ones.
72
+ The possible tasks are:
73
+ * `vcf_check` - includes syntax validation and other checks on VCF files
74
+ * `assembly_check` - includes all checks involving the FASTA file
75
+ * `metadata_check` - includes syntactic and semantic checks on metadata
76
+ * `sample_check` - includes sample coherence checks between VCF files and metadata
77
+
78
+ For example, to run only `vcf_check` and `sample_check`:
79
+ ```shell
80
+ eva-sub-cli.py --metadata_xlsx <metadata file> --submission_dir <submission directory> --validation_tasks vcf_check sample_check
81
+ ```
82
+
68
83
  ### VCF files and reference FASTA
69
84
  These can be provided either in the metadata file directly, or on the command line using the `--vcf_files` and
70
85
  `--reference_fasta` options. Note that if you are using more than one reference FASTA, you **cannot** use the command
@@ -34,12 +34,12 @@ Body: The body of the VCF file contains the actual variant data, with each row r
34
34
  # Metadata Spreadsheet
35
35
 
36
36
  The spreadsheet provides comprehensive contextual information about the dataset, ensuring that each submission is accompanied by detailed descriptions that facilitate proper understanding and use of the data. Key elements included in the metadata spreadsheet are analysis and project information, sample information, sequencing methodologies, experimental details.
37
- The spreadsheet is composed of editable tabs that will receive the metadata and non-editable helper tabs that provide detailed description of each column.
38
-
39
- | WORKSHEET | EXPLANATION |
40
- |-------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
41
- | Submitter Details | This sheet captures the details of the submitter |
42
- | Project | The objective of this sheet is to gather general information about the Project including submitter, submitting centre, collaborators, project title, description and publications. |
43
- | Sample | Projects consist of analyses that are run on samples. We accept sample information in the form of BioSample, ENA or EGA accession(s). We also accept BioSamples sampleset accessions. If your samples are not yet accessioned, and are therefore novel, please use the "Novel sample(s)" sections of the Sample(s) worksheet to have them registered at BioSample |
44
- | Analysis | For EVA, each analysis is one vcf file, plus an unlimited number of ancillary files. This sheet allows EVA to link vcf files to a project and to other EVA analyses. Additionally, this worksheet contains experimental meta-data detailing the methodology of each analysis. Important to note; one project can have multiple associated analyses |
45
- | Files | Filenames and associated checking data associated with this EVA submission should be entered into this worksheet. Each file should be linked to exactly one analysis. |
37
+ The spreadsheet is organized into editable tabs, designed for metadata entry, and non-editable helper tabs, which offer detailed explanations and guidance for each column. Users are required to complete all relevant sections within the editable tabs. Mandatory fields in each section are indicated in bold to highlight essential information that must be provided for a valid submission. However, users are strongly encouraged to provide as much additional information as possible to enhance the completeness and usefulness of the metadata.
38
+
39
+ | WORKSHEET | EXPLANATION |
40
+ |-------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
41
+ | Submitter Details | This sheet captures the details of the submitter |
42
+ | Project | The objective of this sheet is to gather general information about the Project. If you are submitting to an existing project, you can skip the other details and just provide the project accession and analyses will be linked to that project. In case of a new project, please provide the relevant details including submitter, submitting centre, collaborators, project title, description and publications. |
43
+ | Sample | Projects consist of analyses that are run on samples. We accept sample information in the form of BioSample, ENA or EGA accession(s). We also accept BioSamples sampleset accessions. If your samples are not yet accessioned, and are therefore novel, please use the "Novel sample(s)" sections of the Sample(s) worksheet to have them registered at BioSample |
44
+ | Analysis | For EVA, each analysis is one vcf file, plus an unlimited number of ancillary files. This sheet allows EVA to link vcf files to a project and to other EVA analyses. Additionally, this worksheet contains experimental meta-data detailing the methodology of each analysis. Important to note; one project can have multiple associated analyses |
45
+ | Files | Filenames and associated checking data associated with this EVA submission should be entered into this worksheet. Each file should be linked to exactly one analysis. |
@@ -17,7 +17,7 @@ Key points to note before validating your metadata spreadsheet with the eva-sub-
17
17
 
18
18
  - Please do not change the existing structure of the spreadsheet
19
19
  - Ensure all mandatory columns (marked in bold) are filled.
20
- - Pre-registered samples must be released and not kept in private status
20
+ - Pre-registered project and samples must be released and not kept in private status
21
21
  - Sample names in the spreadsheet must match those in the VCF file.
22
22
  - Analysis aliases must match across the sheets (Analysis, Sample, and File sheets).
23
23
 
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
28
28
  commit_id: COMMIT_ID
29
29
  __commit_id__: COMMIT_ID
30
30
 
31
- __version__ = version = '0.4.14'
32
- __version_tuple__ = version_tuple = (0, 4, 14)
31
+ __version__ = version = '0.5.1'
32
+ __version_tuple__ = version_tuple = (0, 5, 1)
33
33
 
34
- __commit_id__ = commit_id = 'g1808b05b1'
34
+ __commit_id__ = commit_id = 'g8a5f488cb'
@@ -64,66 +64,81 @@
64
64
  }
65
65
  },
66
66
  "project": {
67
- "type": "object",
68
- "description": "General information about the Project",
69
- "required": [
70
- "title",
71
- "description",
72
- "taxId",
73
- "centre"
74
- ],
75
- "properties": {
76
- "title": {
77
- "type": "string",
78
- "description": "Title of the project",
79
- "maxLength": 500
80
- },
81
- "description": {
82
- "type": "string",
83
- "description": "Description of the project",
84
- "maxLength": 5000
85
- },
86
- "centre": {
87
- "type": "string",
88
- "description": "Centre, institute or company submitting the project"
89
- },
90
- "taxId": {
91
- "type": "number",
92
- "description": "Taxonomy ID of the species associated with the project (found here: http://www.ncbi.nlm.nih.gov/taxonomy)"
93
- },
94
- "publications": {
95
- "type": "array",
96
- "items": true
97
- },
98
- "parentProject": {
99
- "$ref": "#/definitions/projectAccession"
100
- },
101
- "childProjects": {
102
- "type": "array",
103
- "items": {
104
- "$ref": "#/definitions/projectAccession"
105
- }
106
- },
107
- "peerProjects": {
108
- "type": "array",
109
- "items": {
110
- "$ref": "#/definitions/projectAccession"
67
+ "oneOf": [
68
+ {
69
+ "type": "object",
70
+ "required": [
71
+ "projectAccession"
72
+ ],
73
+ "properties": {
74
+ "projectAccession": {
75
+ "$ref": "#/definitions/projectAccession"
76
+ }
111
77
  }
112
78
  },
113
- "links": {
114
- "type": "array",
115
- "items": {
116
- "type": "string"
79
+ {
80
+ "type": "object",
81
+ "description": "General information about the Project",
82
+ "required": [
83
+ "title",
84
+ "description",
85
+ "taxId",
86
+ "centre"
87
+ ],
88
+ "properties": {
89
+ "title": {
90
+ "type": "string",
91
+ "description": "Title of the project",
92
+ "maxLength": 500
93
+ },
94
+ "description": {
95
+ "type": "string",
96
+ "description": "Description of the project",
97
+ "maxLength": 5000
98
+ },
99
+ "centre": {
100
+ "type": "string",
101
+ "description": "Centre, institute or company submitting the project"
102
+ },
103
+ "taxId": {
104
+ "type": "number",
105
+ "description": "Taxonomy ID of the species associated with the project (found here: http://www.ncbi.nlm.nih.gov/taxonomy)"
106
+ },
107
+ "publications": {
108
+ "type": "array",
109
+ "items": true
110
+ },
111
+ "parentProject": {
112
+ "$ref": "#/definitions/projectAccession"
113
+ },
114
+ "childProjects": {
115
+ "type": "array",
116
+ "items": {
117
+ "$ref": "#/definitions/projectAccession"
118
+ }
119
+ },
120
+ "peerProjects": {
121
+ "type": "array",
122
+ "items": {
123
+ "$ref": "#/definitions/projectAccession"
124
+ }
125
+ },
126
+ "links": {
127
+ "type": "array",
128
+ "items": {
129
+ "type": "string"
130
+ }
131
+ },
132
+ "holdDate": {
133
+ "type": "string",
134
+ "format": "date",
135
+ "examples": [
136
+ "2018-11-13"
137
+ ]
138
+ }
117
139
  }
118
- },
119
- "holdDate": {
120
- "type": "string",
121
- "format": "date",
122
- "examples": [
123
- "2018-11-13"
124
- ]
125
140
  }
126
- }
141
+ ]
127
142
  },
128
143
  "analysis": {
129
144
  "type": "array",
@@ -0,0 +1,139 @@
1
+ #Configuration for defining which fields in which worksheets to be parsed in an Excel file
2
+ #
3
+ #At the top level, "worksheets" defines a list of worksheets (titles) to be parsed.
4
+ #Then for each worksheet, you define which fields to be parsed from this worksheet.
5
+ #These fields are either "required", "optional" and/or header_row. "required" fields must be found
6
+ #in the worksheet so that this worksheet is valid. Otherwise this worksheet is not
7
+ #valid. The field names must match those values in the header row in the worksheet.
8
+ #
9
+ #This is a YAML file
10
+
11
+ # List of worksheets to be parsed.
12
+ worksheets:
13
+ Submitter Details: submitterDetails
14
+ Project: project
15
+ Analysis: analysis
16
+ Sample: sample
17
+ Files: files
18
+
19
+ Submitter Details:
20
+ required:
21
+ Last Name: lastName
22
+ First Name: firstName
23
+ Email Address: email
24
+ Laboratory: laboratory
25
+ Center: centre
26
+ optional:
27
+ Address: address
28
+
29
+ # List of field for Projects
30
+ Project:
31
+ header_row: 3
32
+ optional:
33
+ Project Accession: projectAccession
34
+ Project Title: title
35
+ Description: description
36
+ Center: centre
37
+ Taxonomy ID: taxId
38
+ Publication(s): publications
39
+ Parent Project: parentProject
40
+ Child Project(s): childProjects
41
+ Peer Project(s): peerProjects
42
+ Link(s): links
43
+ Hold Date: holdDate
44
+ Collaborator(s): collaborators
45
+ Strain: strain
46
+ Breed: breed
47
+ Broker: broker
48
+ cast:
49
+ Publication(s): list
50
+ Child Project(s): list
51
+ Peer Project(s): list
52
+ Link(s): list
53
+ Hold Date: date
54
+ Taxonomy ID: integer
55
+
56
+ Analysis:
57
+ required:
58
+ Analysis Title: analysisTitle
59
+ Analysis Alias: analysisAlias
60
+ Description: description
61
+ Experiment Type: experimentType
62
+ Reference: referenceGenome
63
+ optional:
64
+ Reference Fasta Path: referenceFasta
65
+ Platform: platform
66
+ Software: software
67
+ Pipeline Description: pipelineDescriptions
68
+ Imputation: imputation
69
+ Phasing: phasing
70
+ Centre: centre
71
+ Date: date
72
+ Link(s): links
73
+ Run Accession(s): runAccessions
74
+ cast:
75
+ Run Accession(s): list
76
+ Imputation: boolean
77
+ Phasing: boolean
78
+ Date: date
79
+ Software: list
80
+
81
+ Sample:
82
+ header_row: 3
83
+ required:
84
+ Analysis Alias: analysisAlias
85
+ Sample Name in VCF: sampleInVCF
86
+ # Sample property names taken from ERC000011 where present
87
+ # See https://www.ebi.ac.uk/biosamples/schema-store/registry/schemas/ERC000011
88
+ optional:
89
+ Sample Accession: bioSampleAccession
90
+ BioSample Name: bioSampleName
91
+ Title: title
92
+ Description: description
93
+ Unique Name: uniqueName
94
+ Prefix: prefix
95
+ Subject: subject
96
+ Derived From: derivedFrom
97
+ Taxonomy ID: taxId
98
+ Scientific Name: scientific name
99
+ Common Name: commonName
100
+ mating_type: mating_type
101
+ sex: sex
102
+ population: population
103
+ cell_type: cell_type
104
+ dev_stage: dev_stage
105
+ germline: germline
106
+ tissue_lib: tissue_lib
107
+ tissue_type: tissue_type
108
+ bio_material: bio_material
109
+ culture_collection: culture_collection
110
+ specimen_voucher: specimen_voucher
111
+ collected_by: collected_by
112
+ collection_date: collection date
113
+ geographic location (country and/or sea): geographic location (country and/or sea)
114
+ geographic location (region and locality): geographic location (region and locality)
115
+ host: host
116
+ identified_by: identified_by
117
+ isolation_source: isolation_source
118
+ lat_lon: lat_lon
119
+ lab_host: lab_host
120
+ environmental_sample: environmental_sample
121
+ cultivar: cultivar
122
+ ecotype: ecotype
123
+ isolate: isolate
124
+ strain: strain
125
+ sub_species: sub_species
126
+ variety: variety
127
+ sub_strain: sub_strain
128
+ cell_line: cell_line
129
+ serotype: serotype
130
+ serovar: serovar
131
+ cast:
132
+ collection_date: date
133
+ Taxonomy ID: integer
134
+
135
+
136
+ Files:
137
+ required:
138
+ Analysis Alias: analysisAlias
139
+ File Name: fileName
@@ -50,6 +50,8 @@ Project:
50
50
  Peer Project(s): list
51
51
  Link(s): list
52
52
  Hold Date: date
53
+ Taxonomy ID: integer
54
+
53
55
 
54
56
  Analysis:
55
57
  required:
@@ -127,12 +129,12 @@ Sample:
127
129
  serotype: serotype
128
130
  serovar: serovar
129
131
  cast:
130
- Sample ID: string
131
- BioSample Name: string
132
132
  collection_date: date
133
+ Taxonomy ID: integer
134
+
133
135
 
134
136
 
135
137
  Files:
136
138
  required:
137
139
  Analysis Alias: analysisAlias
138
- File Name: fileName
140
+ File Name: fileName
@@ -0,0 +1,5 @@
1
+
2
+ class SubmissionUploadException(Exception):
3
+ def __init__(self, message):
4
+ self.message = message
5
+ super().__init__(self.message)
@@ -1,14 +1,11 @@
1
1
  import sys
2
2
 
3
- import eva_sub_cli
4
- from eva_sub_cli.exceptions.metadata_template_version_exception import MetadataTemplateVersionException, \
5
- MetadataTemplateVersionNotFoundException
6
- from eva_sub_cli.exceptions.submission_not_found_exception import SubmissionNotFoundException
7
- from eva_sub_cli.exceptions.submission_status_exception import SubmissionStatusException
3
+ from requests import HTTPError
4
+
5
+ from eva_sub_cli.exceptions.submission_upload_exception import SubmissionUploadException
8
6
 
9
7
  if not sys.warnoptions:
10
8
  import warnings
11
-
12
9
  warnings.simplefilter("ignore")
13
10
 
14
11
  import logging
@@ -16,9 +13,15 @@ import os
16
13
  from argparse import ArgumentParser
17
14
  from ebi_eva_common_pyutils.logger import logging_config
18
15
 
16
+ import eva_sub_cli
19
17
  from eva_sub_cli import orchestrator
20
- from eva_sub_cli.orchestrator import VALIDATE, SUBMIT, DOCKER, NATIVE
18
+ from eva_sub_cli.exceptions.metadata_template_version_exception import MetadataTemplateVersionException, \
19
+ MetadataTemplateVersionNotFoundException
20
+ from eva_sub_cli.exceptions.submission_status_exception import SubmissionStatusException
21
+ from eva_sub_cli.exceptions.submission_not_found_exception import SubmissionNotFoundException
21
22
  from eva_sub_cli.file_utils import is_submission_dir_writable, DirLockError, DirLock
23
+ from eva_sub_cli.orchestrator import VALIDATE, SUBMIT, DOCKER, NATIVE
24
+ from eva_sub_cli.validators.validator import ALL_VALIDATION_TASKS
22
25
 
23
26
 
24
27
  def validate_command_line_arguments(args, argparser):
@@ -93,6 +96,9 @@ def parse_args(cmd_line_args):
93
96
  help='Select a task to perform (default SUBMIT). VALIDATE will run the validation'
94
97
  ' regardless of the outcome of previous runs. SUBMIT will run validate only if'
95
98
  ' the validation was not performed successfully before and then run the submission.')
99
+ argparser.add_argument('--validation_tasks', nargs='+', choices=ALL_VALIDATION_TASKS, default=ALL_VALIDATION_TASKS,
100
+ type=str.lower, help='Select only a subset of the validation tasks to run. Note that all '
101
+ 'tasks need to be successful for the validation to pass')
96
102
  argparser.add_argument('--executor', choices=[DOCKER, NATIVE], default=NATIVE, type=str.lower,
97
103
  help='Select the execution type for running validation (default native)')
98
104
  credential_group = argparser.add_argument_group('Credentials', 'Specify the ENA Webin credentials you want to use '
@@ -125,7 +131,6 @@ def main():
125
131
 
126
132
  try:
127
133
  # lock the submission directory
128
-
129
134
  with DirLock(os.path.join(args.submission_dir)) as lock:
130
135
  # Create the log file
131
136
  logging_config.add_file_handler(os.path.join(args.submission_dir, 'eva_submission.log'), logging.DEBUG)
@@ -151,4 +156,15 @@ def main():
151
156
  except MetadataTemplateVersionNotFoundException as mte:
152
157
  print(mte)
153
158
  exit_status = 70
159
+ except SubmissionUploadException as sue:
160
+ print(sue)
161
+ exit_status = 71
162
+ except HTTPError as http_err:
163
+ print(http_err)
164
+ if http_err.response is not None and http_err.response.text:
165
+ print(http_err.response.text)
166
+ exit_status = 72
167
+ except Exception as ex:
168
+ print(ex)
169
+ exit_status = 73
154
170
  return exit_status
@@ -103,9 +103,7 @@ class XlsxParser:
103
103
  @staticmethod
104
104
  def cast_value(value, type_name):
105
105
  # Do not cast None values
106
- if type_name and value is not None:
107
- if type_name == 'string':
108
- return str(value)
106
+ if value is not None:
109
107
  if type_name == 'boolean':
110
108
  return str(value).lower() in ['true', '1', "'1'", 't', 'y', 'yes']
111
109
  if type_name == 'list':
@@ -114,6 +112,10 @@ class XlsxParser:
114
112
  return [v for v in split_values if v]
115
113
  if type_name == 'date':
116
114
  return XlsxParser.serialize(value)
115
+ if type_name == 'integer':
116
+ return int(value)
117
+ # Default any value is cast to string
118
+ return str(value)
117
119
  return value
118
120
 
119
121
  @staticmethod
@@ -179,15 +181,20 @@ class XlsxParser:
179
181
  return rows
180
182
 
181
183
  def get_project_json_data(self):
184
+ json_key = self.xlsx_conf[WORKSHEETS_KEY_NAME][PROJECT]
185
+
182
186
  project_data = self.get_rows()
183
187
  if len(project_data) > 1:
184
188
  logger.warning(f"{PROJECT} worksheet expects a single row of info but more than one found in the file. "
185
189
  f"Only the first row's data will be taken into consideration")
190
+ elif len(project_data) == 0:
191
+ logger.error(f"{PROJECT} worksheet expects a single row of info but no info is found in the file.")
192
+ self.add_error(f"{PROJECT} worksheet expects a single row of info but no info is found in the file.")
193
+ return {json_key: {}}
186
194
 
187
195
  first_row = project_data[0]
188
196
  first_row.pop('row_num')
189
197
 
190
- json_key = self.xlsx_conf[WORKSHEETS_KEY_NAME][PROJECT]
191
198
  json_value = {self.translate_header(PROJECT, k): v for k, v in first_row.items() if v is not None}
192
199
  return {json_key: json_value}
193
200
 
@@ -1,6 +1,6 @@
1
1
 
2
2
  {% macro file_validation_report(validation_results, file_name) -%}
3
- {% for check_type, check_per_file in validation_results.items() %}
3
+ {% for check_type, check_per_file in validation_results.items() if check_type not in ["trim_down", "version"] %}
4
4
  {% set result = check_per_file.get(file_name, {}) %}
5
5
  {% if check_type == "assembly_check" %}
6
6
  {{ assembly_check(result) }}
@@ -20,7 +20,7 @@
20
20
  {% set row_class = "report-section fail collapsible" %}
21
21
  {% elif error_count > 0 %}
22
22
  {% set expand_icon = "&#9654;" %}
23
- {% set icon = "&#10060;" %}
23
+ {% set icon = "&#10071;" %}
24
24
  {% set row_class = "report-section warn collapsible" %}
25
25
  {% else %}
26
26
  {% set icon = "&#10004;" %}
@@ -6,6 +6,13 @@
6
6
  {% from 'metadata_validation.html' import metadata_validation_report %}
7
7
  {% from 'shallow_validation.html' import optional_shallow_validation_report %}
8
8
 
9
+ {% macro validation_not_run_yet_message(text="Process not run yet") %}
10
+ {% set icon = "&#9208;" %} {# pause symbol ⏸ #}
11
+ {% set row_class = "report-section info" %}
12
+ {% set expand_icon = "" %}
13
+ <div class='{{ row_class }}'><span class="expand_icon">{{ expand_icon }}</span> {{ icon }} {{ text }}</div>
14
+ {% endmacro %}
15
+
9
16
  <html lang="EN">
10
17
  <head>
11
18
  <meta charset="UTF-8">
@@ -63,7 +70,12 @@
63
70
  Ensures that required fields are present and values are formatted correctly.
64
71
  For requirements, please refer to the <a href="https://www.ebi.ac.uk/eva/?Submit-Data" target=”_blank”>EVA website</a>.
65
72
  </div>
66
- {{ metadata_validation_report(validation_results) }}
73
+ {% set run_status = validation_results.get('metadata_check', {}).get('run_status', '') %}
74
+ {% if run_status %}
75
+ {{ metadata_validation_report(validation_results) }}
76
+ {% else %}
77
+ {{ validation_not_run_yet_message() }}
78
+ {% endif %}
67
79
  </section>
68
80
 
69
81
  <section>
@@ -72,12 +84,17 @@
72
84
  Checks whether each file is compliant with the <a href="http://samtools.github.io/hts-specs/VCFv4.4.pdf" target=”_blank”>VCF specification</a>.
73
85
  Also checks whether the variants' reference alleles match against the reference assembly.
74
86
  </div>
75
- {% for file_name in vcf_files %}
76
- {% if file_name != "pass"%}
77
- <h3>{{ file_name }}</h3>
78
- {{ file_validation_report(validation_results, file_name) }}
79
- {% endif %}
80
- {% endfor %}
87
+ {% set run_status = validation_results.get('vcf_check', {}).get('run_status', '') %}
88
+ {% if run_status %}
89
+ {% for file_name in vcf_files %}
90
+ {% if file_name != "pass"%}
91
+ <h3>{{ file_name }}</h3>
92
+ {{ file_validation_report(validation_results, file_name) }}
93
+ {% endif %}
94
+ {% endfor %}
95
+ {% else %}
96
+ {{ validation_not_run_yet_message() }}
97
+ {% endif %}
81
98
  </section>
82
99
 
83
100
  <section>
@@ -85,7 +102,13 @@
85
102
  <div class="description">
86
103
  Checks whether information in the metadata is concordant with that contained in the VCF files, in particular sample names.
87
104
  </div>
88
- {{ sample_name_check_report(validation_results)}}
105
+ {% set run_status = validation_results.get('sample_check', {}).get('run_status', '') %}
106
+ {% if run_status %}
107
+ {{ sample_name_check_report(validation_results)}}
108
+ {% else %}
109
+ {{ validation_not_run_yet_message() }}
110
+
111
+ {% endif %}
89
112
  </section>
90
113
 
91
114
  <section>
@@ -94,12 +117,17 @@
94
117
  Checks that the reference sequences in the FASTA file used to call the variants are accessioned in INSDC.
95
118
  Also checks if the reference assembly accession in the metadata matches the one determined from the FASTA file.
96
119
  </div>
97
- {% for file_name in fasta_files %}
98
- {% if file_name != "pass"%}
99
- <h3>{{ file_name }}</h3>
100
- {{ fasta_check_report(validation_results, file_name) }}
101
- {% endif %}
102
- {% endfor %}
120
+ {% set run_status = validation_results.get('fasta_check', {}).get('run_status', '') %}
121
+ {% if run_status %}
122
+ {% for file_name in fasta_files %}
123
+ {% if file_name != "pass"%}
124
+ <h3>{{ file_name }}</h3>
125
+ {{ fasta_check_report(validation_results, file_name) }}
126
+ {% endif %}
127
+ {% endfor %}
128
+ {% else %}
129
+ {{ validation_not_run_yet_message() }}
130
+ {% endif %}
103
131
  </section>
104
132
 
105
133
  <script>
@@ -2,7 +2,7 @@
2
2
  {% macro optional_shallow_validation_report(validation_results) -%}
3
3
  {% set results = validation_results.get('shallow_validation', {}) %}
4
4
 
5
- {% if results.get('required') %}
5
+ {% if validation_results.get('trim_down') %}
6
6
  <section>
7
7
  <div class="report-section fail collapsible"> <span class="expand_icon">&#9654;</span>
8
8
  &#10060; <b>You requested to run the shallow validation, please run full validation before submitting the data</b>