eva-sub-cli 0.4.10__tar.gz → 0.4.11__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/.github/workflows/trigger-eva-sub-cli-integration-tests.yml +2 -2
  2. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/PKG-INFO +2 -1
  3. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/docker/Dockerfile +1 -1
  4. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/_version.py +2 -2
  5. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/etc/eva_schema.json +9 -1
  6. eva_sub_cli-0.4.11/eva_sub_cli/executables/evidence_type_checker.py +51 -0
  7. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/executables/samples_checker.py +33 -12
  8. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/file_utils.py +80 -2
  9. eva_sub_cli-0.4.11/eva_sub_cli/jinja_templates/html/project_details.html +65 -0
  10. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/jinja_templates/html/report.html +1 -1
  11. eva_sub_cli-0.4.11/eva_sub_cli/jinja_templates/text/project_details.txt +37 -0
  12. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/jinja_templates/text/report.txt +1 -1
  13. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/metadata.py +4 -0
  14. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/nextflow/validation.nf +21 -0
  15. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/report.py +15 -11
  16. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/semantic_metadata.py +13 -0
  17. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/validators/docker_validator.py +1 -1
  18. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/validators/validator.py +67 -9
  19. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli.egg-info/PKG-INFO +2 -1
  20. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli.egg-info/SOURCES.txt +3 -0
  21. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli.egg-info/entry_points.txt +1 -0
  22. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli.egg-info/requires.txt +1 -0
  23. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/pyproject.toml +1 -0
  24. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/requirements.txt +2 -1
  25. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/validation_reports/expected_report_metadata_json.html +19 -0
  26. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/validation_reports/expected_report_metadata_json.txt +6 -0
  27. eva_sub_cli-0.4.10/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.html → eva_sub_cli-0.4.11/tests/resources/validation_reports/expected_report_metadata_xlsx.html +19 -24
  28. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/validation_reports/expected_report_metadata_xlsx.txt +6 -0
  29. eva_sub_cli-0.4.10/tests/resources/validation_reports/expected_report_metadata_xlsx.html → eva_sub_cli-0.4.11/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.html +43 -0
  30. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.txt +6 -0
  31. eva_sub_cli-0.4.11/tests/resources/validation_reports/validation_output/other_validations/evidence_type_checker.yml +3 -0
  32. eva_sub_cli-0.4.11/tests/test_evidence_type_checker.py +66 -0
  33. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/test_report.py +28 -2
  34. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/test_semantic_metadata.py +52 -0
  35. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/test_validator.py +29 -11
  36. eva_sub_cli-0.4.10/eva_sub_cli/jinja_templates/html/project_details.html +0 -37
  37. eva_sub_cli-0.4.10/eva_sub_cli/jinja_templates/text/project_details.txt +0 -21
  38. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/.github/workflows/publish.yml +0 -0
  39. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/.github/workflows/publish_docs.yml +0 -0
  40. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/.github/workflows/publish_packages.yml +0 -0
  41. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/.github/workflows/tests.yml +0 -0
  42. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/.gitignore +0 -0
  43. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/LICENSE +0 -0
  44. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/README.md +0 -0
  45. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/conda/build.sh +0 -0
  46. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/conda/meta.yaml +0 -0
  47. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/docs/README.md +0 -0
  48. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/docs/_config.yml +0 -0
  49. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/docs/how_to_submit.md +0 -0
  50. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/docs/input_file_overview.md +0 -0
  51. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/docs/installation.md +0 -0
  52. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/docs/using_nextflow.md +0 -0
  53. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/docs/validation_overview.md +0 -0
  54. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/__init__.py +0 -0
  55. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/auth.py +0 -0
  56. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/date_utils.py +0 -0
  57. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/etc/EVA_Submission_Example.xlsx +0 -0
  58. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/etc/EVA_Submission_template.xlsx +0 -0
  59. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/etc/eva-biosamples.json +0 -0
  60. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/etc/eva_logo.png +0 -0
  61. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/etc/spreadsheet2json_conf.yaml +0 -0
  62. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/exceptions/__init__.py +0 -0
  63. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/exceptions/invalid_file_type_exception.py +0 -0
  64. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/exceptions/metadata_template_version_exception.py +0 -0
  65. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/exceptions/submission_not_found_exception.py +0 -0
  66. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/exceptions/submission_status_exception.py +0 -0
  67. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/executables/__init__.py +0 -0
  68. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/executables/check_fasta_insdc.py +0 -0
  69. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/executables/check_metadata_semantics.py +0 -0
  70. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/executables/cli.py +0 -0
  71. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/executables/trim_down.py +0 -0
  72. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/executables/xlsx2json.py +0 -0
  73. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/jinja_templates/html/fasta_check.html +0 -0
  74. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/jinja_templates/html/file_validation.html +0 -0
  75. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/jinja_templates/html/metadata_validation.html +0 -0
  76. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/jinja_templates/html/sample_name_check.html +0 -0
  77. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/jinja_templates/html/shallow_validation.html +0 -0
  78. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/jinja_templates/text/fasta_check.txt +0 -0
  79. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/jinja_templates/text/file_validation.txt +0 -0
  80. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/jinja_templates/text/metadata_validation.txt +0 -0
  81. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/jinja_templates/text/sample_name_check.txt +0 -0
  82. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/jinja_templates/text/shallow_validation.txt +0 -0
  83. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/orchestrator.py +0 -0
  84. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/submission_ws.py +0 -0
  85. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/submit.py +0 -0
  86. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/validators/__init__.py +0 -0
  87. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/validators/native_validator.py +0 -0
  88. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli/validators/validation_results_parsers.py +0 -0
  89. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli.egg-info/dependency_links.txt +0 -0
  90. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/eva_sub_cli.egg-info/top_level.txt +0 -0
  91. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/pytest.ini +0 -0
  92. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/setup.cfg +0 -0
  93. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/setup.py +0 -0
  94. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/build_and_test_docker_locally.py +0 -0
  95. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/ignore_test_nextflow_validation.yml +0 -0
  96. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/EVA_Submission_Docker_Test.xlsx +0 -0
  97. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/EVA_Submission_test.json +0 -0
  98. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/EVA_Submission_test.xlsx +0 -0
  99. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/EVA_Submission_test_fails.xlsx +0 -0
  100. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/EVA_Submission_test_version_missing.xlsx +0 -0
  101. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/EVA_Submission_test_with_asm_report.json +0 -0
  102. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/assembly_check/invalid.vcf.assembly_check.log +0 -0
  103. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/assembly_check/invalid.vcf.text_assembly_report.txt +0 -0
  104. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/assembly_reports/input_passed.txt +0 -0
  105. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/fasta_files/Saccharomyces_cerevisiae_I.fa +0 -0
  106. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/fasta_files/input_passed.fa +0 -0
  107. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/fasta_files/multi_seq.fa +0 -0
  108. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/metadata_with_filename.json +0 -0
  109. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/sample_checker/example1.vcf.gz +0 -0
  110. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/sample_checker/example2.vcf +0 -0
  111. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/sample_checker/example3.vcf +0 -0
  112. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/sample_checker/metadata.json +0 -0
  113. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/validation_reports/validation_output/assembly_check/input_passed.vcf.assembly_check.log +0 -0
  114. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/validation_reports/validation_output/assembly_check/input_passed.vcf.text_assembly_report.1680102476700.txt +0 -0
  115. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/validation_reports/validation_output/assembly_check/input_passed.vcf.valid_assembly_report.1680102476700.txt +0 -0
  116. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/validation_reports/validation_output/other_validations/input_passed.fa_check.yml +0 -0
  117. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/validation_reports/validation_output/other_validations/metadata_conversion_errors.yml +0 -0
  118. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/validation_reports/validation_output/other_validations/metadata_semantic_check.yml +0 -0
  119. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/validation_reports/validation_output/other_validations/metadata_validation.txt +0 -0
  120. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/validation_reports/validation_output/other_validations/sample_checker.yml +0 -0
  121. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/validation_reports/validation_output/vcf_format/input_passed.vcf.errors.1680102476854.db +0 -0
  122. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/validation_reports/validation_output/vcf_format/input_passed.vcf.errors.1680102476854.txt +0 -0
  123. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/validation_reports/validation_output/vcf_format/input_passed.vcf.vcf_format.log +0 -0
  124. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/vcf_files/example2.vcf.gz +0 -0
  125. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/resources/vcf_files/input_passed.vcf +0 -0
  126. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/test_auth.py +0 -0
  127. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/test_check_fasta_insdc.py +0 -0
  128. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/test_cli.py +0 -0
  129. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/test_docker_validator.py +0 -0
  130. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/test_file_utils.py +0 -0
  131. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/test_metadata.py +0 -0
  132. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/test_orchestrator.py +0 -0
  133. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/test_samples_checker.py +0 -0
  134. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/test_submit.py +0 -0
  135. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/test_trim_down.py +0 -0
  136. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/test_utils.py +0 -0
  137. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/test_validaton_results_parsers.py +0 -0
  138. {eva_sub_cli-0.4.10 → eva_sub_cli-0.4.11}/tests/test_xlsx2json.py +0 -0
@@ -2,7 +2,7 @@ name: Trigger Eva Sub Cli Integration Tests
2
2
 
3
3
  on:
4
4
  push:
5
- branches: [main, test-trigger-integration-tests]
5
+ branches: [main]
6
6
 
7
7
  jobs:
8
8
  trigger-integration-tests:
@@ -11,6 +11,6 @@ jobs:
11
11
  - name: Trigger eva sub cli integration tests
12
12
  uses: peter-evans/repository-dispatch@v2
13
13
  with:
14
- token: ${{ secrets.TRIGGER_INTEGRATION_TESTS }}
14
+ token: ${{ secrets.WORKFLOW_ACCESS_TOKEN }}
15
15
  repository: EBIvariation/eva-integration-tests
16
16
  event-type: trigger-eva-sub-cli-tests
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: eva_sub_cli
3
- Version: 0.4.10
3
+ Version: 0.4.11
4
4
  Summary: EBI EVA - validation and submission command line tool
5
5
  Classifier: Development Status :: 5 - Production/Stable
6
6
  Classifier: Environment :: Console
@@ -24,6 +24,7 @@ Requires-Dist: jinja2==3.*,>=3.0.0
24
24
  Requires-Dist: jsonschema==4.*
25
25
  Requires-Dist: openpyxl==3.*
26
26
  Requires-Dist: pyyaml==6.*
27
+ Requires-Dist: pysam==0.*,>=0.23.3
27
28
  Requires-Dist: requests==2.*
28
29
  Requires-Dist: packaging==24.*
29
30
  Requires-Dist: setuptools-scm
@@ -1,6 +1,6 @@
1
1
  FROM python:3.10
2
2
 
3
- ENV vcf_validator_version=0.10.0
3
+ ENV vcf_validator_version=0.10.2
4
4
  ENV NXF_VER=22.10.6
5
5
 
6
6
  WORKDIR /opt
@@ -17,5 +17,5 @@ __version__: str
17
17
  __version_tuple__: VERSION_TUPLE
18
18
  version_tuple: VERSION_TUPLE
19
19
 
20
- __version__ = version = '0.4.10'
21
- __version_tuple__ = version_tuple = (0, 4, 10)
20
+ __version__ = version = '0.4.11'
21
+ __version_tuple__ = version_tuple = (0, 4, 11)
@@ -168,6 +168,14 @@
168
168
  "type": "string",
169
169
  "description": "Reference against which the analysis was performed. ENA or NCBI accession (starting with GCA) accepted."
170
170
  },
171
+ "evidenceType": {
172
+ "type": "string",
173
+ "description": "The type of evidence (allele frequencies or genotypes) provided in the VCF files of the analysis",
174
+ "enum": [
175
+ "genotype",
176
+ "allele_frequency"
177
+ ]
178
+ },
171
179
  "referenceFasta": {
172
180
  "type": "string",
173
181
  "description": "The path to the reference in fasta format that was used to create the VCF. (This is a local path)"
@@ -366,7 +374,7 @@
366
374
  "projectAccession": {
367
375
  "type": "string",
368
376
  "description": "ENA project accession",
369
- "pattern": "^PRJ(EB|NA)\\d+$"
377
+ "pattern": "^PRJ(E|D|N)[A-Z][0-9]+$"
370
378
  }
371
379
  }
372
380
  }
@@ -0,0 +1,51 @@
1
+ import argparse
2
+
3
+ import yaml
4
+ from ebi_eva_common_pyutils.logger import logging_config
5
+
6
+ from eva_sub_cli.file_utils import detect_vcf_evidence_type
7
+ from eva_sub_cli.metadata import EvaMetadataJson
8
+
9
+ logger = logging_config.get_logger(__name__)
10
+
11
+
12
+ def check_evidence_type(metadata_json, output_yaml):
13
+ metadata = EvaMetadataJson(metadata_json)
14
+ results_per_analysis_alias = {}
15
+
16
+ for analysis_alias, vcf_file_set in metadata.files_per_analysis.items():
17
+ results_per_analysis_alias[analysis_alias] = {
18
+ 'evidence_type': None,
19
+ 'errors': None
20
+ }
21
+ vcf_files = list(vcf_file_set)
22
+ evidence_types_for_vcf_files = [detect_vcf_evidence_type(vcf_file) for vcf_file in vcf_files]
23
+
24
+ if len(set(evidence_types_for_vcf_files)) == 1 and None not in evidence_types_for_vcf_files:
25
+ results_per_analysis_alias[analysis_alias]['evidence_type'] = set(evidence_types_for_vcf_files).pop()
26
+ elif None in evidence_types_for_vcf_files:
27
+ indices = [i for i, x in enumerate(evidence_types_for_vcf_files) if x is None]
28
+ results_per_analysis_alias[analysis_alias][
29
+ 'errors'] = f'VCF file evidence type could not be determined: {", ".join([vcf_files[i] for i in indices])}'
30
+ else:
31
+ results_per_analysis_alias[analysis_alias][
32
+ 'errors'] = f'Multiple evidence types found: {", ".join(set(evidence_types_for_vcf_files))}'
33
+
34
+ write_result_yaml(output_yaml, results_per_analysis_alias)
35
+
36
+
37
+ def write_result_yaml(output_yaml, results_per_analysis_alias):
38
+ with open(output_yaml, 'w') as open_yaml:
39
+ yaml.safe_dump(data=results_per_analysis_alias, stream=open_yaml, default_flow_style=False)
40
+
41
+
42
+ def main():
43
+ arg_parser = argparse.ArgumentParser(description='Check the evidence type in the VCF files for all analysis')
44
+ arg_parser.add_argument('--metadata_json', required=True, dest='metadata_json',
45
+ help='EVA metadata json file')
46
+ arg_parser.add_argument('--output_yaml', required=True, dest='output_yaml',
47
+ help='Path to the location of the results')
48
+
49
+ args = arg_parser.parse_args()
50
+ logging_config.add_stdout_handler()
51
+ check_evidence_type(args.metadata_json, args.output_yaml)
@@ -5,7 +5,7 @@ from ebi_eva_common_pyutils.logger import logging_config
5
5
 
6
6
  import yaml
7
7
 
8
- from eva_sub_cli.file_utils import open_gzip_if_required
8
+ from eva_sub_cli.file_utils import open_gzip_if_required, detect_vcf_evidence_type
9
9
  from eva_sub_cli.metadata import EvaMetadataJson
10
10
 
11
11
  logger = logging_config.get_logger(__name__)
@@ -64,20 +64,41 @@ def compare_all_analysis(metadata, files_per_analysis):
64
64
  file_path: get_samples_from_vcf(file_path)
65
65
  for file_path in files_per_analysis.get(analysis_alias, [])
66
66
  }
67
- (
68
- has_difference, more_per_submitted_files_metadata,
69
- more_submitted_files_metadata, more_metadata_submitted_files
70
- ) = compare_names_in_files_and_samples(sample_name_in_analysis, sample_name_per_file)
71
- overall_differences = overall_differences or has_difference
72
- results_per_analysis_alias[analysis_alias] = {
73
- 'difference': has_difference,
74
- 'more_per_submitted_files_metadata': more_per_submitted_files_metadata,
75
- 'more_submitted_files_metadata': more_submitted_files_metadata,
76
- 'more_metadata_submitted_files': more_metadata_submitted_files
77
- }
67
+
68
+ if need_to_check_samples(sample_name_per_file):
69
+ (
70
+ has_difference, more_per_submitted_files_metadata,
71
+ more_submitted_files_metadata, more_metadata_submitted_files
72
+ ) = compare_names_in_files_and_samples(sample_name_in_analysis, sample_name_per_file)
73
+ overall_differences = overall_differences or has_difference
74
+ results_per_analysis_alias[analysis_alias] = {
75
+ 'difference': has_difference,
76
+ 'more_per_submitted_files_metadata': more_per_submitted_files_metadata,
77
+ 'more_submitted_files_metadata': more_submitted_files_metadata,
78
+ 'more_metadata_submitted_files': more_metadata_submitted_files
79
+ }
80
+ else:
81
+ results_per_analysis_alias[analysis_alias] = {
82
+ 'difference': False,
83
+ 'more_per_submitted_files_metadata': [],
84
+ 'more_submitted_files_metadata': [],
85
+ 'more_metadata_submitted_files': []
86
+ }
87
+
88
+
78
89
  return overall_differences, results_per_analysis_alias
79
90
 
80
91
 
92
+ def need_to_check_samples(sample_name_per_file):
93
+ no_samples_in_vcf = all(len(v) == 0 for v in sample_name_per_file.values())
94
+ if no_samples_in_vcf:
95
+ evidence_types_for_vcf_files = [detect_vcf_evidence_type(vcf_file) for vcf_file in sample_name_per_file.keys()]
96
+ if set(evidence_types_for_vcf_files) == {'allele_frequency'}:
97
+ return False
98
+
99
+ return True
100
+
101
+
81
102
  def associate_vcf_path_with_analysis(metadata, vcf_files):
82
103
  """
83
104
  Match the files names associated with analysis provided in the metadata with the file path given on the command
@@ -5,6 +5,11 @@ import shutil
5
5
  import time
6
6
  from itertools import groupby
7
7
 
8
+ import pysam
9
+ from ebi_eva_common_pyutils.logger import logging_config
10
+
11
+ logger = logging_config.get_logger(__name__)
12
+
8
13
 
9
14
  def resolve_single_file_path(file_path):
10
15
  files = glob.glob(file_path)
@@ -23,12 +28,86 @@ def is_submission_dir_writable(submission_dir):
23
28
  return False
24
29
  return True
25
30
 
31
+
26
32
  def is_vcf_file(file_path):
27
33
  if file_path:
28
34
  file_path = file_path.strip().lower()
29
35
  return file_path.endswith('.vcf') or file_path.endswith('.vcf.gz')
30
36
  return False
31
37
 
38
+
39
+ def detect_vcf_evidence_type(vcf_file):
40
+ """
41
+ Detect the type of evidence (genotype aggregation) provided in the VCF file by checking the first 10 data lines
42
+ The evidence type is determined to be "genotype" (meaning genotype are all present) if a GT field can be found in
43
+ all the samples. It is determined to be "allele_frequency" if it is not "genotype" and an AF field or AN and AC fields are found
44
+ in every line checked.
45
+ Otherwise, it returns None meaning that the evidence type could not be determined.
46
+ """
47
+
48
+ try:
49
+ samples, af_in_info, gt_in_format = _assess_vcf_evidence_type_with_pysam(vcf_file)
50
+ except Exception:
51
+ logger.error(f"Pysam Failed to open and read {vcf_file}")
52
+ try:
53
+ samples, af_in_info, gt_in_format = _assess_vcf_evidence_type_manual(vcf_file)
54
+ except Exception:
55
+ logger.error(f"Manual parsing Failed to open or read {vcf_file}")
56
+ return None
57
+ if len(samples) > 0 and gt_in_format:
58
+ return 'genotype'
59
+ elif len(samples) == 0 and af_in_info:
60
+ return 'allele_frequency'
61
+ else:
62
+ logger.error(f'Aggregation type could not be detected for {vcf_file}')
63
+ return None
64
+
65
+
66
+ def _assess_vcf_evidence_type_with_pysam(vcf_file):
67
+ with pysam.VariantFile(vcf_file, 'r') as vcf_in:
68
+ samples = list(vcf_in.header.samples)
69
+ # check that the first 10 lines have genotypes for all the samples present and if they have allele frequency
70
+ nb_line_checked = 0
71
+ max_line_check = 10
72
+ gt_in_format = True
73
+ af_in_info = True
74
+ for vcf_rec in vcf_in:
75
+ gt_in_format = gt_in_format and all('GT' in vcf_rec.samples.get(sample, {}) for sample in samples)
76
+ af_in_info = af_in_info and ('AF' in vcf_rec.info or ('AC' in vcf_rec.info and 'AN' in vcf_rec.info))
77
+ nb_line_checked += 1
78
+ if nb_line_checked >= max_line_check:
79
+ break
80
+ return samples, af_in_info, gt_in_format
81
+
82
+
83
+ def _assess_vcf_evidence_type_manual(vcf_file):
84
+ try:
85
+ if vcf_file.endswith('.gz'):
86
+ open_file = gzip.open(vcf_file, 'rt')
87
+ else:
88
+ open_file = open(vcf_file, 'r')
89
+
90
+ nb_line_checked = 0
91
+ max_line_check = 10
92
+ gt_in_format = True
93
+ af_in_info = True
94
+ samples = []
95
+ for line in open_file:
96
+ sp_line = line.strip().split('\t')
97
+ if line.startswith('#CHROM'):
98
+ if len(sp_line) > 9:
99
+ samples = sp_line[9:]
100
+ if not line.startswith('#'):
101
+ gt_in_format = gt_in_format and len(sp_line) > 8 and 'GT' in sp_line[8]
102
+ af_in_info = af_in_info and (
103
+ sp_line[7].find('AF=') or (sp_line[7].find('AC=') and sp_line[7].find('AN=')))
104
+ if nb_line_checked >= max_line_check:
105
+ break
106
+ return samples, af_in_info, gt_in_format
107
+ finally:
108
+ open_file.close()
109
+
110
+
32
111
  def backup_file_or_directory(file_name, max_backups=None):
33
112
  """
34
113
  Rename a file or directory by adding a '.1' at the end. If the '.1' file exists it move it to a '.2' and so on.
@@ -83,7 +162,6 @@ class DirLockError(Exception):
83
162
 
84
163
 
85
164
  class DirLock(object):
86
-
87
165
  _SPIN_PERIOD_SECONDS = 0.05
88
166
 
89
167
  def __init__(self, dirname, timeout=3):
@@ -119,4 +197,4 @@ class DirLock(object):
119
197
  return self
120
198
 
121
199
  def __exit__(self, type_, value, traceback):
122
- self.release()
200
+ self.release()
@@ -0,0 +1,65 @@
1
+ {% macro project_details_report(validation_results, project_title, validation_date, submission_dir, consent_statement_required, vcf_fasta_analysis_mapping) -%}
2
+ <div class="report-section info">
3
+ {% if project_title %}
4
+ <p><b>Project Title:</b> {{ project_title }}</p>
5
+ {% else %}
6
+ <p><b>Project Title:</b> Not Found</p>
7
+ {% endif %}
8
+ <p><b>Validation Date:</b> {{ validation_date }}</p>
9
+ <p><b>Submission Directory:</b> {{ submission_dir }}</p>
10
+ {% if consent_statement_required %}
11
+ <br>
12
+ <p><b>Note for human genotype data:</b></p> <p>You will need to provide a signed copy of our <a href="https://docs.google.com/document/d/1UaRmimAe919IZYIH55mDXMe-4sHsRVqaP4moqx2IYE4" target="_blank">Consent Statement</a> via email to <a href="mailto:eva-helpdesk@ebi.ac.uk">eva-helpdesk@ebi.ac.uk</a> after submission.
13
+ Your submission will only be processed after receiving the consent statement. For more info, please see our <a href="https://www.ebi.ac.uk/eva/?Help#submissionPanel&link=consent-statement-for-human-genotype-data" target="_blank">help</a> section.</p>
14
+ {% endif %}
15
+ </div>
16
+
17
+
18
+ {% if vcf_fasta_analysis_mapping %}
19
+ {% set row_class = "report-section info collapsible" %}
20
+ {% set expand_icon = "&#9654;" %}
21
+ <div class='{{ row_class }}'><span class="expand_icon">{{ expand_icon }}</span> Files mapping</div>
22
+ <div class="no-show">
23
+ <table>
24
+ <tr>
25
+ <th><strong>VCF File</strong></th>
26
+ <th><strong>Fasta File</strong></th>
27
+ <th><strong>Analysis</strong></th>
28
+ </tr>
29
+ {% for mapping in vcf_fasta_analysis_mapping %}
30
+ <tr>
31
+ <td>{{ mapping.vcf_file }}</td>
32
+ <td>{{ mapping.fasta_file }}</td>
33
+ <td>{{ mapping.analysis }}</td>
34
+ </tr>
35
+ {% endfor %}
36
+ </table>
37
+ </div>
38
+ </div>
39
+ {% endif %}
40
+
41
+
42
+ {% set evidence_type_check_results = validation_results.get('evidence_type_check', {}) %}
43
+ {% set evidence_check_type_pass = evidence_type_check_results.get('pass', true) %}
44
+ {% if not evidence_check_type_pass %}
45
+ <div class="report-section fail collapsible"><span class="expand_icon">&#9654;</span>&#10060; Evidence Type Check Failures </div>
46
+ <div class="error-list">
47
+ <table>
48
+ <tr>
49
+ <th>Analysis Alias</th><th>Errors</th>
50
+ </tr>
51
+ {% for alias, alias_result in evidence_type_check_results.items() if alias != 'pass' %}
52
+ {% if alias_result.evidence_type is none and alias_result.errors %}
53
+ <tr>
54
+ <td><strong>{{ alias }} </strong></td>
55
+ <td>{{ alias_result.errors }}</td>
56
+ </tr>
57
+ {% endif %}
58
+ {% endfor %}
59
+ </table>
60
+ </div>
61
+ {% endif %}
62
+
63
+
64
+
65
+ {%- endmacro %}
@@ -54,7 +54,7 @@
54
54
  <div class="description">
55
55
  General details about the project
56
56
  </div>
57
- {{ project_details_report(project_title, validation_date, submission_dir, vcf_fasta_analysis_mapping) }}
57
+ {{ project_details_report(validation_results, project_title, validation_date, submission_dir, consent_statement_required, vcf_fasta_analysis_mapping) }}
58
58
  </section>
59
59
 
60
60
  <section>
@@ -0,0 +1,37 @@
1
+ {% macro project_details_report(validation_results, project_title, validation_date, submission_dir, consent_statement_required, vcf_fasta_analysis_mapping) -%}
2
+
3
+ {% if project_title %}
4
+ Project Title: {{ project_title }}
5
+ {% else %}
6
+ Project Title: Not Found
7
+ {% endif %}
8
+ Validation Date: {{ validation_date }}
9
+ Submission Directory: {{ submission_dir }}
10
+ {% if consent_statement_required %}
11
+ Note for human genotype data:
12
+ You will need to provide a signed copy of our Consent Statement (https://docs.google.com/document/d/1UaRmimAe919IZYIH55mDXMe-4sHsRVqaP4moqx2IYE4) via email to eva-helpdesk@ebi.ac.uk after submission.
13
+ Your submission will only be processed after receiving the consent statement.For more info, please see our help section (https://www.ebi.ac.uk/eva/?Help#submissionPanel&link=consent-statement-for-human-genotype-data).
14
+ {% endif %}
15
+
16
+ {% if vcf_fasta_analysis_mapping %}
17
+ Files mapping:
18
+ {% for mapping in vcf_fasta_analysis_mapping %}
19
+ ---
20
+ VCF File: {{ mapping.vcf_file }}
21
+ Fasta File: {{ mapping.fasta_file }}
22
+ Analysis: {{ mapping.analysis }}
23
+ {% endfor %}
24
+ {% endif %}
25
+
26
+ {% set evidence_type_check_results = validation_results.get('evidence_type_check', {}) %}
27
+ {% set evidence_check_type_pass = evidence_type_check_results.get('pass', true) %}
28
+ {% if not evidence_check_type_pass %}
29
+ Evidence Type Check Failures:
30
+ {% for alias, alias_result in evidence_type_check_results.items() if alias != 'pass' %}
31
+ {% if alias_result.evidence_type is none and alias_result.errors %}
32
+ {{ alias }}: {{ alias_result.errors }}
33
+ {% endif %}
34
+ {% endfor %}
35
+ {% endif %}
36
+
37
+ {%- endmacro %}
@@ -15,7 +15,7 @@ eva-sub-cli v{{cli_version}}
15
15
  PROJECT SUMMARY
16
16
  General details about the project
17
17
 
18
- {{ project_details_report(project_title, validation_date, submission_dir, vcf_fasta_analysis_mapping) }}
18
+ {{ project_details_report(validation_results, project_title, validation_date, submission_dir, consent_statement_required, vcf_fasta_analysis_mapping) }}
19
19
 
20
20
  -
21
21
 
@@ -49,6 +49,10 @@ class EvaMetadataJson(AppLogger):
49
49
  if self.resolved_files:
50
50
  del self.resolved_files
51
51
 
52
+ def set_analyses(self, analysis_list):
53
+ assert isinstance(analysis_list, list)
54
+ self.content['analysis'] = analysis_list
55
+
52
56
  @cached_property
53
57
  def samples_per_analysis(self):
54
58
  """Returns mapping of analysis alias to sample names, based on metadata."""
@@ -28,6 +28,7 @@ params.executable = [
28
28
  // python scripts - installed as part of eva-sub-cli
29
29
  params.python_scripts = [
30
30
  "samples_checker": "samples_checker.py",
31
+ "evidence_type_checker": "evidence_type_checker.py",
31
32
  "fasta_checker": "check_fasta_insdc.py",
32
33
  "xlsx2json": "xlsx2json.py",
33
34
  "semantic_checker": "check_metadata_semantics.py",
@@ -99,6 +100,7 @@ workflow {
99
100
  metadata_json_validation(metadata_json)
100
101
  metadata_semantic_check(metadata_json)
101
102
  sample_name_concordance(metadata_json, vcf_files.collect())
103
+ evidence_type_check(metadata_json)
102
104
  insdc_checker(metadata_json, fasta_to_vcfs)
103
105
  }
104
106
  }
@@ -265,6 +267,25 @@ process sample_name_concordance {
265
267
  }
266
268
 
267
269
 
270
+ process evidence_type_check {
271
+ publishDir output_dir,
272
+ overwrite: true,
273
+ mode: "copy"
274
+
275
+ input:
276
+ path(metadata_json)
277
+
278
+ output:
279
+ path "evidence_type_checker.yml", emit: evidence_type_checker_yml
280
+ path "evidence_type_checker.log", emit: evidence_type_checker_log
281
+
282
+ script:
283
+ """
284
+ $params.python_scripts.evidence_type_checker --metadata_json $metadata_json --output_yaml evidence_type_checker.yml > evidence_type_checker.log 2>&1
285
+ """
286
+ }
287
+
288
+
268
289
  process insdc_checker {
269
290
  publishDir output_dir,
270
291
  overwrite: true,
@@ -16,12 +16,13 @@ def get_logo_data():
16
16
 
17
17
 
18
18
  def generate_report(validation_results, validation_date, submission_dir, vcf_fasta_analysis_mapping, project_title,
19
- subdir, template_file):
19
+ consent_statement_required, subdir, template_file):
20
+ results_for_report = {k: v for k, v in validation_results.items() if k != 'ready_for_submission_to_eva'}
20
21
  vcf_files = sorted(set([file_name
21
- for check in validation_results if check in ["vcf_check", "assembly_check"]
22
- for file_name in validation_results[check]
22
+ for check in results_for_report if check in ["vcf_check", "assembly_check"]
23
+ for file_name in results_for_report[check]
23
24
  ]))
24
- fasta_files = sorted([file_name for file_name in validation_results['fasta_check']])
25
+ fasta_files = sorted([file_name for file_name in results_for_report['fasta_check']])
25
26
  template = Environment(
26
27
  loader=FileSystemLoader(os.path.join(current_dir, 'jinja_templates', subdir))
27
28
  ).get_template(template_file)
@@ -33,17 +34,20 @@ def generate_report(validation_results, validation_date, submission_dir, vcf_fas
33
34
  vcf_files=vcf_files,
34
35
  fasta_files=fasta_files,
35
36
  submission_dir=submission_dir,
37
+ consent_statement_required=consent_statement_required,
36
38
  vcf_fasta_analysis_mapping=vcf_fasta_analysis_mapping,
37
- validation_results=validation_results
39
+ validation_results=results_for_report
38
40
  )
39
41
  return re.sub('\s+\n', '\n', rendered_template)
40
42
 
41
43
 
42
- def generate_html_report(validation_results, validation_date, submission_dir, vcf_fasta_analysis_mapping, project_title):
43
- return generate_report(validation_results, validation_date, submission_dir, vcf_fasta_analysis_mapping, project_title,
44
- subdir='html', template_file='report.html')
44
+ def generate_html_report(validation_results, validation_date, submission_dir, vcf_fasta_analysis_mapping, project_title,
45
+ consent_statement_required):
46
+ return generate_report(validation_results, validation_date, submission_dir, vcf_fasta_analysis_mapping,
47
+ project_title, consent_statement_required, subdir='html', template_file='report.html')
45
48
 
46
49
 
47
- def generate_text_report(validation_results, validation_date, submission_dir, vcf_fasta_analysis_mapping, project_title):
48
- return generate_report(validation_results, validation_date, submission_dir, vcf_fasta_analysis_mapping, project_title,
49
- subdir='text', template_file='report.txt')
50
+ def generate_text_report(validation_results, validation_date, submission_dir, vcf_fasta_analysis_mapping, project_title,
51
+ consent_statement_required):
52
+ return generate_report(validation_results, validation_date, submission_dir, vcf_fasta_analysis_mapping,
53
+ project_title, consent_statement_required, subdir='text', template_file='report.txt')
@@ -57,6 +57,7 @@ class SemanticMetadataChecker(AppLogger):
57
57
  self.check_existing_biosamples()
58
58
  self.check_all_analysis_run_accessions()
59
59
  self.check_analysis_alias_coherence()
60
+ self.check_all_analysis_contain_samples()
60
61
 
61
62
  def check_all_project_accessions(self):
62
63
  """Check that ENA project accessions exist and are public."""
@@ -288,3 +289,15 @@ class SemanticMetadataChecker(AppLogger):
288
289
  property=json_path,
289
290
  description=f'{",".join(list2_list1)} present in {list2_desc} not in {list1_desc}'
290
291
  )
292
+
293
+ def check_all_analysis_contain_samples(self):
294
+ sample_analysis_aliases_set = set()
295
+ for sample in self.metadata.get(SAMPLE_KEY, []):
296
+ for analysis_alias in sample.get(ANALYSIS_ALIAS_KEY, []):
297
+ sample_analysis_aliases_set.add(analysis_alias)
298
+
299
+ for idx, analysis in enumerate(self.metadata.get(ANALYSIS_KEY, [])):
300
+ analysis_alias = analysis.get(ANALYSIS_ALIAS_KEY)
301
+ if analysis_alias and analysis_alias not in sample_analysis_aliases_set:
302
+ json_path = f'/{ANALYSIS_KEY}/{idx}'
303
+ self.add_error(property=json_path, description=f'No sample found for the analysis. Should have at the least one sample.')
@@ -12,7 +12,7 @@ from eva_sub_cli.validators.validator import Validator
12
12
  logger = logging_config.get_logger(__name__)
13
13
 
14
14
  default_container_image = 'ebivariation/eva-sub-cli'
15
- default_container_tag = 'v0.0.4.dev0'
15
+ default_container_tag = 'v0.0.5'
16
16
  container_validation_dir = '/opt/vcf_validation'
17
17
  container_validation_output_dir = 'vcf_validation_output'
18
18