eva-sub-cli 0.4.11__tar.gz → 0.4.12__tar.gz

{eva_sub_cli-0.4.11 → eva_sub_cli-0.4.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eva_sub_cli
-Version: 0.4.11
+Version: 0.4.12
 Summary: EBI EVA - validation and submission command line tool
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
@@ -19,7 +19,7 @@ Classifier: License :: OSI Approved :: Apache Software License
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: ebi_eva_common_pyutils==0.*,>=0.7.3
+Requires-Dist: ebi_eva_common_pyutils==0.*,>=0.7.4
 Requires-Dist: jinja2==3.*,>=3.0.0
 Requires-Dist: jsonschema==4.*
 Requires-Dist: openpyxl==3.*

{eva_sub_cli-0.4.11 → eva_sub_cli-0.4.12}/eva_sub_cli/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.4.11'
-__version_tuple__ = version_tuple = (0, 4, 11)
+__version__ = version = '0.4.12'
+__version_tuple__ = version_tuple = (0, 4, 12)

{eva_sub_cli-0.4.11 → eva_sub_cli-0.4.12}/eva_sub_cli/etc/eva_schema.json

@@ -330,7 +330,7 @@
                 "$ref": "eva-biosamples.json"
               },
               {
-                "$ref": "https://www.ebi.ac.uk/biosamples/schema/store/registry/schemas/ERC000011"
+                "$ref": "https://www.ebi.ac.uk/biosamples/schema-store/registry/schemas/ERC000011"
               },
               {
                 "$ref": "https://www.ebi.ac.uk/biosamples/schemas/certification/plant-miappe.json"

{eva_sub_cli-0.4.11 → eva_sub_cli-0.4.12}/eva_sub_cli/executables/evidence_type_checker.py

@@ -3,22 +3,30 @@ import argparse
 import yaml
 from ebi_eva_common_pyutils.logger import logging_config
 
-from eva_sub_cli.file_utils import detect_vcf_evidence_type
+from eva_sub_cli.file_utils import detect_vcf_evidence_type, associate_vcf_path_with_analysis
 from eva_sub_cli.metadata import EvaMetadataJson
 
 logger = logging_config.get_logger(__name__)
 
 
-def check_evidence_type(metadata_json, output_yaml):
+def check_evidence_type(metadata_json, vcf_files, output_yaml):
     metadata = EvaMetadataJson(metadata_json)
+    file_path_per_analysis = associate_vcf_path_with_analysis(metadata, vcf_files)
+
     results_per_analysis_alias = {}
 
-    for analysis_alias, vcf_file_set in metadata.files_per_analysis.items():
+    all_analysis_alias = set(metadata.files_per_analysis) | set(file_path_per_analysis)
+    for analysis_alias in all_analysis_alias:
         results_per_analysis_alias[analysis_alias] = {
             'evidence_type': None,
             'errors': None
         }
-        vcf_files = list(vcf_file_set)
+        vcf_files = list(file_path_per_analysis.get(analysis_alias, []))
+
+        if not vcf_files:
+            results_per_analysis_alias[analysis_alias]['errors'] = f'No VCF file found for the analysis'
+            continue
+
         evidence_types_for_vcf_files = [detect_vcf_evidence_type(vcf_file) for vcf_file in vcf_files]
 
         if len(set(evidence_types_for_vcf_files)) == 1 and None not in evidence_types_for_vcf_files:
@@ -43,9 +51,11 @@ def main():
     arg_parser = argparse.ArgumentParser(description='Check the evidence type in the VCF files for all analysis')
     arg_parser.add_argument('--metadata_json', required=True, dest='metadata_json',
                             help='EVA metadata json file')
+    arg_parser.add_argument('--vcf_files', dest='vcf_files', nargs='+',
+                            help='Path to the vcf files to compare to the metadata')
     arg_parser.add_argument('--output_yaml', required=True, dest='output_yaml',
                             help='Path to the location of the results')
 
     args = arg_parser.parse_args()
     logging_config.add_stdout_handler()
-    check_evidence_type(args.metadata_json, args.output_yaml)
+    check_evidence_type(args.metadata_json, args.vcf_files, args.output_yaml)

{eva_sub_cli-0.4.11 → eva_sub_cli-0.4.12}/eva_sub_cli/executables/samples_checker.py

@@ -5,7 +5,7 @@ from ebi_eva_common_pyutils.logger import logging_config
 
 import yaml
 
-from eva_sub_cli.file_utils import open_gzip_if_required, detect_vcf_evidence_type
+from eva_sub_cli.file_utils import open_gzip_if_required, detect_vcf_evidence_type, associate_vcf_path_with_analysis
 from eva_sub_cli.metadata import EvaMetadataJson
 
 logger = logging_config.get_logger(__name__)
@@ -99,32 +99,6 @@ def need_to_check_samples(sample_name_per_file):
     return True
 
 
-def associate_vcf_path_with_analysis(metadata, vcf_files):
-    """
-    Match the files names associated with analysis provided in the metadata with the file path given on the command
-    line.
-    :param vcf_files the list of full path to the vcf files
-    :param files_per_analysis: dictionary of the analysis and their associated VCF file names
-    :returns dictionary of analysis and their associated vcf file path
-    """
-    result_files_per_analysis = dict()
-    for analysis in metadata.files_per_analysis:
-        result_files_per_analysis[analysis] = []
-    for vcf_file in vcf_files:
-        analysis_aliases = metadata.get_analysis_for_vcf_file(vcf_file)
-        if len(analysis_aliases) == 1:
-            result_files_per_analysis[analysis_aliases[0]].append(vcf_file)
-        elif len(analysis_aliases) == 0:
-            logger.error(f'No analysis found for vcf {vcf_file}')
-            if 'No analysis' not in result_files_per_analysis:
-                result_files_per_analysis['No analysis'] = []
-            result_files_per_analysis['No analysis'].append(vcf_file)
-        else:
-            logger.error(f'More than one analysis were match to vcf {vcf_file}')
-
-    return result_files_per_analysis
-
-
 def write_result_yaml(output_yaml, overall_differences, results_per_analysis_alias):
     with open(output_yaml, 'w') as open_yaml:
         yaml.safe_dump(data={

{eva_sub_cli-0.4.11 → eva_sub_cli-0.4.12}/eva_sub_cli/file_utils.py

@@ -35,6 +35,30 @@ def is_vcf_file(file_path):
         return file_path.endswith('.vcf') or file_path.endswith('.vcf.gz')
     return False
 
+def associate_vcf_path_with_analysis(metadata, vcf_files):
+    """
+    Match the files names associated with analysis provided in the metadata with the given file path
+
+    :param vcf_files the list of full path to the vcf files
+    :param files_per_analysis: dictionary of the analysis and their associated VCF file names
+    :returns dictionary of analysis and their associated vcf file path
+    """
+    result_files_per_analysis = dict()
+    for analysis in metadata.files_per_analysis:
+        result_files_per_analysis[analysis] = []
+    for vcf_file in vcf_files:
+        analysis_aliases = metadata.get_analysis_for_vcf_file(vcf_file)
+        if len(analysis_aliases) == 1:
+            result_files_per_analysis[analysis_aliases[0]].append(vcf_file)
+        elif len(analysis_aliases) == 0:
+            logger.error(f'No analysis found for vcf {vcf_file}')
+            if 'No analysis' not in result_files_per_analysis:
+                result_files_per_analysis['No analysis'] = []
+            result_files_per_analysis['No analysis'].append(vcf_file)
+        else:
+            logger.error(f'More than one analysis were match to vcf {vcf_file}')
+
+    return result_files_per_analysis
 
 def detect_vcf_evidence_type(vcf_file):
     """

{eva_sub_cli-0.4.11 → eva_sub_cli-0.4.12}/eva_sub_cli/nextflow/validation.nf

@@ -100,7 +100,7 @@ workflow {
         metadata_json_validation(metadata_json)
         metadata_semantic_check(metadata_json)
         sample_name_concordance(metadata_json, vcf_files.collect())
-        evidence_type_check(metadata_json)
+        evidence_type_check(metadata_json, vcf_files.collect())
         insdc_checker(metadata_json, fasta_to_vcfs)
     }
 }
@@ -274,6 +274,7 @@ process evidence_type_check {
 
     input:
     path(metadata_json)
+    path(vcf_files)
 
     output:
     path "evidence_type_checker.yml", emit: evidence_type_checker_yml
@@ -281,7 +282,7 @@ process evidence_type_check {
 
     script:
     """
-    $params.python_scripts.evidence_type_checker --metadata_json $metadata_json --output_yaml evidence_type_checker.yml > evidence_type_checker.log 2>&1
+    $params.python_scripts.evidence_type_checker --metadata_json $metadata_json --vcf_files $vcf_files --output_yaml evidence_type_checker.yml > evidence_type_checker.log 2>&1
     """
 }
 

{eva_sub_cli-0.4.11 → eva_sub_cli-0.4.12}/eva_sub_cli/validators/docker_validator.py

@@ -12,7 +12,7 @@ from eva_sub_cli.validators.validator import Validator
 logger = logging_config.get_logger(__name__)
 
 default_container_image = 'ebivariation/eva-sub-cli'
-default_container_tag = 'v0.0.5'
+default_container_tag = 'v0.0.6'
 container_validation_dir = '/opt/vcf_validation'
 container_validation_output_dir = 'vcf_validation_output'
 

{eva_sub_cli-0.4.11 → eva_sub_cli-0.4.12}/eva_sub_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eva_sub_cli
-Version: 0.4.11
+Version: 0.4.12
 Summary: EBI EVA - validation and submission command line tool
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
@@ -19,7 +19,7 @@ Classifier: License :: OSI Approved :: Apache Software License
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: ebi_eva_common_pyutils==0.*,>=0.7.3
+Requires-Dist: ebi_eva_common_pyutils==0.*,>=0.7.4
 Requires-Dist: jinja2==3.*,>=3.0.0
 Requires-Dist: jsonschema==4.*
 Requires-Dist: openpyxl==3.*

{eva_sub_cli-0.4.11 → eva_sub_cli-0.4.12}/eva_sub_cli.egg-info/requires.txt

@@ -1,4 +1,4 @@
-ebi_eva_common_pyutils==0.*,>=0.7.3
+ebi_eva_common_pyutils==0.*,>=0.7.4
 jinja2==3.*,>=3.0.0
 jsonschema==4.*
 openpyxl==3.*

{eva_sub_cli-0.4.11 → eva_sub_cli-0.4.12}/requirements.txt

@@ -1,4 +1,4 @@
-ebi_eva_common_pyutils >= 0.7.3, == 0.*
+ebi_eva_common_pyutils >= 0.7.4, == 0.*
 jinja2 >= 3.0.0, == 3.*
 jsonschema == 4.*
 openpyxl == 3.*

{eva_sub_cli-0.4.11 → eva_sub_cli-0.4.12}/tests/build_and_test_docker_locally.py

@@ -141,13 +141,14 @@ class TestDockerValidator(TestCase):
         expected_metadata_val = 'Validation passed successfully.'
         expected_semantic_val = {'description': 'SAME123 does not exist or is private',
                                  'property': '/sample/0/bioSampleAccession'}
+        expected_evidence_type_val = {'AA': {'evidence_type': 'genotype', 'errors': None}}
         expected_metadata_files_json = [
             {'analysisAlias': 'AA', 'fileName': 'input_passed.vcf', 'fileType': 'vcf',
              'md5': '96a80c9368cc3c37095c86fbe6044fb2', 'fileSize': 45050}
         ]
 
         self.assert_validation_results(self.validator, expected_sample_checker, expected_metadata_files_json,
-                                       expected_metadata_val, expected_semantic_val)
+                                       expected_metadata_val, expected_semantic_val, expected_evidence_type_val)
 
     def test_validate_with_xlsx(self):
         self.xlsx_validator.validate()
@@ -156,13 +157,14 @@ class TestDockerValidator(TestCase):
         expected_metadata_val = 'Validation passed successfully.'
         expected_semantic_val = {'description': 'SAME123 does not exist or is private',
                                  'property': '/sample/0/bioSampleAccession'}
+        expected_evidence_type_val = {'AA': {'evidence_type': 'genotype', 'errors': None}}
         expected_metadata_files_json = [
             {'analysisAlias': 'AA', 'fileName': 'input_passed.vcf',
              'md5': '96a80c9368cc3c37095c86fbe6044fb2', 'fileSize': 45050}
         ]
 
         self.assert_validation_results(self.xlsx_validator, expected_sample_checker, expected_metadata_files_json,
-                                       expected_metadata_val, expected_semantic_val)
+                                       expected_metadata_val, expected_semantic_val, expected_evidence_type_val)
 
     def get_submission_json_metadata(self):
         return {
@@ -230,7 +232,7 @@ class TestDockerValidator(TestCase):
         return docker_cmd
 
     def assert_validation_results(self, validator, expected_sample_checker, expected_metadata_files_json,
-                                  expected_metadata_val, expected_semantic_val):
+                                  expected_metadata_val, expected_semantic_val, expected_evidence_type_val):
         vcf_format_dir = os.path.join(validator.output_dir, 'vcf_format')
         self.assertTrue(os.path.exists(vcf_format_dir))
 
@@ -260,7 +262,10 @@ class TestDockerValidator(TestCase):
             self.assertEqual('[info] Percentage of matches: 100%\n', assembly_check_logs[5])
 
         # Assert Samples concordance
-        self.assert_sample_checker(validator._sample_check_yaml, expected_sample_checker)
+        self.assert_yaml_file(validator._sample_check_yaml, expected_sample_checker)
+
+        # assert evidence type check
+        self.assert_yaml_file(validator._evidence_type_check_yaml, expected_evidence_type_val)
 
         with open(validator.metadata_json_post_validation) as open_file:
             json_data = json.load(open_file)
@@ -278,10 +283,10 @@ class TestDockerValidator(TestCase):
             semantic_output = yaml.safe_load(open_yaml)
             assert semantic_output[0] == expected_semantic_val
 
-    def assert_sample_checker(self, sample_checker_file, expected_checker):
-        self.assertTrue(os.path.isfile(sample_checker_file))
-        with open(sample_checker_file) as open_yaml:
-            self.assert_same_dict_and_unordered_list(yaml.safe_load(open_yaml), expected_checker)
+    def assert_yaml_file(self, yaml_file, expected_data):
+        self.assertTrue(os.path.isfile(yaml_file))
+        with open(yaml_file) as open_yaml:
+            self.assert_same_dict_and_unordered_list(yaml.safe_load(open_yaml), expected_data)
 
     def assert_same_dict_and_unordered_list(self, o1, o2):
         if isinstance(o1, dict) and isinstance(o2, dict):

{eva_sub_cli-0.4.11 → eva_sub_cli-0.4.12}/tests/test_docker_validator.py

@@ -158,7 +158,7 @@ class TestDockerValidator(TestCase):
             {'analysisAlias': 'AA', 'fileName': 'input_passed.vcf', 'fileType': 'vcf',
              'md5': '96a80c9368cc3c37095c86fbe6044fb2', 'fileSize': 45050}
         ]
-        expected_metadata_val = 'must match pattern "^PRJ(EB|NA)\\d+$"'
+        expected_metadata_val = 'must match pattern "^PRJ(E|D|N)[A-Z][0-9]+$"'
         expected_semantic_val = {'description': 'SAME123 does not exist or is private',
                                           'property': '/sample/0/bioSampleAccession'}
 

{eva_sub_cli-0.4.11 → eva_sub_cli-0.4.12}/tests/test_evidence_type_checker.py

@@ -21,7 +21,12 @@ class TestEvidenceTypeChecker(TestCase):
         metadata_json = os.path.join(working_dir, 'metadata.json')
         # Change to working_dir so filenames in metadata.json are resolvable
         os.chdir(working_dir)
-        self.run_and_assert_evidence_type_check(metadata_json)
+        vcf_files = [
+            os.path.join(working_dir, file_name)
+            for file_name in ['example1.vcf.gz', 'example2.vcf', 'example3.vcf']
+        ]
+
+        self.run_and_assert_evidence_type_check(metadata_json, vcf_files)
 
     def test_evidence_check_absolute_paths(self):
         working_dir = os.path.join(self.resource_dir, 'sample_checker')
@@ -39,18 +44,17 @@ class TestEvidenceTypeChecker(TestCase):
         updated_metadata = os.path.join(working_dir, 'updated_metadata.json')
         metadata.write(updated_metadata)
 
-        self.run_and_assert_evidence_type_check(updated_metadata)
+        self.run_and_assert_evidence_type_check(updated_metadata, vcf_files)
 
         if os.path.exists(updated_metadata):
             os.remove(updated_metadata)
 
-    def run_and_assert_evidence_type_check(self, metadata_json):
-        check_evidence_type(metadata_json, self.output_yaml)
-        failing_vcf = os.path.join(self.resource_dir,'sample_checker','example1.vcf.gz.tbi')
+    def run_and_assert_evidence_type_check(self, metadata_json, vcf_files):
+        check_evidence_type(metadata_json, vcf_files, self.output_yaml)
         expected_results = {
             'VD1': {
-                'errors': f'VCF file evidence type could not be determined: {failing_vcf}',
-                'evidence_type': None
+                'evidence_type': 'genotype',
+                'errors': None
             },
             'VD2': {
                 'evidence_type': 'genotype',

{eva_sub_cli-0.4.11 → eva_sub_cli-0.4.12}/tests/test_submit.py

@@ -79,8 +79,10 @@ class TestSubmit(unittest.TestCase):
         assert is_submission_dir_writable(self.test_sub_dir)
         assert not os.path.exists(self.config_file)
         sub_config = WritableConfig(self.config_file, version='version1.0')
+        sub_config.set('metadata_json', value=os.path.join(self.resource_dir, 'EVA_Submission_test.json'))
         sub_config.set(READY_FOR_SUBMISSION_TO_EVA, value=True)
         sub_config.write()
+        self.submitter = StudySubmitter(submission_dir=self.test_sub_dir, submission_config=sub_config)
 
         test_submission_ws_client = SubmissionWSClient()