eva-sub-cli 0.4.7__tar.gz → 0.4.8__tar.gz

eva_sub_cli-0.4.8/.github/workflows/trigger-eva-sub-cli-integration-tests.yml

@@ -0,0 +1,16 @@
+name: Trigger Eva Sub Cli Integration Tests
+
+on:
+  push:
+    branches: [main, test-trigger-integration-tests]
+
+jobs:
+  trigger-integration-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Trigger eva sub cli integration tests
+        uses: peter-evans/repository-dispatch@v2
+        with:
+          token: ${{ secrets.TRIGGER_INTEGRATION_TESTS }}
+          repository: EBIvariation/eva-integration-tests
+          event-type: trigger-eva-sub-cli-tests

{eva_sub_cli-0.4.7/eva_sub_cli.egg-info → eva_sub_cli-0.4.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eva_sub_cli
-Version: 0.4.7
+Version: 0.4.8
 Summary: EBI EVA - validation and submission command line tool
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console

{eva_sub_cli-0.4.7 → eva_sub_cli-0.4.8}/eva_sub_cli/__init__.py

@@ -6,7 +6,7 @@ PACKAGE_DIR = os.path.dirname(os.path.abspath(__file__))
 NEXTFLOW_DIR = os.path.join(PACKAGE_DIR, 'nextflow')
 ETC_DIR = os.path.join(PACKAGE_DIR, 'etc')
 LSRI_CLIENT_ID = "aa0fcc42-096a-4f9d-b871-aceb1a97d174"
-DEFAULT_METADATA_XLSX_TEMPLATE_VERSION = '1.1.6'
+MINIMUM_METADATA_XLSX_TEMPLATE_VERSION = '2.0.0'
 
 SUB_CLI_CONFIG_FILE = ".eva_sub_cli_config.yml"
 

{eva_sub_cli-0.4.7 → eva_sub_cli-0.4.8}/eva_sub_cli/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.4.7'
-__version_tuple__ = version_tuple = (0, 4, 7)
+__version__ = version = '0.4.8'
+__version_tuple__ = version_tuple = (0, 4, 8)

{eva_sub_cli-0.4.7 → eva_sub_cli-0.4.8}/eva_sub_cli/etc/eva_schema.json

@@ -310,9 +310,10 @@
             "type": "string",
             "description": "Sample Name used in the VCF file"
           },
-          "sampleAccession": {
+          "bioSampleAccession": {
             "type": "string",
-            "description": "Accession of the sample (BioSamples, ENA or EGA)"
+            "pattern": "^SAM(E|D|N)[A-Z]?[0-9]+$",
+            "description": "BioSample accession of pre-existing sample. Only use for preregistered samples"
           },
           "bioSampleObject": {
             "type": "object",

{eva_sub_cli-0.4.7 → eva_sub_cli-0.4.8}/eva_sub_cli/executables/check_fasta_insdc.py

@@ -1,8 +1,5 @@
 import argparse
-import gzip
 import hashlib
-import json
-from itertools import groupby
 
 import requests
 import yaml
@@ -11,8 +8,7 @@ from requests import HTTPError
 from retry import retry
 
 from eva_sub_cli.file_utils import fasta_iter
-from eva_sub_cli.metadata_utils import get_files_per_analysis, get_analysis_for_vcf_file, \
-    get_reference_assembly_for_analysis
+from eva_sub_cli.metadata import EvaMetadataJson
 
 REFGET_SERVER = 'https://www.ebi.ac.uk/ena/cram'
 CONTIG_ALIAS_SERVER = 'https://www.ebi.ac.uk/eva/webservices/contig-alias/v1/chromosomes/md5checksum'
@@ -52,7 +48,7 @@ def _get_containing_assemblies_paged(url):
                 results.add(contigEntity['assembly']['insdcAccession'])
         if '_links' in response_data and 'next' in response_data['_links']:
             # Add results from next page if needed
-            results |= _get_containing_assemblies_paged(response_data['_links']['next'])
+            results = results.union(_get_containing_assemblies_paged(response_data['_links']['next']['href']))
         return results
     return set()
 
@@ -94,7 +90,7 @@ def assess_fasta(input_fasta, analyses, assembly_in_metadata):
                 possible_assemblies = containing_assemblies
             else:
                 possible_assemblies &= containing_assemblies
-    except (ConnectionError, HTTPError) as e:
+    except Exception as e:
         # Server errors from either ENA refget or EVA contig alias will halt the check prematurely.
         # Report the error but do not return from the method, so that incomplete results can be reported
         # (i.e. any sequences found to be INSDC and any compatible assemblies so far)
@@ -121,23 +117,21 @@ def get_analyses_and_reference_genome_from_metadata(vcf_files_for_fasta, json_fi
       single assembly accession
     :param json_file: JSON file of the metadata
     """
-    with open(json_file) as open_json:
-        metadata = json.load(open_json)
-        files_per_analysis = get_files_per_analysis(metadata)
-        # Get all analyses associated with all vcf files that are linked with a single fasta file
-        all_analyses = set()
-        for vcf_file in vcf_files_for_fasta:
-            analysis_aliases = get_analysis_for_vcf_file(vcf_file, files_per_analysis)
-            if len(analysis_aliases) != 1:
-                logger.error(f'Could not determine analysis associated with VCF file: {vcf_file}')
-            else:
-                all_analyses.add(analysis_aliases[0])
-        # Get (single) assembly associated with these analyses
-        assemblies = [get_reference_assembly_for_analysis(metadata, analysis) for analysis in all_analyses]
-        if len(assemblies) != 1:
-            logger.error(f'Could not determine assembly accession to check against fasta file, out of: {assemblies}')
-            return all_analyses, None
-        return all_analyses, assemblies[0]
+    metadata = EvaMetadataJson(json_file)
+    # Get all analyses associated with all vcf files that are linked with a single fasta file
+    all_analyses = set()
+    for vcf_file in vcf_files_for_fasta:
+        analysis_aliases = metadata.get_analysis_for_vcf_file(vcf_file)
+        if len(analysis_aliases) != 1:
+            logger.error(f'Could not determine analysis associated with VCF file: {vcf_file}')
+        else:
+            all_analyses.add(analysis_aliases[0])
+    # Get (single) assembly associated with these analyses
+    assemblies = [metadata.get_reference_assembly_for_analysis(analysis) for analysis in all_analyses]
+    if len(assemblies) != 1:
+        logger.error(f'Could not determine assembly accession to check against fasta file, out of: {assemblies}')
+        return all_analyses, None
+    return all_analyses, assemblies[0]
 
 
 def main():
@@ -156,4 +150,3 @@ def main():
     analyses, metadata_insdc = get_analyses_and_reference_genome_from_metadata(args.vcf_files, args.metadata_json)
     results = assess_fasta(args.input_fasta, analyses, metadata_insdc)
     write_result_yaml(args.output_yaml, results)
-

{eva_sub_cli-0.4.7 → eva_sub_cli-0.4.8}/eva_sub_cli/executables/samples_checker.py

@@ -1,6 +1,4 @@
 import argparse
-import gzip
-import json
 import os
 
 from ebi_eva_common_pyutils.logger import logging_config
@@ -8,7 +6,7 @@ from ebi_eva_common_pyutils.logger import logging_config
 import yaml
 
 from eva_sub_cli.file_utils import open_gzip_if_required
-from eva_sub_cli.metadata_utils import get_samples_per_analysis, get_files_per_analysis, get_analysis_for_vcf_file
+from eva_sub_cli.metadata import EvaMetadataJson
 
 logger = logging_config.get_logger(__name__)
 
@@ -24,8 +22,10 @@ def get_samples_from_vcf(vcf_file):
                 if len(sp_line) > 9:
                     return sp_line[9:]
                 else:
-                    logger.warning(f"No Sample names found in file {vcf_file}")
+                    logger.warning(f"No sample names found in file {vcf_file}")
                     return []
+        logger.warning(f"No sample names found in file {vcf_file}")
+        return []
 
 
 def compare_names_in_files_and_samples(sample_name_in_analysis, sample_name_per_file):
@@ -54,12 +54,12 @@ def compare_names_in_files_and_samples(sample_name_in_analysis, sample_name_per_
             more_metadata_submitted_files)
 
 
-def compare_all_analysis(samples_per_analysis, files_per_analysis):
+def compare_all_analysis(metadata, files_per_analysis):
     overall_differences = False
     results_per_analysis_alias = {}
-    all_analysis_alias = set(samples_per_analysis) | set(files_per_analysis)
+    all_analysis_alias = set(metadata.samples_per_analysis) | set(files_per_analysis)
     for analysis_alias in all_analysis_alias:
-        sample_name_in_analysis = samples_per_analysis.get(analysis_alias, [])
+        sample_name_in_analysis = metadata.samples_per_analysis.get(analysis_alias, [])
         sample_name_per_file = {
             file_path: get_samples_from_vcf(file_path)
             for file_path in files_per_analysis.get(analysis_alias, [])
@@ -78,13 +78,7 @@ def compare_all_analysis(samples_per_analysis, files_per_analysis):
     return overall_differences, results_per_analysis_alias
 
 
-def read_metadata_json(json_file):
-    with open(json_file) as open_json:
-        metadata = json.load(open_json)
-        return get_samples_per_analysis(metadata), get_files_per_analysis(metadata)
-
-
-def associate_vcf_path_with_analysis(vcf_files, files_per_analysis):
+def associate_vcf_path_with_analysis(metadata, vcf_files):
     """
     Match the files names associated with analysis provided in the metadata with the file path given on the command
     line.
@@ -93,10 +87,10 @@ def associate_vcf_path_with_analysis(vcf_files, files_per_analysis):
     :returns dictionary of analysis and their associated vcf file path
     """
     result_files_per_analysis = dict()
-    for analysis in files_per_analysis:
+    for analysis in metadata.files_per_analysis:
         result_files_per_analysis[analysis] = []
     for vcf_file in vcf_files:
-        analysis_aliases = get_analysis_for_vcf_file(vcf_file, files_per_analysis)
+        analysis_aliases = metadata.get_analysis_for_vcf_file(vcf_file)
         if len(analysis_aliases) == 1:
             result_files_per_analysis[analysis_aliases[0]].append(vcf_file)
         elif len(analysis_aliases) == 0:
@@ -123,9 +117,9 @@ def check_sample_name_concordance(metadata_json, vcf_files, output_yaml):
     Take the metadata following EVA standard and formatted in JSON then compare the sample names in it to the ones
     found in the VCF files
     """
-    samples_per_analysis, files_per_analysis = read_metadata_json(metadata_json)
-    file_path_per_analysis = associate_vcf_path_with_analysis(vcf_files, files_per_analysis)
-    overall_differences, results_per_analysis_alias = compare_all_analysis(samples_per_analysis, file_path_per_analysis)
+    metadata = EvaMetadataJson(metadata_json)
+    file_path_per_analysis = associate_vcf_path_with_analysis(metadata, vcf_files)
+    overall_differences, results_per_analysis_alias = compare_all_analysis(metadata, file_path_per_analysis)
     write_result_yaml(output_yaml, overall_differences, results_per_analysis_alias)
 
 

{eva_sub_cli-0.4.7 → eva_sub_cli-0.4.8}/eva_sub_cli/executables/xlsx2json.py

@@ -105,11 +105,11 @@ class XlsxParser:
             if type_name == 'string':
                 return str(value)
             if type_name == 'boolean':
-                return str(value).lower() in ['true', '1', 't', 'y', 'yes']
+                return str(value).lower() in ['true', '1', "'1'", 't', 'y', 'yes']
             if type_name == 'list':
                 # split and remove empty values
-                return [XlsxParser.trim_value(v)
-                        for v in value.split(',') if v]
+                split_values = [XlsxParser.trim_value(v) for v in value.split(',') if v]
+                return [v for v in split_values if v]
             if type_name == 'date':
                 return XlsxParser.serialize(value)
         return value

{eva_sub_cli-0.4.7 → eva_sub_cli-0.4.8}/eva_sub_cli/file_utils.py

@@ -23,6 +23,11 @@ def is_submission_dir_writable(submission_dir):
         return False
     return True
 
+def is_vcf_file(file_path):
+    if file_path:
+        file_path = file_path.strip().lower()
+        return file_path.endswith('.vcf') or file_path.endswith('.vcf.gz')
+    return False
 
 def backup_file_or_directory(file_name, max_backups=None):
     """

eva_sub_cli-0.4.8/eva_sub_cli/metadata.py

@@ -0,0 +1,95 @@
+import json
+import os
+from collections import defaultdict
+from copy import deepcopy
+from functools import cached_property
+
+from ebi_eva_common_pyutils.logger import AppLogger
+
+
+class EvaMetadataJson(AppLogger):
+
+    def __init__(self, path_to_json):
+        with open(path_to_json) as open_json:
+            self.content = json.load(open_json)
+
+    @property
+    def project(self):
+        return self.content.get('project', {})
+
+    @property
+    def submitter_details(self):
+        return self.content.get('submitterDetails', {})
+
+    @property
+    def analyses(self):
+        return self.content.get('analysis', [])
+
+    @property
+    def samples(self):
+        return self.content.get('sample', [])
+
+    @property
+    def files(self):
+        return self.content.get('files', [])
+
+    @cached_property
+    def resolved_files(self):
+        """Returns list of files with fileName resolved to an absolute path."""
+        mod_files = deepcopy(self.content.get('files', []))
+        for file_info in mod_files:
+            if 'fileName' in file_info:
+                file_info['fileName'] = os.path.abspath(file_info['fileName'])
+        return mod_files
+
+    def set_files(self, file_list):
+        assert isinstance(file_list, list)
+        self.content['files'] = file_list
+        # invalidate the cached property
+        if self.resolved_files:
+            del self.resolved_files
+
+    @cached_property
+    def samples_per_analysis(self):
+        """Returns mapping of analysis alias to sample names, based on metadata."""
+        samples_per_analysis = defaultdict(list)
+        for sample_info in self.samples:
+            for analysis_alias in sample_info.get('analysisAlias', []):
+                samples_per_analysis[analysis_alias].append(sample_info.get('sampleInVCF'))
+        return {
+            analysis_alias: set(samples)
+            for analysis_alias, samples in samples_per_analysis.items()
+        }
+
+    @cached_property
+    def files_per_analysis(self):
+        """Returns mapping of analysis alias to filenames, based on metadata."""
+        files_per_analysis = defaultdict(list)
+        for file_info in self.resolved_files:
+            files_per_analysis[file_info.get('analysisAlias')].append(file_info.get('fileName'))
+        return {
+            analysis_alias: set(filepaths)
+            for analysis_alias, filepaths in files_per_analysis.items()
+        }
+
+    def get_reference_assembly_for_analysis(self, analysis_alias):
+        """Returns the reference assembly for this analysis (does not validate format)."""
+        for analysis in self.analyses:
+            if analysis.get('analysisAlias') == analysis_alias:
+                return analysis.get('referenceGenome')
+        return None
+
+    def get_analysis_for_vcf_file(self, vcf_file):
+        """Returns list of analysis aliases associated with the vcf file path."""
+        if not os.path.exists(vcf_file):
+            raise FileNotFoundError(f'{vcf_file} cannot be resolved')
+        analysis_aliases = [analysis_alias for analysis_alias in self.files_per_analysis
+                            if vcf_file in self.files_per_analysis[analysis_alias]
+                            or os.path.basename(vcf_file) in [
+                                os.path.basename(p) for p in self.files_per_analysis[analysis_alias]
+                            ]]
+        return analysis_aliases
+
+    def write(self, output_path):
+        with open(output_path, 'w') as open_file:
+            json.dump(self.content, open_file)

{eva_sub_cli-0.4.7 → eva_sub_cli-0.4.8}/eva_sub_cli/orchestrator.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 import csv
-import json
 import os
 import re
 from collections import defaultdict
@@ -14,13 +13,15 @@ from requests import HTTPError
 from retry import retry
 
 import eva_sub_cli
-from eva_sub_cli import DEFAULT_METADATA_XLSX_TEMPLATE_VERSION
+from eva_sub_cli import MINIMUM_METADATA_XLSX_TEMPLATE_VERSION
 from eva_sub_cli import SUB_CLI_CONFIG_FILE, __version__
 from eva_sub_cli.exceptions.invalid_file_type_exception import InvalidFileTypeError
 from eva_sub_cli.exceptions.metadata_template_version_exception import MetadataTemplateVersionException, \
     MetadataTemplateVersionNotFoundException
 from eva_sub_cli.exceptions.submission_not_found_exception import SubmissionNotFoundException
 from eva_sub_cli.exceptions.submission_status_exception import SubmissionStatusException
+from eva_sub_cli.file_utils import is_vcf_file
+from eva_sub_cli.metadata import EvaMetadataJson
 from eva_sub_cli.submission_ws import SubmissionWSClient
 from eva_sub_cli.submit import StudySubmitter, SUB_CLI_CONFIG_KEY_SUBMISSION_ID, \
     SUB_CLI_CONFIG_KEY_SUBMISSION_UPLOAD_URL
@@ -76,6 +77,8 @@ def get_project_title_and_create_vcf_files_mapping(submission_dir, vcf_files, re
         elif metadata_xlsx:
             project_title, vcf_files_mapping = get_project_and_vcf_fasta_mapping_from_metadata_xlsx(metadata_xlsx, True)
 
+        # Filter out non-vcf files
+        vcf_files_mapping = [(vcf, fasta, report) for vcf, fasta, report in vcf_files_mapping if is_vcf_file(vcf)]
         validate_vcf_mapping(vcf_files_mapping)
         for mapping in vcf_files_mapping:
             writer.writerow(mapping)
@@ -103,27 +106,24 @@ def validate_vcf_mapping(vcf_mapping):
 
 
 def get_project_and_vcf_fasta_mapping_from_metadata_json(metadata_json, mapping_req=False):
-    with open(metadata_json) as file:
-        json_metadata = json.load(file)
-
-        if 'project' in json_metadata:
-            if 'title' in json_metadata['project']:
-                project_title = json_metadata['project']['title']
-
-        vcf_fasta_report_mapping = []
-        if mapping_req:
-            analysis_alias_dict = defaultdict(dict)
-            for analysis in json_metadata['analysis']:
-                analysis_alias_dict[analysis['analysisAlias']]['referenceFasta'] = analysis['referenceFasta']
-                analysis_alias_dict[analysis['analysisAlias']]['assemblyReport'] = analysis['assemblyReport'] \
-                    if 'assemblyReport' in analysis else ''
-
-            for file_dict in json_metadata['files']:
-                reference_fasta = analysis_alias_dict[file_dict['analysisAlias']]['referenceFasta']
-                assembly_report = analysis_alias_dict[file_dict['analysisAlias']]['assemblyReport']
-                vcf_fasta_report_mapping.append([os.path.abspath(file_dict['fileName']),
-                                                 os.path.abspath(reference_fasta),
-                                                 os.path.abspath(assembly_report) if assembly_report else ''])
+    metadata = EvaMetadataJson(metadata_json)
+
+    project_title = metadata.project.get('title')
+
+    vcf_fasta_report_mapping = []
+    if mapping_req:
+        analysis_alias_dict = defaultdict(dict)
+        for analysis in metadata.analyses:
+            analysis_alias_dict[analysis['analysisAlias']]['referenceFasta'] = analysis['referenceFasta']
+            analysis_alias_dict[analysis['analysisAlias']]['assemblyReport'] = analysis['assemblyReport'] \
+                if 'assemblyReport' in analysis else ''
+
+        for file_dict in metadata.resolved_files:
+            reference_fasta = analysis_alias_dict[file_dict['analysisAlias']]['referenceFasta']
+            assembly_report = analysis_alias_dict[file_dict['analysisAlias']]['assemblyReport']
+            vcf_fasta_report_mapping.append([os.path.abspath(file_dict['fileName']),
+                                             os.path.abspath(reference_fasta),
+                                             os.path.abspath(assembly_report) if assembly_report else ''])
 
     return project_title, vcf_fasta_report_mapping
 
@@ -225,35 +225,36 @@ def get_project_and_vcf_fasta_mapping_from_metadata_xlsx(metadata_xlsx, mapping_
 
 def check_validation_required(tasks, sub_config, username=None, password=None):
     # Validation is mandatory so if submit is requested then VALIDATE must have run before or be requested as well
-    if SUBMIT in tasks:
-        if not sub_config.get(READY_FOR_SUBMISSION_TO_EVA, False):
-            return True
-        # If we are working with an existing submission check its status to see if it was submitted and failed before.
-        submission_id = sub_config.get(SUB_CLI_CONFIG_KEY_SUBMISSION_ID, None)
-        if submission_id:
-            try:
-                submission_status = SubmissionWSClient(username, password).get_submission_status(submission_id)
-                if submission_status == 'FAILED':
-                    # Reset the submission_id which will force the creation of a new one
-                    sub_config.set(SUB_CLI_CONFIG_KEY_SUBMISSION_ID, value=None)
-                    sub_config.set(SUB_CLI_CONFIG_KEY_SUBMISSION_UPLOAD_URL, value=None)
-                    return True
-                else:
-                    return False
-            except requests.HTTPError as ex:
-                if ex.response.status_code == 404:
-                    logger.error(
-                        f'Submission with id {submission_id} could not be found: '
-                        f'status code: {ex.response.status_code} response: {ex.response.text}')
-                    raise SubmissionNotFoundException(f'Submission with id {submission_id} could not be found')
-                else:
-                    logger.error(f'Error occurred while getting status of the submission with Id {submission_id}: '
-                                 f'status code: {ex.response.status_code} response: {ex.response.text}')
-                    raise SubmissionStatusException(f'Error occurred while getting status of the submission '
-                                                    f'with Id {submission_id}')
-
-        logger.debug(f'submission id not found in config. This might be the first time user is submitting')
+    if SUBMIT not in tasks:
         return False
+    if not sub_config.get(READY_FOR_SUBMISSION_TO_EVA, False):
+        return True
+    # If we are working with an existing submission check its status to see if it was submitted and failed before.
+    submission_id = sub_config.get(SUB_CLI_CONFIG_KEY_SUBMISSION_ID, None)
+    if submission_id:
+        try:
+            submission_status = SubmissionWSClient(username, password).get_submission_status(submission_id)
+            if submission_status == 'FAILED':
+                # Reset the submission_id which will force the creation of a new one
+                sub_config.set(SUB_CLI_CONFIG_KEY_SUBMISSION_ID, value=None)
+                sub_config.set(SUB_CLI_CONFIG_KEY_SUBMISSION_UPLOAD_URL, value=None)
+                return True
+            else:
+                return False
+        except requests.HTTPError as ex:
+            if ex.response.status_code == 404:
+                logger.error(
+                    f'Submission with id {submission_id} could not be found: '
+                    f'status code: {ex.response.status_code} response: {ex.response.text}')
+                raise SubmissionNotFoundException(f'Submission with id {submission_id} could not be found')
+            else:
+                logger.error(f'Error occurred while getting status of the submission with Id {submission_id}: '
+                             f'status code: {ex.response.status_code} response: {ex.response.text}')
+                raise SubmissionStatusException(f'Error occurred while getting status of the submission '
+                                                f'with Id {submission_id}')
+
+    logger.debug(f'submission id not found in config. This might be the first time user is submitting')
+    return False
 
 
 def orchestrate_process(submission_dir, vcf_files, reference_fasta, metadata_json, metadata_xlsx,
@@ -272,7 +273,7 @@ def orchestrate_process(submission_dir, vcf_files, reference_fasta, metadata_jso
     if metadata_xlsx:
         metadata_xlsx = os.path.abspath(metadata_xlsx)
         # check metadata xlsx version is not lower than the required min metadata template version
-        verify_metadata_xlsx_version(metadata_xlsx, DEFAULT_METADATA_XLSX_TEMPLATE_VERSION)
+        verify_metadata_xlsx_version(metadata_xlsx, MINIMUM_METADATA_XLSX_TEMPLATE_VERSION)
 
     # Get the provided Project Title and VCF files mapping (VCF, Fasta and Report)
     project_title, vcf_files_mapping = get_project_title_and_create_vcf_files_mapping(

{eva_sub_cli-0.4.7 → eva_sub_cli-0.4.8}/eva_sub_cli/validators/docker_validator.py

@@ -12,7 +12,7 @@ from eva_sub_cli.validators.validator import Validator
 logger = logging_config.get_logger(__name__)
 
 default_container_image = 'ebivariation/eva-sub-cli'
-default_container_tag = 'v0.0.3'
+default_container_tag = 'v0.0.4.dev0'
 container_validation_dir = '/opt/vcf_validation'
 container_validation_output_dir = 'vcf_validation_output'
 

{eva_sub_cli-0.4.7 → eva_sub_cli-0.4.8}/eva_sub_cli/validators/native_validator.py

@@ -22,6 +22,10 @@ class NativeValidator(Validator):
         self.assembly_checker_path = assembly_checker_path
         self.biovalidator_path = biovalidator_path
 
+    @staticmethod
+    def _validation_file_path_for(file_path):
+        return file_path
+
     def _validate(self):
         self.run_validator()
 

{eva_sub_cli-0.4.7 → eva_sub_cli-0.4.8}/eva_sub_cli/validators/validator.py

@@ -12,6 +12,7 @@ from ebi_eva_common_pyutils.config import WritableConfig
 
 from eva_sub_cli import ETC_DIR, SUB_CLI_CONFIG_FILE, __version__
 from eva_sub_cli.file_utils import backup_file_or_directory, resolve_single_file_path
+from eva_sub_cli.metadata import EvaMetadataJson
 from eva_sub_cli.report import generate_html_report
 from ebi_eva_common_pyutils.logger import logging_config, AppLogger
 
@@ -119,7 +120,7 @@ class Validator(AppLogger):
 
     @staticmethod
     def _validation_file_path_for(file_path):
-        return file_path
+        raise NotImplementedError
 
     def verify_files_present(self):
         # verify mapping file exists
@@ -431,29 +432,25 @@ class Validator(AppLogger):
                 f"Cannot locate file_info.txt at {os.path.join(self.output_dir, 'other_validations', 'file_info.txt')}"
             )
         if self.metadata_json_post_validation:
-            with open(self.metadata_json_post_validation) as open_file:
-                try:
-                    json_data = json.load(open_file)
-                    file_rows = []
-                    files_from_metadata = json_data.get('files', [])
-                    if files_from_metadata:
-                        for file_dict in json_data.get('files', []):
-                            file_path = self._validation_file_path_for(file_dict.get('fileName'))
-                            file_dict['md5'] = file_path_2_md5.get(file_path) or \
-                                               file_name_2_md5.get(file_dict.get('fileName')) or ''
-                            file_dict['fileSize'] = file_path_2_file_size.get(file_path) or \
-                                               file_name_2_file_size.get(file_dict.get('fileName')) or ''
-                            file_rows.append(file_dict)
-                    else:
-                        self.error('No file found in metadata and multiple analysis alias exist: '
-                                   'cannot infer the relationship between files and analysis alias')
-                    json_data['files'] = file_rows
-                except Exception as e:
-                    # Skip adding the md5
-                    self.error('Error while loading or parsing metadata json: ' + str(e))
-            if json_data:
-                with open(self.metadata_json_post_validation, 'w') as open_file:
-                    json.dump(json_data, open_file)
+            metadata = EvaMetadataJson(self.metadata_json_post_validation)
+            try:
+                file_rows = []
+                if metadata.files:
+                    for file_dict in metadata.files:
+                        file_path = self._validation_file_path_for(file_dict.get('fileName'))
+                        file_dict['md5'] = file_path_2_md5.get(file_path) or \
+                                           file_name_2_md5.get(file_dict.get('fileName')) or ''
+                        file_dict['fileSize'] = file_path_2_file_size.get(file_path) or \
+                                           file_name_2_file_size.get(file_dict.get('fileName')) or ''
+                        file_rows.append(file_dict)
+                else:
+                    self.error('No file found in metadata and multiple analysis alias exist: '
+                               'cannot infer the relationship between files and analysis alias')
+                metadata.set_files(file_rows)
+            except Exception as e:
+                # Skip adding the md5
+                self.error('Error while loading or parsing metadata json: ' + str(e))
+            metadata.write(self.metadata_json_post_validation)
         else:
             self.error(f'Cannot locate the metadata in JSON format in {os.path.join(self.output_dir, "metadata.json")}')
 
@@ -479,27 +476,26 @@ class Validator(AppLogger):
                 vcf_fasta_analysis_mapping.append({'vcf_file': row['vcf'], 'fasta_file': row['fasta']})
 
         if self.metadata_json_post_validation:
-            with open(self.metadata_json_post_validation) as open_file:
-                try:
-                    vcf_analysis_dict = {}
-                    json_data = json.load(open_file)
-                    if json_data.get('files', []):
-                        for file in json_data.get('files', []):
-                            if file.get('fileName', []) and file.get('analysisAlias', []):
-                                vcf_analysis_dict[file.get('fileName')] = file.get('analysisAlias')
-
-                    for vcf_fasta_mapping in vcf_fasta_analysis_mapping:
-                        vcf_file = vcf_fasta_mapping.get('vcf_file')
-                        if vcf_file in vcf_analysis_dict:
-                            vcf_fasta_mapping.update({'analysis': vcf_analysis_dict.get(vcf_file)})
-                        else:
-                            vcf_fasta_mapping.update({'analysis': 'Could not be linked'})
-
-                    return vcf_fasta_analysis_mapping
-                except Exception as e:
-                    self.error('Error building Validation Report : Error getting info from metadata file' + str(e))
+            metadata = EvaMetadataJson(self.metadata_json_post_validation)
+            try:
+                vcf_analysis_dict = {}
+                for file in metadata.resolved_files:
+                    if file.get('fileName', []) and file.get('analysisAlias', []):
+                        vcf_analysis_dict[file.get('fileName')] = file.get('analysisAlias')
+
+                for vcf_fasta_mapping in vcf_fasta_analysis_mapping:
+                    vcf_file = vcf_fasta_mapping.get('vcf_file')
+                    if vcf_file in vcf_analysis_dict:
+                        vcf_fasta_mapping.update({'analysis': vcf_analysis_dict.get(vcf_file)})
+                    else:
+                        vcf_fasta_mapping.update({'analysis': 'Could not be linked'})
+
+                return vcf_fasta_analysis_mapping
+            except Exception as e:
+                self.error('Error building Validation Report : Error getting info from metadata file' + str(e))
         else:
             self.error('Error building validation report : Metadata file not present')
+        return []
 
     def create_reports(self):
         report_html = generate_html_report(self.results, self.validation_date, self.submission_dir,