eva-sub-cli 0.4.4__py3-none-any.whl

eva_sub_cli/submit.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+import json
+import os
+
+import requests
+from ebi_eva_common_pyutils.config import WritableConfig
+from ebi_eva_common_pyutils.logger import AppLogger
+from retry import retry
+
+from eva_sub_cli import SUB_CLI_CONFIG_FILE, __version__
+from eva_sub_cli.submission_ws import SubmissionWSClient
+from eva_sub_cli.validators.validator import READY_FOR_SUBMISSION_TO_EVA
+
+SUB_CLI_CONFIG_KEY_SUBMISSION_ID = "submission_id"
+SUB_CLI_CONFIG_KEY_SUBMISSION_UPLOAD_URL = "submission_upload_url"
+SUB_CLI_CONFIG_KEY_COMPLETE = "submission_complete"
+
+
+class StudySubmitter(AppLogger):
+    def __init__(self, submission_dir, submission_config: WritableConfig = None, username=None, password=None):
+        self.submission_ws_client = SubmissionWSClient(username, password)
+        self.submission_dir = submission_dir
+        if submission_config:
+            self.sub_config = submission_config
+        else:
+            config_file = os.path.join(submission_dir, SUB_CLI_CONFIG_FILE)
+            self.sub_config = WritableConfig(config_file, version=__version__)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.sub_config.backup()
+        self.sub_config.write()
+
+    @property
+    def metadata_json_file(self):
+        return self.sub_config.get('metadata_json')
+
+    @property
+    def metadata_json(self):
+        with open(self.metadata_json_file) as open_file:
+            return json.load(open_file)
+
+    @property
+    def vcf_files(self):
+        return self.sub_config.get('vcf_files')
+
+    def update_config_with_submission_id_and_upload_url(self, submission_id, upload_url):
+        self.sub_config.set(SUB_CLI_CONFIG_KEY_SUBMISSION_ID, value=submission_id)
+        self.sub_config.set(SUB_CLI_CONFIG_KEY_SUBMISSION_UPLOAD_URL, value=upload_url)
+
+    def _upload_submission(self):
+        if READY_FOR_SUBMISSION_TO_EVA not in self.sub_config or not self.sub_config[READY_FOR_SUBMISSION_TO_EVA]:
+            raise Exception(f'There are still validation errors that needs to be addressed. '
+                            f'Please review, address and re-validate before uploading.')
+
+        submission_upload_url = self.sub_config[SUB_CLI_CONFIG_KEY_SUBMISSION_UPLOAD_URL]
+
+        for f in self.vcf_files:
+            self._upload_file(submission_upload_url, f)
+
+    @retry(tries=5, delay=10, backoff=5)
+    def _upload_file(self, submission_upload_url, input_file):
+        base_name = os.path.basename(input_file)
+        self.debug(f'Transfer {base_name} to EVA FTP')
+        with open(input_file, 'rb') as f:
+            r = requests.put(os.path.join(submission_upload_url, base_name), data=f)
+        r.raise_for_status()
+        self.debug(f'Upload of {base_name} completed')
+
+    def _initiate_submission(self):
+        response_json = self.submission_ws_client.initiate_submission()
+        self.debug(f'Submission ID {response_json["submissionId"]} received!!')
+        # update config with submission id and upload url
+        self.update_config_with_submission_id_and_upload_url(response_json["submissionId"], response_json["uploadUrl"])
+
+    def _complete_submission(self):
+        self.submission_ws_client.mark_submission_uploaded(self.sub_config.get(SUB_CLI_CONFIG_KEY_SUBMISSION_ID), self.metadata_json)
+        self.debug("Submission ID {} Complete".format(self.sub_config.get(SUB_CLI_CONFIG_KEY_SUBMISSION_ID)))
+        # update config with completion of the submission
+        self.sub_config.set(SUB_CLI_CONFIG_KEY_COMPLETE, value=True)
+
+    def is_submission_status_open(self):
+        submission_id = self.sub_config.get(SUB_CLI_CONFIG_KEY_SUBMISSION_ID)
+        if submission_id:
+            status = self.submission_ws_client.get_submission_status(submission_id)
+            return status == 'OPEN'
+        return False
+
+    def submit(self):
+        if READY_FOR_SUBMISSION_TO_EVA not in self.sub_config or not self.sub_config[READY_FOR_SUBMISSION_TO_EVA]:
+            raise Exception(f'There are still validation errors that need to be addressed. '
+                            f'Please review, address and re-validate before submitting.')
+        if not self.sub_config.get(SUB_CLI_CONFIG_KEY_SUBMISSION_UPLOAD_URL):
+            self.info(f'Initiate submission')
+            self._initiate_submission()
+
+        if not self.is_submission_status_open():
+            self.warning(f'You requested the submission using {self.submission_dir}. '
+                         f'This directory contains an already completed submission. '
+                         f'Please create a new submission directory to resubmit.')
+            return
+
+        # upload submission
+        self.info(f'Upload data')
+        self._upload_submission()
+
+        # Complete the submission
+        self.info(f'Complete submission')
+        self._complete_submission()

eva_sub_cli/validators/docker_validator.py

@@ -0,0 +1,205 @@
+import csv
+import os
+import re
+import subprocess
+import time
+
+from ebi_eva_common_pyutils.logger import logging_config
+from retry import retry
+
+from eva_sub_cli.validators.validator import Validator
+
+logger = logging_config.get_logger(__name__)
+
+container_image = 'ebivariation/eva-sub-cli'
+container_tag = 'v0.0.2'
+container_validation_dir = '/opt/vcf_validation'
+container_validation_output_dir = 'vcf_validation_output'
+
+
+class DockerValidator(Validator):
+
+    def __init__(self, mapping_file, submission_dir, project_title, metadata_json=None,
+                 metadata_xlsx=None, shallow_validation=False, docker_path='docker', submission_config=None):
+        super().__init__(mapping_file, submission_dir, project_title,
+                         metadata_json=metadata_json, metadata_xlsx=metadata_xlsx,
+                         shallow_validation=shallow_validation, submission_config=submission_config)
+        self.docker_path = docker_path
+        submission_basename = re.sub('[^a-zA-Z0-9]', '', os.path.basename(submission_dir))
+        self.container_name = f'{container_image.split("/")[1]}.{container_tag}.{submission_basename}'
+
+    def _validate(self):
+        self.run_docker_validator()
+
+    @staticmethod
+    def _validation_file_path_for(file_path):
+        return f'{container_validation_dir}/{file_path}'
+
+    def get_docker_validation_cmd(self):
+        docker_cmd = ''.join([
+            f"{self.docker_path} exec {self.container_name} nextflow run eva_sub_cli/nextflow/validation.nf ",
+            f"--base_dir {container_validation_dir} ",
+            f"--vcf_files_mapping {self.mapping_file} ",
+            f"--metadata_xlsx {self.metadata_xlsx} " if self.metadata_xlsx and not self.metadata_json
+                                                     else f"--metadata_json {self.metadata_json} ",
+            f"--shallow_validation true " if self.shallow_validation else "",
+            f"--output_dir {container_validation_output_dir}"
+        ])
+        return docker_cmd
+
+    def run_docker_validator(self):
+        # check if docker container is ready for running validation
+        self.verify_docker_is_installed()
+        self.download_container_image_if_needed()
+        self.run_container_if_required()
+        try:
+            # remove all existing files from container
+            self._run_quiet_command(
+                "Remove existing files from validation directory in container",
+                f"{self.docker_path} exec {self.container_name} rm -rf work {container_validation_dir}"
+            )
+
+            # copy all required files to container (mapping file, vcf and fasta)
+            self.copy_files_to_container()
+            # create the output directory
+            self._run_quiet_command(
+                "Create output directory in container",
+                f"{self.docker_path} exec {self.container_name} mkdir -p {container_validation_dir}/{container_validation_output_dir}"
+            )
+
+            docker_cmd = self.get_docker_validation_cmd()
+            # start validation
+            self._run_quiet_command("Run Validation using Nextflow", docker_cmd)
+        except subprocess.CalledProcessError as ex:
+            logger.error(ex)
+        finally:
+            # copy validation result to user host
+            self._run_quiet_command(
+                "Copy validation output from container to host",
+                f"{self.docker_path} cp {self.container_name}:{container_validation_dir}/{container_validation_output_dir}/. {self.output_dir}"
+            )
+            self.stop_running_container()
+
+    def verify_docker_is_installed(self):
+        try:
+            self._run_quiet_command(
+                "check docker is installed and available on the path",
+                f"{self.docker_path} --version"
+            )
+        except subprocess.CalledProcessError as ex:
+            logger.error(ex)
+            raise RuntimeError(f"Please make sure docker ({self.docker_path}) is installed and available on the path")
+
+    def verify_container_is_running(self):
+        try:
+            container_run_cmd_output = self._run_quiet_command("check if container is running", f"{self.docker_path} ps", return_process_output=True)
+            if container_run_cmd_output is not None and self.container_name in container_run_cmd_output:
+                logger.debug(f"Container ({self.container_name}) is running")
+                return True
+            else:
+                logger.debug(f"Container ({self.container_name}) is not running")
+                return False
+        except subprocess.CalledProcessError as ex:
+            logger.error(ex)
+            raise RuntimeError(f"Please make sure docker ({self.docker_path}) is installed and running in the background")
+
+    def verify_container_is_stopped(self):
+        container_stop_cmd_output = self._run_quiet_command(
+            "check if container is stopped", f"{self.docker_path} ps -a",  return_process_output=True
+        )
+        if container_stop_cmd_output is not None and self.container_name in container_stop_cmd_output:
+            logger.debug(f"Container ({self.container_name}) is in stop state")
+            return True
+        else:
+            logger.debug(f"Container ({self.container_name}) is not in stop state")
+            return False
+
+    def try_restarting_container(self):
+        logger.info(f"Trying to restart container {self.container_name}")
+        try:
+            self._run_quiet_command("Try restarting container", f"{self.docker_path} start {self.container_name}")
+            if not self.verify_container_is_running():
+                raise RuntimeError(f"Container ({self.container_name}) could not be restarted")
+        except subprocess.CalledProcessError as ex:
+            logger.error(ex)
+            raise RuntimeError(f"Container ({self.container_name}) could not be restarted")
+
+    def verify_image_available_locally(self):
+        container_images_cmd_ouptut = self._run_quiet_command(
+            "Check if validator image is present",
+            f"{self.docker_path} images",
+            return_process_output=True
+        )
+        if container_images_cmd_ouptut is not None and re.search(container_image + r'\s+' + container_tag, container_images_cmd_ouptut):
+            logger.debug(f"Container ({container_image}) image is available locally")
+            return True
+        else:
+            logger.debug(f"Container ({container_image}) image is not available locally")
+            return False
+
+    def run_container_if_required(self):
+        if self.verify_container_is_running():
+            raise RuntimeError(f"Container ({self.container_name}) is already running. "
+                               f"Did you start multiple validation for the same directory ?")
+        if self.verify_container_is_stopped():
+            logger.warn(f"Container {self.container_name} was stopped but not cleaned up before.")
+            self.try_restarting_container()
+        else:
+            logger.debug(f"Trying to start container {self.container_name}")
+            try:
+                self._run_quiet_command(
+                    "Running container",
+                    f"{self.docker_path} run -it --rm -d --name {self.container_name} {container_image}:{container_tag}"
+                )
+                # Wait to give some time to container to get up and running
+                time.sleep(5)
+                if not self.verify_container_is_running():
+                    raise RuntimeError(f"Container ({self.container_name}) could not be started")
+            except subprocess.CalledProcessError as ex:
+                logger.error(ex)
+                raise RuntimeError(f"Container ({self.container_name}) could not be started")
+
+    def stop_running_container(self):
+        if self.verify_container_is_running():
+            self._run_quiet_command(
+                "Stop the running container",
+                f"{self.docker_path} stop {self.container_name}"
+            )
+
+    @retry(RuntimeError, tries=3, delay=5, backoff=1, jitter=2, logger=logger)
+    def download_container_image_if_needed(self):
+        if not self.verify_image_available_locally():
+            logger.debug(f"Pulling container ({container_image}) image")
+            try:
+                self._run_quiet_command("pull container image", f"{self.docker_path} pull {container_image}:{container_tag}")
+            except subprocess.CalledProcessError as ex:
+                logger.error(ex)
+                raise RuntimeError(f"Cannot pull container ({container_image}) image")
+            # Give the pull command some time to complete
+            time.sleep(5)
+
+    def copy_files_to_container(self):
+        def _copy(file_description, file_path):
+            self._run_quiet_command(
+                f"Create directory structure for copying {file_description} into container",
+                (f"{self.docker_path} exec {self.container_name} "
+                 f"mkdir -p {container_validation_dir}/{os.path.dirname(file_path)}")
+            )
+            self._run_quiet_command(
+                f"Copy {file_description} to container",
+                (f"{self.docker_path} cp {file_path} "
+                 f"{self.container_name}:{container_validation_dir}/{file_path}")
+            )
+        _copy('vcf metadata file', self.mapping_file)
+        if self.metadata_json:
+            _copy('json metadata file', self.metadata_json)
+        if self.metadata_xlsx:
+            _copy('excel metadata file', self.metadata_xlsx)
+        with open(self.mapping_file) as open_file:
+            reader = csv.DictReader(open_file, delimiter=',')
+            for row in reader:
+                _copy('vcf files', row['vcf'])
+                _copy('fasta files', row['fasta'])
+                # report is optional
+                if row.get('report'):
+                    _copy('assembly report files', row['report'])

eva_sub_cli/validators/native_validator.py

@@ -0,0 +1,68 @@
+import os
+import subprocess
+
+from ebi_eva_common_pyutils.logger import logging_config
+
+from eva_sub_cli.validators.validator import Validator
+
+logger = logging_config.get_logger(__name__)
+
+
+class NativeValidator(Validator):
+
+    def __init__(self, mapping_file, submission_dir, project_title, metadata_json=None, metadata_xlsx=None,
+                 shallow_validation=False, vcf_validator_path='vcf_validator',
+                 assembly_checker_path='vcf_assembly_checker', biovalidator_path='biovalidator',
+                 submission_config=None):
+        super().__init__(mapping_file, submission_dir, project_title, metadata_json=metadata_json,
+                         metadata_xlsx=metadata_xlsx, shallow_validation=shallow_validation,
+                         submission_config=submission_config)
+        self.vcf_validator_path = vcf_validator_path
+        self.assembly_checker_path = assembly_checker_path
+        self.biovalidator_path = biovalidator_path
+
+    def _validate(self):
+        self.run_validator()
+
+    def run_validator(self):
+        self.verify_executables_installed()
+        curr_wd = os.getcwd()
+        try:
+            command = self.get_validation_cmd()
+            os.chdir(self.submission_dir)
+            self._run_quiet_command("Run Validation using Nextflow", command)
+        except subprocess.CalledProcessError as ex:
+            logger.error(ex)
+        finally:
+            os.chdir(curr_wd)
+
+    def get_validation_cmd(self):
+        if self.metadata_xlsx and not self.metadata_json:
+            metadata_flag = f"--metadata_xlsx {self.metadata_xlsx}"
+        else:
+            metadata_flag = f"--metadata_json {self.metadata_json}"
+        path_to_workflow = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+                                        'nextflow/validation.nf')
+        return ''.join([
+            f"nextflow run {path_to_workflow} ",
+            f"--vcf_files_mapping {self.mapping_file} ",
+            f"{metadata_flag} ",
+            f"--output_dir {self.output_dir} ",
+            f"--shallow_validation true " if self.shallow_validation else "",
+            f"--executable.vcf_validator {self.vcf_validator_path} ",
+            f"--executable.vcf_assembly_checker {self.assembly_checker_path} ",
+            f"--executable.biovalidator {self.biovalidator_path}"
+        ])
+
+    def verify_executables_installed(self):
+        for name, path in [('vcf-validator', self.vcf_validator_path),
+                           ('vcf-assembly-checker', self.assembly_checker_path),
+                           ('biovalidator', self.biovalidator_path)]:
+            try:
+                self._run_quiet_command(
+                    f"Check {name} is installed and available on the path",
+                    f"{path} --version"
+                )
+            except subprocess.CalledProcessError as ex:
+                logger.error(ex)
+                raise RuntimeError(f"Please make sure {name} ({path}) is installed and available on the path")

eva_sub_cli/validators/validation_results_parsers.py

@@ -0,0 +1,210 @@
+import re
+
+from ebi_eva_common_pyutils.logger import logging_config
+
+logger = logging_config.get_logger(__name__)
+
+def parse_assembly_check_log(assembly_check_log):
+    error_list = []
+    nb_error, nb_mismatch = 0, 0
+    match = total = None
+    with open(assembly_check_log) as open_file:
+        for line in open_file:
+            if line.startswith('[error]'):
+                nb_error += 1
+                if nb_error < 11:
+                    error_list.append(line.strip()[len('[error] '):])
+            elif line.startswith('[info] Number of matches:'):
+                match, total = line.strip()[len('[info] Number of matches: '):].split('/')
+                match = int(match)
+                total = int(total)
+    return error_list, nb_error, match, total
+
+
+def parse_assembly_check_report(assembly_check_report):
+    mismatch_list = []
+    nb_mismatch = 0
+    nb_error = 0
+    error_list = []
+    with open(assembly_check_report) as open_file:
+        for line in open_file:
+            if 'does not match the reference sequence' in line:
+                nb_mismatch += 1
+                if nb_mismatch < 11:
+                    mismatch_list.append(line.strip())
+            elif 'Multiple synonyms' in line:
+                nb_error += 1
+                if nb_error < 11:
+                    error_list.append(line.strip())
+            # Contig not found in FASTA is reported here rather than in logs when no assembly report is used.
+            # Count and report once per contig name rather than once per line, to avoid redundant errors.
+            elif 'is not present in FASTA file' in line:
+                line_num, error_msg = line.split(': ')
+                error_msg = error_msg.strip()
+                if error_msg not in error_list:
+                    nb_error += 1
+                    if nb_error < 11:
+                        error_list.append(error_msg)
+    return mismatch_list, nb_mismatch, error_list, nb_error
+
+
+def parse_vcf_check_report(vcf_check_report):
+    valid = True
+    max_error_reported = 10
+    error_list, critical_list = [], []
+    warning_count = error_count = critical_count = 0
+    with open(vcf_check_report) as open_file:
+        for line in open_file:
+            if 'warning' in line:
+                warning_count = 1
+            elif line.startswith('According to the VCF specification'):
+                if 'not' in line:
+                    valid = False
+            elif vcf_check_errors_is_critical(line.strip()):
+                critical_count += 1
+                if critical_count <= max_error_reported:
+                    critical_list.append(line.strip())
+            else:
+                error_count += 1
+                if error_count <= max_error_reported:
+                    error_list.append(line.strip())
+
+    return valid, warning_count, error_count, critical_count, error_list, critical_list
+
+
+def vcf_check_errors_is_critical(error):
+    """
+    This function identify VCF check errors that are not critical for the processing of the VCF within EVA.
+    They affect specific INFO or FORMAT fields that are used in the variant detection but less so in the downstream
+    analysis.
+    Critical:
+    Reference and alternate alleles must not be the same.
+    Requested evidence presence with --require-evidence. Please provide genotypes (GT field in FORMAT and samples),
+    or allele frequencies (AF field in INFO), or allele counts (AC and AN fields in INFO)..
+    Contig is not sorted by position. Contig chr10 position 41695506 found after 41883113.
+    Duplicated variant chr1A:1106203:A>G found.
+    Metadata description string is not valid.
+
+    Error
+    Sample #10, field PL does not match the meta specification Number=G (expected 2 value(s)). PL=.. It must derive
+    its number of values from the ploidy of GT (if present), or assume diploidy. Contains 1 value(s), expected 2
+    (derived from ploidy 1).
+    Sample #102, field AD does not match the meta specification Number=R (expected 3 value(s)). AD=..
+    """
+    non_critical_format_fields = ['PL', 'AD', 'AC', 'GQ']
+    non_critical_info_fields = ['AC']
+    regexes = {
+        r'^INFO (\w+) does not match the specification Number': non_critical_info_fields,
+        r'^INFO (\w+) metadata Number is not ': non_critical_info_fields,
+        r'^Line \d+: Sample #\d+, field (\w+) does not match the meta specification Number=': non_critical_format_fields,
+        r'^Line \d+: FORMAT (\w+) metadata Type is not ': non_critical_format_fields,
+        r'^Line \d+: FORMAT (\w+) metadata Number is not ': non_critical_format_fields,
+        r'^Line \d+: INFO SVLEN must be equal to "length of ALT - length of REF" for non-symbolic alternate alleles. SVLEN=': None
+    }
+    for regex in regexes:
+        match = re.match(regex, error)
+        if match:
+            if regexes[regex] is None:
+                # No list of value to match against
+                return False
+            field_affected = match.group(1)
+            if field_affected in regexes[regex]:
+                return False
+    return True
+
+
+def parse_biovalidator_validation_results(metadata_check_file):
+    """
+    Read the biovalidator's report and extract the list of validation errors
+    """
+    ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
+
+    def clean_read(ifile):
+        l = ifile.readline()
+        if l:
+            return ansi_escape.sub('', l).strip()
+
+    if not metadata_check_file:
+        return
+
+    errors = []
+
+    with open(metadata_check_file) as open_file:
+        collect = False
+        while True:
+            line = clean_read(open_file)
+            if line is None:
+                break  # EOF
+            elif not line:
+                continue  # Empty line
+            if not collect:
+                if line.startswith('Validation failed with following error(s):'):
+                    collect = True
+            else:
+                while line and not line.startswith('/'):
+                    # Sometimes there are multiple (possibly redundant) errors listed under a single property,
+                    # we only report the first
+                    line = clean_read(open_file)
+                line2 = clean_read(open_file)
+                if line is None or line2 is None:
+                    break  # EOF
+                errors.append({'property': line, 'description': line2})
+    return errors
+
+
+def convert_metadata_sheet(json_attribute, xls2json_conf):
+    if json_attribute is None:
+        return None
+    for sheet_name in xls2json_conf['worksheets']:
+        if xls2json_conf['worksheets'][sheet_name] == json_attribute:
+            return sheet_name
+
+
+def convert_metadata_row(sheet, json_row, xls2json_conf):
+    if json_row is None:
+        return ''
+    if 'header_row' in xls2json_conf[sheet]:
+        return int(json_row) + xls2json_conf[sheet]['header_row']
+    else:
+        return int(json_row) + 2
+
+
+def convert_metadata_attribute(sheet, json_attribute, xls2json_conf):
+    if json_attribute is None:
+        return ''
+    attributes_dict = {}
+    attributes_dict.update(xls2json_conf[sheet].get('required', {}))
+    attributes_dict.update(xls2json_conf[sheet].get('optional', {}))
+    attributes_dict['Scientific Name'] = 'species'
+    attributes_dict['BioSample Name'] = 'name'
+
+    for attribute in attributes_dict:
+        if attributes_dict[attribute] == json_attribute:
+            return attribute
+
+
+def parse_metadata_property(property_str):
+    if property_str.startswith('.'):
+        return property_str.strip('./'), None, None
+    # First attempt to parse as BioSample object
+    sheet, row, col = parse_sample_metadata_property(property_str)
+    if sheet is not None and row is not None and col is not None:
+        return sheet, row, col
+    match = re.match(r'/(\w+)(/(\d+))?([./](\w+))?', property_str)
+    if match:
+        return match.group(1), match.group(3), match.group(5)
+    else:
+        logger.error(f'Cannot parse {property_str} in JSON metadata error')
+        return None, None, None
+
+
+def parse_sample_metadata_property(property_str):
+    # Check characteristics
+    match = re.match(r'/sample/(\d+)/bioSampleObject/characteristics/(\w+)', property_str)
+    if match:
+        return 'sample', match.group(1), match.group(2)
+    # Check name
+    match = re.match(r'/sample/(\d+)/bioSampleObject/name', property_str)
+    if match:
+        return 'sample', match.group(1), 'name'
+    return None, None, None