eva-sub-cli 0.4.4__py3-none-any.whl

eva_sub_cli/executables/check_metadata_semantics.py

@@ -0,0 +1,18 @@
+import argparse
+import json
+
+from eva_sub_cli.semantic_metadata import SemanticMetadataChecker
+
+
+def main():
+    arg_parser = argparse.ArgumentParser(description='Perform semantic checks on the metadata')
+    arg_parser.add_argument('--metadata_json', required=True, dest='metadata_json', help='EVA metadata json file')
+    arg_parser.add_argument('--output_yaml', required=True, dest='output_yaml',
+                            help='Path to the location of the results')
+    args = arg_parser.parse_args()
+
+    with open(args.metadata_json) as open_json:
+        metadata = json.load(open_json)
+        checker = SemanticMetadataChecker(metadata)
+        checker.check_all()
+        checker.write_result_yaml(args.output_yaml)

eva_sub_cli/executables/cli.py

@@ -0,0 +1,115 @@
+import sys
+
+import eva_sub_cli
+from eva_sub_cli.exceptions.submission_not_found_exception import SubmissionNotFoundException
+from eva_sub_cli.exceptions.submission_status_exception import SubmissionStatusException
+
+if not sys.warnoptions:
+    import warnings
+    warnings.simplefilter("ignore")
+
+import logging
+import os
+from argparse import ArgumentParser
+from ebi_eva_common_pyutils.logger import logging_config
+
+from eva_sub_cli import orchestrator
+from eva_sub_cli.orchestrator import VALIDATE, SUBMIT, DOCKER, NATIVE
+from eva_sub_cli.file_utils import is_submission_dir_writable, DirLockError, DirLock
+
+
+def validate_command_line_arguments(args, argparser):
+    if (args.vcf_files and not args.reference_fasta) or (not args.vcf_files and args.reference_fasta):
+        print("When using --vcf_files and --reference_fasta, both need to be specified")
+        argparser.print_usage()
+        sys.exit(1)
+
+    if SUBMIT in args.tasks and (
+            not (args.username or os.environ.get('ENAWEBINACCOUNT')) or
+            not (args.password or os.environ.get('ENAWEBINPASSWORD'))):
+        print("To submit your data, you need to provide a Webin username and password")
+        argparser.print_usage()
+        sys.exit(1)
+
+    if not is_submission_dir_writable(args.submission_dir):
+        print(f"'{args.submission_dir}' does not have write permissions or is not a directory.")
+        sys.exit(1)
+
+
+def parse_args(cmd_line_args):
+    argparser = ArgumentParser(prog='eva-sub-cli',
+                               description='EVA Submission CLI - validate and submit data to EVA. '
+                               'For full details, please see https://github.com/EBIvariation/eva-sub-cli')
+    argparser.add_argument('--version', action='version', version=f'%(prog)s {eva_sub_cli.__version__}')
+    argparser.add_argument('--submission_dir', required=True, type=str,
+                           help='Path to the directory where all processing is done and submission info is stored')
+    vcf_group = argparser.add_argument_group(
+        'Input VCF and assembly',
+        "Specify the VCF files and associated assembly with the following options. If you used different assemblies "
+        "for different VCF files, then you must include these in the metadata file rather than specifying them here."
+    )
+    vcf_group.add_argument('--vcf_files', nargs='+', help="One or more VCF files to validate")
+    vcf_group.add_argument('--reference_fasta',
+                           help="The FASTA file containing the reference genome from which the variants were derived")
+
+    metadata_group = argparser.add_argument_group('Metadata', 'Specify the metadata in a spreadsheet or in a JSON file')
+    metadata_group = metadata_group.add_mutually_exclusive_group(required=True)
+    metadata_group.add_argument("--metadata_json",
+                                help="JSON file that describes the project, analysis, samples and files")
+    metadata_group.add_argument("--metadata_xlsx",
+                                help="Excel spreadsheet that describes the project, analysis, samples and files")
+    argparser.add_argument('--tasks', nargs='+', choices=[VALIDATE, SUBMIT], default=[SUBMIT], type=str.lower,
+                           help='Select a task to perform (default SUBMIT). VALIDATE will run the validation'
+                                ' regardless of the outcome of previous runs. SUBMIT will run validate only if'
+                                ' the validation was not performed successfully before and then run the submission.')
+    argparser.add_argument('--executor', choices=[DOCKER, NATIVE], default=NATIVE, type=str.lower,
+                           help='Select the execution type for running validation (default native)')
+    credential_group = argparser.add_argument_group('Credentials', 'Specify the ENA Webin credentials you want to use '
+                                                                   'to submit to the EVA')
+    credential_group.add_argument("--username", help="Username for your ENA Webin account")
+    credential_group.add_argument("--password", help="Password for your ENA Webin account")
+    argparser.add_argument('--shallow', action='store_true', default=False, dest='shallow_validation',
+                           help='Set the validation to be performed on the first 10000 records of the VCF. '
+                                'Only applies if the number of records exceed 10000')
+    argparser.add_argument('--debug', action='store_true', default=False,
+                           help='Set the script to output debug messages')
+    args = argparser.parse_args(cmd_line_args)
+    validate_command_line_arguments(args, argparser)
+    return args
+
+
+def main():
+    exit_status = 0
+    args = parse_args(sys.argv[1:])
+
+    args.submission_dir = os.path.abspath(args.submission_dir)
+
+    if args.debug:
+        logging_config.add_stdout_handler(logging.DEBUG)
+    else:
+        logging_config.add_stdout_handler(logging.INFO)
+
+    try:
+        # lock the submission directory
+
+        with DirLock(os.path.join(args.submission_dir)) as lock:
+            # Create the log file
+            logging_config.add_file_handler(os.path.join(args.submission_dir, 'eva_submission.log'), logging.DEBUG)
+            # Pass on all the arguments to the orchestrator
+            orchestrator.orchestrate_process(**args.__dict__)
+    except DirLockError:
+        print(f'Could not acquire the lock file for {args.submission_dir} because another process is using this '
+              f'directory or a previous process did not terminate correctly. '
+              f'If the problem persists, remove the lock file manually.')
+        exit_status = 65
+    except FileNotFoundError as fne:
+        print(fne)
+        exit_status = 66
+    except SubmissionNotFoundException as snfe:
+        print(f'{snfe}. Please contact EVA Helpdesk')
+        exit_status = 67
+    except SubmissionStatusException as sse:
+        print(f'{sse}. Please try again later. If the problem persists, please contact EVA Helpdesk')
+        exit_status = 68
+    return exit_status
+

eva_sub_cli/executables/samples_checker.py

@@ -0,0 +1,144 @@
+import argparse
+import gzip
+import json
+import os
+
+from ebi_eva_common_pyutils.logger import logging_config
+
+import yaml
+
+from eva_sub_cli.file_utils import open_gzip_if_required
+from eva_sub_cli.metadata_utils import get_samples_per_analysis, get_files_per_analysis, get_analysis_for_vcf_file
+
+logger = logging_config.get_logger(__name__)
+
+
+def get_samples_from_vcf(vcf_file):
+    """
+    Get the list of samples present in a single VCF file
+    """
+    with open_gzip_if_required(vcf_file) as vcf_in:
+        for line in vcf_in:
+            if line.startswith('#CHROM'):
+                sp_line = line.strip().split('\t')
+                if len(sp_line) > 9:
+                    return sp_line[9:]
+                else:
+                    logger.warning(f"No Sample names found in file {vcf_file}")
+                    return []
+
+
+def compare_names_in_files_and_samples(sample_name_in_analysis, sample_name_per_file):
+    """
+    Compare the sample names provided in vcf files and the one provided in a set of sample rows.
+    This is meant to compare the samples and files provided for a single analysis.
+    """
+    has_difference = False
+    sample_names_in_vcf = set(
+        sample_name
+        for sample_name_list in sample_name_per_file.values()
+        for sample_name in sample_name_list
+    )
+    more_metadata_submitted_files = list(set(sample_name_in_analysis) - sample_names_in_vcf)
+    more_submitted_files_metadata = list(sample_names_in_vcf - set(sample_name_in_analysis))
+    more_per_submitted_files_metadata = {}
+    if more_submitted_files_metadata:
+        for file_name in sample_name_per_file:
+            more_per_submitted_files_metadata[os.path.basename(file_name)] = list(set(sample_name_per_file[file_name]) - set(sample_name_in_analysis))
+        has_difference = True
+
+    if more_metadata_submitted_files:
+        has_difference = True
+
+    return (has_difference, more_per_submitted_files_metadata, more_submitted_files_metadata,
+            more_metadata_submitted_files)
+
+
+def compare_all_analysis(samples_per_analysis, files_per_analysis):
+    overall_differences = False
+    results_per_analysis_alias = {}
+    all_analysis_alias = set(samples_per_analysis) | set(files_per_analysis)
+    for analysis_alias in all_analysis_alias:
+        sample_name_in_analysis = samples_per_analysis.get(analysis_alias, [])
+        sample_name_per_file = {
+            file_path: get_samples_from_vcf(file_path)
+            for file_path in files_per_analysis.get(analysis_alias, [])
+        }
+        (
+            has_difference, more_per_submitted_files_metadata,
+            more_submitted_files_metadata, more_metadata_submitted_files
+        ) = compare_names_in_files_and_samples(sample_name_in_analysis, sample_name_per_file)
+        overall_differences = overall_differences or has_difference
+        results_per_analysis_alias[analysis_alias] = {
+            'difference': has_difference,
+            'more_per_submitted_files_metadata': more_per_submitted_files_metadata,
+            'more_submitted_files_metadata': more_submitted_files_metadata,
+            'more_metadata_submitted_files': more_metadata_submitted_files
+        }
+    return overall_differences, results_per_analysis_alias
+
+
+def read_metadata_json(json_file):
+    with open(json_file) as open_json:
+        metadata = json.load(open_json)
+        return get_samples_per_analysis(metadata), get_files_per_analysis(metadata)
+
+
+def associate_vcf_path_with_analysis(vcf_files, files_per_analysis):
+    """
+    Match the files names associated with analysis provided in the metadata with the file path given on the command
+    line.
+    :param vcf_files the list of full path to the vcf files
+    :param files_per_analysis: dictionary of the analysis and their associated VCF file names
+    :returns dictionary of analysis and their associated vcf file path
+    """
+    result_files_per_analysis = dict()
+    for analysis in files_per_analysis:
+        result_files_per_analysis[analysis] = []
+    for vcf_file in vcf_files:
+        analysis_aliases = get_analysis_for_vcf_file(vcf_file, files_per_analysis)
+        if len(analysis_aliases) == 1:
+            result_files_per_analysis[analysis_aliases[0]].append(vcf_file)
+        elif len(analysis_aliases) == 0:
+            logger.error(f'No analysis found for vcf {vcf_file}')
+            if 'No analysis' not in result_files_per_analysis:
+                result_files_per_analysis['No analysis'] = []
+            result_files_per_analysis['No analysis'].append(vcf_file)
+        else:
+            logger.error(f'More than one analysis were match to vcf {vcf_file}')
+
+    return result_files_per_analysis
+
+
+def write_result_yaml(output_yaml, overall_differences, results_per_analysis_alias):
+    with open(output_yaml, 'w') as open_yaml:
+        yaml.safe_dump(data={
+            'overall_differences': overall_differences,
+            'results_per_analysis': results_per_analysis_alias
+        }, stream=open_yaml)
+
+
+def check_sample_name_concordance(metadata_json, vcf_files, output_yaml):
+    """
+    Take the metadata following EVA standard and formatted in JSON then compare the sample names in it to the ones
+    found in the VCF files
+    """
+    samples_per_analysis, files_per_analysis = read_metadata_json(metadata_json)
+    file_path_per_analysis = associate_vcf_path_with_analysis(vcf_files, files_per_analysis)
+    overall_differences, results_per_analysis_alias = compare_all_analysis(samples_per_analysis, file_path_per_analysis)
+    write_result_yaml(output_yaml, overall_differences, results_per_analysis_alias)
+
+
+def main():
+    arg_parser = argparse.ArgumentParser(
+        description='Compare the sample name in the VCF file and the one specified in the metadata.')
+    arg_parser.add_argument('--metadata_json', required=True, dest='metadata_json',
+                            help='EVA metadata json file')
+    arg_parser.add_argument('--vcf_files', dest='vcf_files', nargs='+',
+                            help='Path to the vcf files to compare to the metadata')
+    arg_parser.add_argument('--output_yaml', required=True, dest='output_yaml',
+                            help='Path to the location of the results')
+
+    args = arg_parser.parse_args()
+    logging_config.add_stdout_handler()
+    check_sample_name_concordance(args.metadata_json, args.vcf_files, args.output_yaml)

eva_sub_cli/executables/trim_down.py

@@ -0,0 +1,78 @@
+import argparse
+import os
+
+import yaml
+from ebi_eva_common_pyutils.logger import logging_config
+from eva_sub_cli.file_utils import open_gzip_if_required, fasta_iter
+
+logger = logging_config.get_logger(__name__)
+
+
+max_nb_lines = 10000
+
+
+def trim_down_vcf(vcf_file, output_vcf):
+    """
+    Produce a smaller vcf files containing a maximum of 10000 records
+    """
+    with open_gzip_if_required(vcf_file) as vcf_in, open(output_vcf, 'w') as vcf_out:
+        line_count = 0
+        ref_seq_names = set()
+        for line in vcf_in:
+            if line.startswith('#') or line_count < max_nb_lines:
+                vcf_out.write(line)
+                if not line.startswith('#'):
+                    line_count += 1
+                    ref_seq_names.add(line.split('\t')[0])
+            else:
+                break
+    if line_count != max_nb_lines:
+        logger.warning(f'Only {line_count} found in the source VCF {vcf_file} ')
+    return line_count, ref_seq_names
+
+
+def trim_down_fasta(fasta_file, output_fasta, ref_seq_names):
+    """
+    Produce a smaller fasta files containing only the reference sequences found in the VCF file
+    """
+    found_sequences = set()
+    with open(output_fasta, 'w') as fasta_out:
+        for header, sequence in fasta_iter(fasta_file):
+            name = header.split()[0]
+            if name in ref_seq_names:
+                found_sequences.add(name)
+                print(f'>{header}', file=fasta_out)
+                for i in range(0, len(sequence), 80):
+                    print(sequence[i:i+80], file=fasta_out)
+    return found_sequences
+
+
+def main():
+    arg_parser = argparse.ArgumentParser(
+        description=f'Take a VCF file and only keep {max_nb_lines} lines and remove unused fasta sequence from the '
+                    f'associated reference genome')
+    arg_parser.add_argument('--vcf_file', dest='vcf_file', required=True,
+                            help='Path to the vcf file to be trimmed down')
+    arg_parser.add_argument('--output_vcf_file', dest='output_vcf_file', required=True,
+                            help='Path to the output vcf file')
+    arg_parser.add_argument('--fasta_file', dest='fasta_file', required=True,
+                            help='Path to the fasta file to be trimmed down')
+    arg_parser.add_argument('--output_fasta_file', dest='output_fasta_file', required=True,
+                            help='Path to the output fasta file')
+    arg_parser.add_argument('--output_yaml_file', dest='output_yaml_file', required=True,
+                            help='Path to the yaml file containing the trim down metrics')
+
+    args = arg_parser.parse_args()
+    logging_config.add_stdout_handler()
+
+    line_count, ref_sequence = trim_down_vcf(args.vcf_file, args.output_vcf_file)
+    sequence_found = trim_down_fasta(args.fasta_file, args.output_fasta_file, ref_sequence)
+    trim_down_metrics = {'trim_down_vcf_record': line_count, 'number_sequence_found': sequence_found,
+                         'trim_down_required': line_count == max_nb_lines}
+    if len(sequence_found) != len(ref_sequence):
+        logger.warning(f'Not all sequences were found in the fasta file. Cancelling trimming down of fasta file')
+        os.link(args.fasta_file, args.output_fasta_file)
+        trim_down_metrics.pop('number_sequence_found')
+    with open(args.output_yaml_file) as open_file:
+        yaml.safe_dump(trim_down_metrics, open_file)
+